kafka-python 3.0.4__tar.gz → 3.0.6__tar.gz

{kafka_python-3.0.4 → kafka_python-3.0.6}/CHANGES.md

@@ -1,3 +1,26 @@
+# 3.0.6 (Jun 25, 2026)
+
+### Fixes
+* consumer: track current leader epoch in addition to record epoch (#3107)
+
+
+### CI
+* Also `wait_for_topics` on <0.10 test fixtures (#3105)
+
+# 3.0.5 (Jun 24, 2026)
+
+### Fixes
+* Reserve buffer capacity before every encode_into write (#3103)
+* Fix backoff on bootstrap init failure after connection_made (#3098)
+
+### CI
+* Bump actions/cache from 5 to 6 (#3099)
+* only save kafka dist cache on master (#3100)
+* Fix flakey test: test_send_during_backoff
+
+### Documentation
+* Add new extended logging args to docs/cli
+
 # 3.0.4 (Jun 23, 2026)
 
 ### Fixes

{kafka_python-3.0.4 → kafka_python-3.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 3.0.4
+Version: 3.0.6
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 License-Expression: Apache-2.0

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/consumer/subscription_state.py

@@ -564,6 +564,18 @@ class TopicPartitionState:
         # until OffsetForLeaderEpoch confirms the position is consistent with
         # the current leader's log; mutually exclusive with awaiting_reset.
         self._awaiting_validation = False
+        # Highest cluster leader epoch this position has already been reconciled
+        # against (a completed validation, or adoption of a position whose
+        # record epoch is unknown). This is deliberately distinct from the
+        # position's *record* epoch (``OffsetAndMetadata.leader_epoch``, which
+        # is sent as last_fetched_epoch for KIP-595 divergence detection): a
+        # leader election bumps the cluster epoch without changing the epoch of
+        # the record we're positioned on, and OffsetForLeaderEpoch reports the
+        # epoch of the requested (older) point, so the record epoch alone can't
+        # tell us whether we've validated against the current leader. Tracking
+        # it separately is what stops poll() from re-validating forever and
+        # stalling the consumer. See #3106.
+        self._current_leader_epoch = -1
         # KIP-392: preferred read replica chosen by the broker (rack-aware).
         # ``_preferred_read_replica_expiration`` is a monotonic deadline; after
         # it passes we fall back to the leader and re-learn. Cleared on
@@ -597,6 +609,7 @@ class TopicPartitionState:
         self._position = None
         self.next_allowed_retry_time = None
         self._awaiting_validation = False
+        self._current_leader_epoch = -1
         self.clear_preferred_read_replica()
 
     def is_reset_allowed(self):
@@ -625,6 +638,11 @@ class TopicPartitionState:
         self.drop_pending_record_batch = True
         self.next_allowed_retry_time = None
         self._awaiting_validation = False
+        # A freshly-seeked position has not been reconciled against any leader;
+        # force the next maybe_validate_position to validate it against the
+        # current leader (the prior position's validation says nothing about
+        # this offset).
+        self._current_leader_epoch = -1
         self.clear_preferred_read_replica()
 
     def pause(self):
@@ -686,7 +704,8 @@ class TopicPartitionState:
         return self._awaiting_validation
 
     def maybe_validate_position(self, current_leader_epoch):
-        """Mark for validation if current leader has advanced beyond our position's epoch.
+        """Mark for validation if the current leader has advanced beyond the
+        leader epoch this position was last reconciled against.
 
         Returns True if the partition is now awaiting validation.
         """
@@ -696,11 +715,26 @@ class TopicPartitionState:
             return False
         if current_leader_epoch is None or current_leader_epoch < 0:
             return False
-        # Positions without a known epoch (legacy data, post-seek to bare offset)
-        # can't be validated; treat as already-fetchable.
-        if self._position.leader_epoch < 0:
+        # Have we already reconciled this position against this leader epoch (or
+        # a newer one)? The floor is the max of:
+        #  - the record epoch of the position itself: a record we actually
+        #    fetched at this offset proves the position is valid through its
+        #    epoch (so a position fetched under the current leader needs no
+        #    validation), and
+        #  - the leader epoch a completed validation reconciled us against,
+        #    which can exceed the record epoch because OffsetForLeaderEpoch
+        #    reports the epoch of the requested (older) point, not the current
+        #    leader's. Without this second term the position is re-marked every
+        #    poll and the partition never becomes fetchable again (#3106).
+        reconciled_epoch = max(self._current_leader_epoch, self._position.leader_epoch)
+        if current_leader_epoch <= reconciled_epoch:
             return False
-        if self._position.leader_epoch >= current_leader_epoch:
+        # We are now tracking this leader epoch regardless of whether the
+        # position itself carries a record epoch we can validate against.
+        self._current_leader_epoch = current_leader_epoch
+        # Positions without a known record epoch (legacy data, post-seek to bare
+        # offset) can't be validated; treat as already-fetchable.
+        if self._position.leader_epoch < 0:
             return False
         self.clear_preferred_read_replica()
         self._awaiting_validation = True

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/net/manager.py

@@ -142,6 +142,9 @@ class KafkaConnectionManager:
                 raise
             except Exception as exc:
                 self._conns.pop(bootstrap_broker.node_id, conn).close(exc)
+                backoff_ms = self.update_backoff(bootstrap_broker.node_id)
+                log.warning('Bootstrap connection to %s failed: %s (backoff %.2f secs)',
+                            bootstrap_broker.node_id, exc, backoff_ms / 1000)
                 continue
 
             try:
@@ -258,6 +261,8 @@ class KafkaConnectionManager:
                 return
             conn.connection_made(transport)
             transport = None  # conn owns cleanup now; skip finally: transport.close()
+            # Note: conn.initialize does not currently raise on error;
+            # errors are pushed to conn.init_future and raised on await conn
             await conn.initialize(timeout_at=timeout_at)
         except Exception as exc:
             log.error('Connection failed: %s', exc)

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/protocol/schemas/fields/array.py

@@ -72,9 +72,10 @@ class ArrayField(BaseField):
         if compact:
             an = ctx.next_var('an')
             ctx.emit(indent, '%s = len(%s) + 1 if %s is not None else 0' % (an, val_expr, val_expr))
-            UnsignedVarInt32.emit_encode_into(ctx, an, indent)
+            UnsignedVarInt32.emit_encode_into(ctx, an, indent)  # reserves the length varint
             ctx.emit(indent, 'if %s is not None:' % val_expr)
         else:
+            ctx.emit_reserve(indent, 4)
             ctx.emit(indent, 'if %s is None:' % val_expr)
             ctx.emit(indent, "    pack_into('>i', buf, pos, -1)")
             ctx.emit(indent, '    pos += 4')
@@ -83,9 +84,23 @@ class ArrayField(BaseField):
             ctx.emit(indent, '    pos += 4')
         guard = indent + '    '
         item_var = ctx.next_var('ai')
-        ctx.emit(guard, 'for %s in %s:' % (item_var, val_expr))
-        self.array_of.emit_encode_into(ctx, item_var, guard + '    ',
-                                       version=version, compact=compact, tagged=tagged)
+        elem = self.array_of
+        if isinstance(elem, SimpleField) and elem.is_batchable():
+            # Fast path: array of fixed-size primitives (e.g. []int32). Reserve
+            # the whole run with a single ensure, then write each element inline
+            # without a per-element capacity check. The reserve sits inside the
+            # `is not None` branch (compact) / `else` branch (non-compact), so
+            # len() is always safe here.
+            be_fmt = elem._type._be_fmt
+            size = elem._type.size
+            ctx.emit_reserve(guard, '%d * len(%s)' % (size, val_expr))
+            ctx.emit(guard, 'for %s in %s:' % (item_var, val_expr))
+            ctx.emit(guard, "    pack_into('%s', buf, pos, %s)" % (be_fmt, item_var))
+            ctx.emit(guard, '    pos += %d' % size)
+        else:
+            ctx.emit(guard, 'for %s in %s:' % (item_var, val_expr))
+            elem.emit_encode_into(ctx, item_var, guard + '    ',
+                                  version=version, compact=compact, tagged=tagged)
 
     def emit_decode_from(self, ctx, var_name, indent, version=None, compact=False, tagged=False):
         n = ctx.next_var('n')

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/protocol/schemas/fields/codecs/encode_buffer.py

@@ -2,7 +2,35 @@ import threading
 
 
 class EncodeBuffer:
-    """Growable buffer for encode_into operations."""
+    """Growable byte buffer for the ``encode_into`` fast path.
+
+    The encoders write primitives directly into ``buf`` at offset ``pos``
+    rather than building and joining intermediate ``bytes`` objects. The
+    buffer starts at a fixed size and grows on demand via :meth:`ensure`.
+
+    Capacity contract
+    -----------------
+    Writing past the end of ``buf`` does not grow it automatically:
+
+      * single-byte index writes (``buf[pos] = x``) raise ``IndexError``,
+      * ``pack_into`` raises ``struct.error``,
+      * slice assignment (``buf[pos:pos+n] = data``) silently *resizes* the
+        bytearray, defeating the preallocation.
+
+    Therefore **every writer must call** ``ensure(n)`` to reserve ``n`` bytes
+    before writing ``n`` bytes at ``pos`` (where ``n`` is the maximum the write
+    can consume - e.g. ``5`` for a varint32, a fixed field's ``size``, or
+    ``len(payload)`` for variable data). See the codecs in ``types.py`` for the
+    pattern, and ``CodegenContext.emit_reserve`` for the compiled equivalent.
+
+    Reallocation note
+    -----------------
+    ``ensure`` may replace ``buf`` with a larger bytearray. Any caller (or
+    generated code) that caches ``buf`` in a local **must re-read** ``self.buf``
+    after a call that can grow it - including indirect growth through a nested
+    ``encode_into`` / ``ensure``. Forgetting to re-read leaves writes targeting
+    the old, discarded buffer (silent data loss) or raises out of range.
+    """
     __slots__ = ('buf', 'pos')
 
     def __init__(self, size=65536):
@@ -14,6 +42,16 @@ class EncodeBuffer:
         self.pos = 0
 
     def ensure(self, needed):
+        """Guarantee at least ``needed`` writable bytes remain at ``pos``.
+
+        Call this *before* writing ``needed`` bytes at ``self.pos``. If the
+        current buffer cannot hold them it is grown (at least doubled) and the
+        existing ``[:pos]`` content is preserved.
+
+        WARNING: this may rebind ``self.buf`` to a new bytearray, so re-read
+        ``self.buf`` afterwards if you hold a local reference to it (see the
+        class docstring).
+        """
         if self.pos + needed > len(self.buf):
             new_size = max(len(self.buf) * 2, self.pos + needed)
             new_buf = bytearray(new_size)
@@ -21,6 +59,7 @@ class EncodeBuffer:
             self.buf = new_buf
 
     def result(self):
+        """Return the encoded bytes written so far (``buf[:pos]``)."""
         # Return a bytearray slice (one copy) rather than bytes(self.buf[:pos])
         # (two copies - the slice creates a bytearray, then bytes() copies
         # again). Downstream consumers (protocol codec slice assignment,

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/protocol/schemas/fields/codecs/tagged_fields.py

@@ -60,6 +60,7 @@ class TaggedFields:
             # Encode value into buffer after reserving space for size prefix.
             # Strategy: assume size fits in 1 byte (< 128), encode value,
             # then fix up if the size varint is larger.
+            out.ensure(1)  # reserve the assumed 1-byte size prefix
             size_pos = out.pos
             out.pos += 1  # reserve 1 byte for size
             val_start = out.pos

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/protocol/schemas/fields/codecs/types.py

@@ -26,6 +26,7 @@ class FixedCodec:
 
     @classmethod
     def encode_into(cls, out, value, compact=False):
+        out.ensure(cls.size)
         pack_into(cls._be_fmt, out.buf, out.pos, value)
         out.pos += cls.size
 
@@ -40,6 +41,7 @@ class FixedCodec:
 
     @classmethod
     def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
+        ctx.emit_reserve(indent, cls.size)
         ctx.emit(indent, "pack_into('%s', buf, pos, %s)" % (cls._be_fmt, val_expr))
         ctx.emit(indent, 'pos += %d' % cls.size)
 
@@ -101,6 +103,7 @@ class UUID:
             b = value.bytes
         else:
             b = uuid.UUID(value).bytes
+        out.ensure(16)
         pos = out.pos
         out.buf[pos:pos+16] = b
         out.pos = pos + 16
@@ -111,6 +114,7 @@ class UUID:
         v = ctx.next_var('uv')
         ctx.emit(indent, '%s = %s' % (v, val_expr))
         ctx.emit(indent, 'if %s is None: %s = _ZERO_UUID' % (v, v))
+        ctx.emit_reserve(indent, 16)
         ctx.emit(indent, 'buf[pos:pos+16] = %s.bytes if hasattr(%s, "bytes") else __import__("uuid").UUID(%s).bytes' % (v, v, v))
         ctx.emit(indent, 'pos += 16')
 
@@ -151,43 +155,52 @@ class String:
     def encode_into(self, out, value, compact=False):
         if compact:
             if value is None:
-                UnsignedVarInt32.encode_into(out,0)
+                UnsignedVarInt32.encode_into(out, 0)  # ensures internally
                 return
             value = str(value).encode(self.encoding)
-            UnsignedVarInt32.encode_into(out,len(value) + 1)
+            UnsignedVarInt32.encode_into(out, len(value) + 1)  # ensures internally
         else:
             if value is None:
+                out.ensure(2)
                 pack_into('>h', out.buf, out.pos, -1)
                 out.pos += 2
                 return
             value = str(value).encode(self.encoding)
+            out.ensure(2)
             pack_into('>h', out.buf, out.pos, len(value))
             out.pos += 2
         n = len(value)
+        out.ensure(n)
         pos = out.pos
         out.buf[pos:pos+n] = value
         out.pos = pos + n
 
     def emit_encode_into(self, ctx, val_expr, indent, compact=False):
         sv = ctx.next_var('sv')
+        body = indent + '    '
         ctx.emit(indent, 'if %s is None:' % val_expr)
         if compact:
+            ctx.emit_reserve(body, 1)
             ctx.emit(indent, '    buf[pos] = 0')
             ctx.emit(indent, '    pos += 1')
             ctx.emit(indent, 'else:')
             sn = ctx.next_var('sn')
             ctx.emit(indent, '    %s = str(%s).encode("utf-8")' % (sv, val_expr))
             ctx.emit(indent, '    %s = len(%s) + 1' % (sn, sv))
-            UnsignedVarInt32.emit_encode_into(ctx, sn, indent + '    ')
+            UnsignedVarInt32.emit_encode_into(ctx, sn, body)  # reserves the length varint
+            ctx.emit_reserve(body, 'len(%s)' % sv)
             ctx.emit(indent, '    buf[pos:pos+len(%s)] = %s' % (sv, sv))
             ctx.emit(indent, '    pos += len(%s)' % sv)
         else:
+            ctx.emit_reserve(body, 2)
             ctx.emit(indent, "    pack_into('>h', buf, pos, -1)")
             ctx.emit(indent, '    pos += 2')
             ctx.emit(indent, 'else:')
             ctx.emit(indent, '    %s = str(%s).encode("utf-8")' % (sv, val_expr))
+            ctx.emit_reserve(body, 2)
             ctx.emit(indent, "    pack_into('>h', buf, pos, len(%s))" % sv)
             ctx.emit(indent, '    pos += 2')
+            ctx.emit_reserve(body, 'len(%s)' % sv)
             ctx.emit(indent, '    buf[pos:pos+len(%s)] = %s' % (sv, sv))
             ctx.emit(indent, '    pos += len(%s)' % sv)
 
@@ -240,14 +253,16 @@ class Bytes:
             value = value.encode()
         if compact:
             if value is None:
-                UnsignedVarInt32.encode_into(out, 0)
+                UnsignedVarInt32.encode_into(out, 0)  # ensures internally
                 return
-            UnsignedVarInt32.encode_into(out, len(value) + 1)
+            UnsignedVarInt32.encode_into(out, len(value) + 1)  # ensures internally
         else:
             if value is None:
+                out.ensure(4)
                 pack_into('>i', out.buf, out.pos, -1)
                 out.pos += 4
                 return
+            out.ensure(4)
             pack_into('>i', out.buf, out.pos, len(value))
             out.pos += 4
         n = len(value)
@@ -260,35 +275,35 @@ class Bytes:
     def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
         bv = ctx.next_var('bv')
         bn = ctx.next_var('bn')
+        body = indent + '    '
         if compact:
             ctx.emit(indent, '%s = %s' % (bv, val_expr))
             ctx.emit(indent, 'if %s is not None and not isinstance(%s, (bytes, bytearray, memoryview)): %s = %s.encode()' % (bv, bv, bv, bv))
             ctx.emit(indent, 'if %s is None:' % bv)
+            ctx.emit_reserve(body, 1)
             ctx.emit(indent, '    buf[pos] = 0')
             ctx.emit(indent, '    pos += 1')
             ctx.emit(indent, 'else:')
             ctx.emit(indent, '    %s = len(%s)' % (bn, bv))
             sn = ctx.next_var('sn')
             ctx.emit(indent, '    %s = %s + 1' % (sn, bn))
-            UnsignedVarInt32.emit_encode_into(ctx, sn, indent + '    ')
-            ctx.emit(indent, '    out.pos = pos')
-            ctx.emit(indent, '    out.ensure(%s)' % bn)
-            ctx.emit(indent, '    buf = out.buf')
+            UnsignedVarInt32.emit_encode_into(ctx, sn, body)  # reserves the length varint
+            ctx.emit_reserve(body, bn)
             ctx.emit(indent, '    buf[pos:pos+%s] = %s' % (bn, bv))
             ctx.emit(indent, '    pos += %s' % bn)
         else:
             ctx.emit(indent, '%s = %s' % (bv, val_expr))
             ctx.emit(indent, 'if %s is not None and not isinstance(%s, (bytes, bytearray, memoryview)): %s = %s.encode()' % (bv, bv, bv, bv))
             ctx.emit(indent, 'if %s is None:' % bv)
+            ctx.emit_reserve(body, 4)
             ctx.emit(indent, "    pack_into('>i', buf, pos, -1)")
             ctx.emit(indent, '    pos += 4')
             ctx.emit(indent, 'else:')
             ctx.emit(indent, '    %s = len(%s)' % (bn, bv))
+            ctx.emit_reserve(body, 4)
             ctx.emit(indent, "    pack_into('>i', buf, pos, %s)" % bn)
             ctx.emit(indent, '    pos += 4')
-            ctx.emit(indent, '    out.pos = pos')
-            ctx.emit(indent, '    out.ensure(%s)' % bn)
-            ctx.emit(indent, '    buf = out.buf')
+            ctx.emit_reserve(body, bn)
             ctx.emit(indent, '    buf[pos:pos+%s] = %s' % (bn, bv))
             ctx.emit(indent, '    pos += %s' % bn)
 
@@ -341,6 +356,7 @@ class UnsignedVarInt32:
 
     @classmethod
     def encode_into(cls, out, value):
+        out.ensure(5)  # a varint32 is at most 5 bytes
         buf = out.buf
         pos = out.pos
         while (value & 0xffffff80) != 0:
@@ -352,6 +368,7 @@ class UnsignedVarInt32:
 
     @classmethod
     def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
+        ctx.emit_reserve(indent, 5)  # a varint32 is at most 5 bytes
         ctx.emit(indent, 'while (%s & 0xffffff80) != 0:' % val_expr)
         ctx.emit(indent, '    buf[pos] = (%s & 0x7f) | 0x80' % val_expr)
         ctx.emit(indent, '    %s >>= 7' % val_expr)
@@ -462,6 +479,7 @@ class BitField:
     def encode_into(cls, out, vals, compact=False):
         if vals is None:
             vals = {31}
+        out.ensure(4)
         pack_into('>I', out.buf, out.pos, cls.to_32_bit_field(vals))
         out.pos += 4
 
@@ -473,6 +491,7 @@ class BitField:
         ctx.emit(indent, 'if %s is None: %s = {31}' % (bf, bf))
         ctx.emit(indent, '%s = 0' % bfi)
         ctx.emit(indent, 'for _b in %s: %s |= 1 << _b' % (bf, bfi))
+        ctx.emit_reserve(indent, 4)
         ctx.emit(indent, "pack_into('>I', buf, pos, %s)" % bfi)
         ctx.emit(indent, 'pos += 4')
 

kafka_python-3.0.6/kafka/protocol/schemas/fields/codegen.py

@@ -0,0 +1,71 @@
+"""Generate flat encode/decode functions for a StructField + version.
+
+Given a StructField and a protocol version, generates Python functions
+that encode/decode directly with zero dispatch overhead - no intermediate
+SimpleField/ArrayField/StructField method calls.
+
+Usage:
+    from kafka.protocol.schemas.fields.codegen import CodegenContext
+    # Encode: see StructField.compiled_encode_into()
+    # Decode: see StructField.compiled_decode_from()
+"""
+
+from struct import pack_into, unpack_from
+
+
+class CodegenContext:
+    """Shared state for code generation."""
+
+    def __init__(self):
+        self.lines = []
+        self.globs = {'pack_into': pack_into, 'unpack_from': unpack_from}
+        self._var_counter = 0
+
+    def next_var(self, prefix='v'):
+        self._var_counter += 1
+        return f'_{prefix}{self._var_counter}'
+
+    def emit(self, indent, line):
+        self.lines.append(f'{indent}{line}')
+
+    def emit_reserve(self, indent, nbytes):
+        """Emit an inline capacity check before a write of up to ``nbytes`` bytes.
+
+        Generated encode functions keep three locals in sync:
+
+          * ``buf``  - the destination bytearray (``out.buf``),
+          * ``pos``  - the current write offset,
+          * ``_cap`` - ``len(buf)``, the cached capacity.
+
+        These are not set up per fragment: they are declared once by the
+        ``def _encode(item, out):`` preamble in
+        ``StructField.encode_into__optimized_context`` (the sole generator of
+        the compiled encode function), and every emitted fragment - including
+        this one - is spliced into that function body. Emit fragments are
+        therefore only valid inside that body; they cannot stand alone.
+
+        ``nbytes`` is the MAXIMUM number of bytes the following write can
+        consume (an ``int`` for fixed/varint fields, or a string expression
+        such as ``'len(_bv1)'`` for a variable payload). The fast path is a
+        single comparison; only on overflow do we sync ``out.pos``, grow via
+        :meth:`EncodeBuffer.ensure`, and re-read the (possibly reallocated)
+        buffer back into ``buf``/``_cap``.
+
+        Because :meth:`EncodeBuffer.ensure` may rebind ``out.buf``, code that
+        instead delegates to a runtime ``encode_into`` (e.g. tagged fields)
+        must re-bind ``buf``/``_cap`` itself afterwards - ``emit_reserve`` only
+        covers writes emitted inline.
+        """
+        self.emit(indent, 'if pos + %s > _cap:' % nbytes)
+        self.emit(indent, '    out.pos = pos; out.ensure(%s); buf = out.buf; _cap = len(buf)' % nbytes)
+
+    def source(self):
+        return '\n'.join(self.lines)
+
+    def print(self):
+        print('GLOBALS:')
+        for var in self.globs:
+            print(f'     {var}={self.globs[var]}')
+        print('\nSOURCE:')
+        for i, line in enumerate(self.lines):
+            print(f'{i+1:<4} {line}')

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/protocol/schemas/fields/struct.py

@@ -116,6 +116,7 @@ class StructField(BaseField):
             and not tuple_access
         )
         if inline_nullable:
+            ctx.emit_reserve(indent, 1)
             ctx.emit(indent, 'if %s is None:' % item_expr)
             ctx.emit(indent, '    buf[pos] = 0xff')
             ctx.emit(indent, '    pos += 1')
@@ -136,8 +137,13 @@ class StructField(BaseField):
             ctx.globs[tf_var] = self.tagged_fields(version)
             ctx.emit(indent, 'out.pos = pos')
             ctx.emit(indent, '%s.encode_into(%s, out, version=%d)' % (tf_var, item_expr, version))
+            # TaggedFields.encode_into may grow (reallocate) out.buf, so re-bind
+            # the cached buf/_cap locals before any further inline writes.
             ctx.emit(indent, 'pos = out.pos')
+            ctx.emit(indent, 'buf = out.buf')
+            ctx.emit(indent, '_cap = len(buf)')
         elif tagged is None:
+            ctx.emit_reserve(indent, 1)
             ctx.emit(indent, 'buf[pos] = 0')
             ctx.emit(indent, 'pos += 1')
 
@@ -170,9 +176,13 @@ class StructField(BaseField):
     def encode_into__optimized_context(self, version, compact=False, tagged=False):
         ctx = CodegenContext()
         indent = '    '
+        # Preamble: declare the buf/pos/_cap locals that every emitted fragment
+        # relies on (see CodegenContext.emit_reserve). Fragments are spliced
+        # into this function body and are not valid outside it.
         ctx.lines.append('def _encode(item, out):')
         ctx.emit(indent, 'buf = out.buf')
         ctx.emit(indent, 'pos = out.pos')
+        ctx.emit(indent, '_cap = len(buf)  # cached len(buf); kept in sync by emit_reserve')
         self.emit_encode_into(ctx, 'item', indent, version=version, compact=compact, tagged=tagged)
         ctx.emit(indent, 'out.pos = pos')
         return ctx

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka/protocol/schemas/fields/struct_array.py

@@ -92,9 +92,10 @@ class StructArrayField(ArrayField):
         if compact:
             an = ctx.next_var('an')
             ctx.emit(indent, '%s = len(%s) + 1 if %s is not None else 0' % (an, val_expr, val_expr))
-            UnsignedVarInt32.emit_encode_into(ctx, an, indent)
+            UnsignedVarInt32.emit_encode_into(ctx, an, indent)  # reserves the length varint
             ctx.emit(indent, 'if %s is not None:' % val_expr)
         else:
+            ctx.emit_reserve(indent, 4)
             ctx.emit(indent, 'if %s is None:' % val_expr)
             ctx.emit(indent, "    pack_into('>i', buf, pos, -1)")
             ctx.emit(indent, '    pos += 4')
@@ -118,8 +119,12 @@ class StructArrayField(ArrayField):
                 ctx.emit(scalar_indent, 'out.pos = pos')
                 ctx.emit(scalar_indent, '# tagged fields for single-field struct')
                 ctx.emit(scalar_indent, '%s.encode_into(%s, out, version=%d)' % (tf_var, item_var, version))
+                # encode_into may reallocate out.buf - re-bind cached locals.
                 ctx.emit(scalar_indent, 'pos = out.pos')
+                ctx.emit(scalar_indent, 'buf = out.buf')
+                ctx.emit(scalar_indent, '_cap = len(buf)')
             elif tagged is None:
+                ctx.emit_reserve(scalar_indent, 1)
                 ctx.emit(scalar_indent, 'buf[pos] = 0')
                 ctx.emit(scalar_indent, 'pos += 1')
         else:

kafka_python-3.0.6/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '3.0.6'

{kafka_python-3.0.4 → kafka_python-3.0.6}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 3.0.4
+Version: 3.0.6
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 License-Expression: Apache-2.0

{kafka_python-3.0.4 → kafka_python-3.0.6}/test/integration/fixtures.py

@@ -34,21 +34,29 @@ def create_topics(broker, topic_names, num_partitions=None, replication_factor=N
         for topic_name in topic_names:
             # TODO: verify kafka-topics.sh support for early 0.8 brokers
             broker._create_topic_via_cli(topic_name, num_partitions, replication_factor)
+        _wait_for_topics(broker, topic_names)
 
 
 def _create_topics_via_admin(broker, topic_names, num_partitions, replication_factor):
-    from kafka.admin import NewTopic
-    params = broker._enrich_client_params({}, client_id='topic_creator')
-    admin = KafkaAdminClient(**params)
-    try:
-        topics = [NewTopic(name, num_partitions, replication_factor) for name in topic_names]
-        admin.create_topics(topics, wait_for_metadata=True)
-    except InvalidReplicationFactorError:
-        time.sleep(0.5)
-        topics = [NewTopic(name, num_partitions, replication_factor) for name in topic_names]
-        admin.create_topics(topics, wait_for_metadata=True)
-    finally:
-        admin.close()
+    params = broker._enrich_client_params({'api_version': env_kafka_version(), 'client_id': 'topic_creator'})
+    with KafkaAdminClient(**params) as admin:
+        try:
+            topics = {name: {'num_partitions': num_partitions, 'replication_factor': replication_factor}
+                      for name in topic_names}
+            admin.create_topics(topics, wait_for_metadata=True)
+        except InvalidReplicationFactorError:
+            # wait and try again
+            # in CI the brokers sometimes take a while to find themselves
+            time.sleep(0.5)
+            topics = {name: {'num_partitions': num_partitions, 'replication_factor': replication_factor}
+                      for name in topic_names}
+            admin.create_topics(topics, wait_for_metadata=True)
+
+
+def _wait_for_topics(broker, topics):
+    params = broker._enrich_client_params({'api_version': env_kafka_version(), 'client_id': 'topic_creator'})
+    with KafkaAdminClient(**params) as admin:
+        admin.wait_for_topics(topics)
 
 
 def client_params(broker, client_id='client', **overrides):