kafka-python 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kafka/__init__.py +34 -0
- kafka/__main__.py +5 -0
- kafka/admin/__init__.py +29 -0
- kafka/admin/__main__.py +5 -0
- kafka/admin/_acls.py +355 -0
- kafka/admin/_cluster.py +359 -0
- kafka/admin/_configs.py +479 -0
- kafka/admin/_groups.py +754 -0
- kafka/admin/_partitions.py +595 -0
- kafka/admin/_topics.py +281 -0
- kafka/admin/_transactions.py +450 -0
- kafka/admin/_users.py +194 -0
- kafka/admin/client.py +373 -0
- kafka/benchmarks/__init__.py +0 -0
- kafka/benchmarks/consumer_performance.py +138 -0
- kafka/benchmarks/load_example.py +109 -0
- kafka/benchmarks/producer_encode_path.py +201 -0
- kafka/benchmarks/producer_performance.py +161 -0
- kafka/benchmarks/profile_protocol.py +138 -0
- kafka/benchmarks/protocol_old_vs_new.py +447 -0
- kafka/benchmarks/record_batch_compose.py +77 -0
- kafka/benchmarks/record_batch_read.py +82 -0
- kafka/benchmarks/varint_speed.py +426 -0
- kafka/cli/__init__.py +36 -0
- kafka/cli/admin/__init__.py +117 -0
- kafka/cli/admin/acls/__init__.py +9 -0
- kafka/cli/admin/acls/common.py +76 -0
- kafka/cli/admin/acls/create.py +19 -0
- kafka/cli/admin/acls/delete.py +23 -0
- kafka/cli/admin/acls/describe.py +16 -0
- kafka/cli/admin/cluster/__init__.py +14 -0
- kafka/cli/admin/cluster/describe.py +11 -0
- kafka/cli/admin/cluster/describe_quorum.py +11 -0
- kafka/cli/admin/cluster/features.py +52 -0
- kafka/cli/admin/cluster/log_dirs.py +43 -0
- kafka/cli/admin/cluster/versions.py +33 -0
- kafka/cli/admin/configs/__init__.py +10 -0
- kafka/cli/admin/configs/alter.py +43 -0
- kafka/cli/admin/configs/common.py +17 -0
- kafka/cli/admin/configs/describe.py +30 -0
- kafka/cli/admin/configs/list.py +16 -0
- kafka/cli/admin/configs/reset.py +20 -0
- kafka/cli/admin/groups/__init__.py +16 -0
- kafka/cli/admin/groups/alter_offsets.py +30 -0
- kafka/cli/admin/groups/delete.py +11 -0
- kafka/cli/admin/groups/delete_offsets.py +29 -0
- kafka/cli/admin/groups/describe.py +11 -0
- kafka/cli/admin/groups/list.py +28 -0
- kafka/cli/admin/groups/list_offsets.py +29 -0
- kafka/cli/admin/groups/remove_members.py +40 -0
- kafka/cli/admin/groups/reset_offsets.py +139 -0
- kafka/cli/admin/partitions/__init__.py +21 -0
- kafka/cli/admin/partitions/alter_reassignments.py +37 -0
- kafka/cli/admin/partitions/create.py +27 -0
- kafka/cli/admin/partitions/delete_records.py +31 -0
- kafka/cli/admin/partitions/describe.py +36 -0
- kafka/cli/admin/partitions/elect_leaders.py +53 -0
- kafka/cli/admin/partitions/list_offsets.py +88 -0
- kafka/cli/admin/partitions/list_reassignments.py +35 -0
- kafka/cli/admin/topics/__init__.py +10 -0
- kafka/cli/admin/topics/create.py +13 -0
- kafka/cli/admin/topics/delete.py +19 -0
- kafka/cli/admin/topics/describe.py +18 -0
- kafka/cli/admin/topics/list.py +11 -0
- kafka/cli/admin/transactions/__init__.py +17 -0
- kafka/cli/admin/transactions/abort.py +38 -0
- kafka/cli/admin/transactions/describe.py +24 -0
- kafka/cli/admin/transactions/describe_producers.py +29 -0
- kafka/cli/admin/transactions/find_hanging.py +26 -0
- kafka/cli/admin/transactions/list.py +37 -0
- kafka/cli/admin/users/__init__.py +8 -0
- kafka/cli/admin/users/alter_user_scram_credentials.py +34 -0
- kafka/cli/admin/users/describe_user_scram_credentials.py +15 -0
- kafka/cli/common.py +95 -0
- kafka/cli/consumer/__init__.py +63 -0
- kafka/cli/producer/__init__.py +57 -0
- kafka/cluster.py +824 -0
- kafka/codec.py +325 -0
- kafka/consumer/__init__.py +5 -0
- kafka/consumer/__main__.py +5 -0
- kafka/consumer/fetcher.py +2012 -0
- kafka/consumer/group.py +1347 -0
- kafka/consumer/subscription_state.py +897 -0
- kafka/coordinator/__init__.py +0 -0
- kafka/coordinator/assignors/__init__.py +0 -0
- kafka/coordinator/assignors/abstract.py +90 -0
- kafka/coordinator/assignors/cooperative_sticky.py +167 -0
- kafka/coordinator/assignors/range.py +81 -0
- kafka/coordinator/assignors/roundrobin.py +101 -0
- kafka/coordinator/assignors/sticky/StickyAssignorUserData.json +37 -0
- kafka/coordinator/assignors/sticky/__init__.py +0 -0
- kafka/coordinator/assignors/sticky/partition_movements.py +149 -0
- kafka/coordinator/assignors/sticky/sorted_set.py +63 -0
- kafka/coordinator/assignors/sticky/sticky_assignor.py +665 -0
- kafka/coordinator/assignors/sticky/user_data.py +8 -0
- kafka/coordinator/base.py +1215 -0
- kafka/coordinator/consumer.py +1224 -0
- kafka/coordinator/heartbeat.py +82 -0
- kafka/coordinator/subscription.py +34 -0
- kafka/errors.py +1004 -0
- kafka/future.py +166 -0
- kafka/metrics/__init__.py +13 -0
- kafka/metrics/compound_stat.py +33 -0
- kafka/metrics/dict_reporter.py +81 -0
- kafka/metrics/kafka_metric.py +36 -0
- kafka/metrics/measurable.py +27 -0
- kafka/metrics/measurable_stat.py +13 -0
- kafka/metrics/metric_config.py +33 -0
- kafka/metrics/metric_name.py +105 -0
- kafka/metrics/metrics.py +261 -0
- kafka/metrics/metrics_reporter.py +53 -0
- kafka/metrics/quota.py +41 -0
- kafka/metrics/stat.py +19 -0
- kafka/metrics/stats/__init__.py +15 -0
- kafka/metrics/stats/avg.py +24 -0
- kafka/metrics/stats/count.py +17 -0
- kafka/metrics/stats/histogram.py +99 -0
- kafka/metrics/stats/max_stat.py +17 -0
- kafka/metrics/stats/min_stat.py +19 -0
- kafka/metrics/stats/percentile.py +14 -0
- kafka/metrics/stats/percentiles.py +75 -0
- kafka/metrics/stats/rate.py +118 -0
- kafka/metrics/stats/sampled_stat.py +99 -0
- kafka/metrics/stats/sensor.py +136 -0
- kafka/metrics/stats/total.py +15 -0
- kafka/net/__init__.py +19 -0
- kafka/net/compat.py +165 -0
- kafka/net/connection.py +593 -0
- kafka/net/http_connect.py +144 -0
- kafka/net/inet.py +122 -0
- kafka/net/manager.py +451 -0
- kafka/net/metrics.py +149 -0
- kafka/net/sasl/__init__.py +32 -0
- kafka/net/sasl/abc.py +28 -0
- kafka/net/sasl/gssapi.py +95 -0
- kafka/net/sasl/msk.py +245 -0
- kafka/net/sasl/oauth.py +98 -0
- kafka/net/sasl/plain.py +42 -0
- kafka/net/sasl/scram.py +135 -0
- kafka/net/sasl/sspi.py +111 -0
- kafka/net/selector.py +644 -0
- kafka/net/socks5.py +262 -0
- kafka/net/transport.py +415 -0
- kafka/net/wakeup_notifier.py +72 -0
- kafka/partitioner/__init__.py +8 -0
- kafka/partitioner/abc.py +8 -0
- kafka/partitioner/default.py +89 -0
- kafka/partitioner/sticky.py +109 -0
- kafka/producer/__init__.py +5 -0
- kafka/producer/__main__.py +5 -0
- kafka/producer/future.py +101 -0
- kafka/producer/kafka.py +1123 -0
- kafka/producer/producer_batch.py +192 -0
- kafka/producer/record_accumulator.py +647 -0
- kafka/producer/sender.py +884 -0
- kafka/producer/transaction_manager.py +1326 -0
- kafka/protocol/__init__.py +0 -0
- kafka/protocol/admin/__init__.py +29 -0
- kafka/protocol/admin/acl.py +83 -0
- kafka/protocol/admin/acl.pyi +375 -0
- kafka/protocol/admin/client_quotas.py +14 -0
- kafka/protocol/admin/client_quotas.pyi +265 -0
- kafka/protocol/admin/cluster.py +31 -0
- kafka/protocol/admin/cluster.pyi +620 -0
- kafka/protocol/admin/configs.py +22 -0
- kafka/protocol/admin/configs.pyi +437 -0
- kafka/protocol/admin/groups.py +24 -0
- kafka/protocol/admin/groups.pyi +261 -0
- kafka/protocol/admin/topics.py +53 -0
- kafka/protocol/admin/topics.pyi +982 -0
- kafka/protocol/admin/transactions.py +18 -0
- kafka/protocol/admin/transactions.pyi +311 -0
- kafka/protocol/admin/users.py +14 -0
- kafka/protocol/admin/users.pyi +223 -0
- kafka/protocol/api_data.py +125 -0
- kafka/protocol/api_header.py +55 -0
- kafka/protocol/api_key.py +97 -0
- kafka/protocol/api_message.py +277 -0
- kafka/protocol/broker_version_data.py +246 -0
- kafka/protocol/consumer/__init__.py +13 -0
- kafka/protocol/consumer/fetch.py +16 -0
- kafka/protocol/consumer/fetch.pyi +298 -0
- kafka/protocol/consumer/group.py +38 -0
- kafka/protocol/consumer/group.pyi +824 -0
- kafka/protocol/consumer/metadata.py +30 -0
- kafka/protocol/consumer/metadata.pyi +89 -0
- kafka/protocol/consumer/offsets.py +75 -0
- kafka/protocol/consumer/offsets.pyi +288 -0
- kafka/protocol/data_container.py +166 -0
- kafka/protocol/frame.py +30 -0
- kafka/protocol/generate_stubs.py +468 -0
- kafka/protocol/metadata/__init__.py +10 -0
- kafka/protocol/metadata/api_versions.py +41 -0
- kafka/protocol/metadata/api_versions.pyi +128 -0
- kafka/protocol/metadata/find_coordinator.py +19 -0
- kafka/protocol/metadata/find_coordinator.pyi +105 -0
- kafka/protocol/metadata/metadata.py +34 -0
- kafka/protocol/metadata/metadata.pyi +160 -0
- kafka/protocol/old/__init__.py +0 -0
- kafka/protocol/old/abstract.py +17 -0
- kafka/protocol/old/add_offsets_to_txn.py +54 -0
- kafka/protocol/old/add_partitions_to_txn.py +71 -0
- kafka/protocol/old/admin.py +1086 -0
- kafka/protocol/old/api.py +205 -0
- kafka/protocol/old/api_versions.py +133 -0
- kafka/protocol/old/commit.py +355 -0
- kafka/protocol/old/consumer_protocol.py +36 -0
- kafka/protocol/old/end_txn.py +53 -0
- kafka/protocol/old/fetch.py +408 -0
- kafka/protocol/old/find_coordinator.py +72 -0
- kafka/protocol/old/group.py +451 -0
- kafka/protocol/old/init_producer_id.py +42 -0
- kafka/protocol/old/list_offsets.py +186 -0
- kafka/protocol/old/metadata.py +290 -0
- kafka/protocol/old/offset_for_leader_epoch.py +133 -0
- kafka/protocol/old/produce.py +247 -0
- kafka/protocol/old/sasl_authenticate.py +38 -0
- kafka/protocol/old/sasl_handshake.py +39 -0
- kafka/protocol/old/struct.py +87 -0
- kafka/protocol/old/txn_offset_commit.py +73 -0
- kafka/protocol/old/types.py +440 -0
- kafka/protocol/parser.py +191 -0
- kafka/protocol/producer/__init__.py +7 -0
- kafka/protocol/producer/produce.py +17 -0
- kafka/protocol/producer/produce.pyi +197 -0
- kafka/protocol/producer/transaction.py +30 -0
- kafka/protocol/producer/transaction.pyi +663 -0
- kafka/protocol/sasl.py +52 -0
- kafka/protocol/sasl.pyi +126 -0
- kafka/protocol/schemas/__init__.py +7 -0
- kafka/protocol/schemas/fields/__init__.py +7 -0
- kafka/protocol/schemas/fields/array.py +127 -0
- kafka/protocol/schemas/fields/base.py +156 -0
- kafka/protocol/schemas/fields/codecs/__init__.py +12 -0
- kafka/protocol/schemas/fields/codecs/encode_buffer.py +82 -0
- kafka/protocol/schemas/fields/codecs/tagged_fields.py +109 -0
- kafka/protocol/schemas/fields/codecs/types.py +505 -0
- kafka/protocol/schemas/fields/codegen.py +40 -0
- kafka/protocol/schemas/fields/simple.py +127 -0
- kafka/protocol/schemas/fields/struct.py +357 -0
- kafka/protocol/schemas/fields/struct_array.py +142 -0
- kafka/protocol/schemas/load_json.py +42 -0
- kafka/protocol/schemas/resources/AddOffsetsToTxnRequest.json +40 -0
- kafka/protocol/schemas/resources/AddOffsetsToTxnResponse.json +35 -0
- kafka/protocol/schemas/resources/AddPartitionsToTxnRequest.json +65 -0
- kafka/protocol/schemas/resources/AddPartitionsToTxnResponse.json +60 -0
- kafka/protocol/schemas/resources/AlterClientQuotasRequest.json +47 -0
- kafka/protocol/schemas/resources/AlterClientQuotasResponse.json +41 -0
- kafka/protocol/schemas/resources/AlterConfigsRequest.json +43 -0
- kafka/protocol/schemas/resources/AlterConfigsResponse.json +39 -0
- kafka/protocol/schemas/resources/AlterPartitionReassignmentsRequest.json +42 -0
- kafka/protocol/schemas/resources/AlterPartitionReassignmentsResponse.json +47 -0
- kafka/protocol/schemas/resources/AlterReplicaLogDirsRequest.json +41 -0
- kafka/protocol/schemas/resources/AlterReplicaLogDirsResponse.json +41 -0
- kafka/protocol/schemas/resources/AlterUserScramCredentialsRequest.json +45 -0
- kafka/protocol/schemas/resources/AlterUserScramCredentialsResponse.json +35 -0
- kafka/protocol/schemas/resources/ApiVersionsRequest.json +34 -0
- kafka/protocol/schemas/resources/ApiVersionsResponse.json +79 -0
- kafka/protocol/schemas/resources/ConsumerProtocolAssignment.json +42 -0
- kafka/protocol/schemas/resources/ConsumerProtocolSubscription.json +49 -0
- kafka/protocol/schemas/resources/CreateAclsRequest.json +46 -0
- kafka/protocol/schemas/resources/CreateAclsResponse.json +37 -0
- kafka/protocol/schemas/resources/CreatePartitionsRequest.json +47 -0
- kafka/protocol/schemas/resources/CreatePartitionsResponse.json +41 -0
- kafka/protocol/schemas/resources/CreateTopicsRequest.json +65 -0
- kafka/protocol/schemas/resources/CreateTopicsResponse.json +72 -0
- kafka/protocol/schemas/resources/DeleteAclsRequest.json +46 -0
- kafka/protocol/schemas/resources/DeleteAclsResponse.json +59 -0
- kafka/protocol/schemas/resources/DeleteGroupsRequest.json +30 -0
- kafka/protocol/schemas/resources/DeleteGroupsResponse.json +36 -0
- kafka/protocol/schemas/resources/DeleteRecordsRequest.json +42 -0
- kafka/protocol/schemas/resources/DeleteRecordsResponse.json +43 -0
- kafka/protocol/schemas/resources/DeleteTopicsRequest.json +43 -0
- kafka/protocol/schemas/resources/DeleteTopicsResponse.json +52 -0
- kafka/protocol/schemas/resources/DescribeAclsRequest.json +43 -0
- kafka/protocol/schemas/resources/DescribeAclsResponse.json +55 -0
- kafka/protocol/schemas/resources/DescribeClientQuotasRequest.json +37 -0
- kafka/protocol/schemas/resources/DescribeClientQuotasResponse.json +47 -0
- kafka/protocol/schemas/resources/DescribeClusterRequest.json +35 -0
- kafka/protocol/schemas/resources/DescribeClusterResponse.json +56 -0
- kafka/protocol/schemas/resources/DescribeConfigsRequest.json +42 -0
- kafka/protocol/schemas/resources/DescribeConfigsResponse.json +69 -0
- kafka/protocol/schemas/resources/DescribeGroupsRequest.json +38 -0
- kafka/protocol/schemas/resources/DescribeGroupsResponse.json +74 -0
- kafka/protocol/schemas/resources/DescribeLogDirsRequest.json +38 -0
- kafka/protocol/schemas/resources/DescribeLogDirsResponse.json +65 -0
- kafka/protocol/schemas/resources/DescribeProducersRequest.json +32 -0
- kafka/protocol/schemas/resources/DescribeProducersResponse.json +55 -0
- kafka/protocol/schemas/resources/DescribeQuorumRequest.json +39 -0
- kafka/protocol/schemas/resources/DescribeQuorumResponse.json +82 -0
- kafka/protocol/schemas/resources/DescribeTopicPartitionsRequest.json +40 -0
- kafka/protocol/schemas/resources/DescribeTopicPartitionsResponse.json +66 -0
- kafka/protocol/schemas/resources/DescribeTransactionsRequest.json +27 -0
- kafka/protocol/schemas/resources/DescribeTransactionsResponse.json +52 -0
- kafka/protocol/schemas/resources/DescribeUserScramCredentialsRequest.json +30 -0
- kafka/protocol/schemas/resources/DescribeUserScramCredentialsResponse.json +45 -0
- kafka/protocol/schemas/resources/ElectLeadersRequest.json +41 -0
- kafka/protocol/schemas/resources/ElectLeadersResponse.json +45 -0
- kafka/protocol/schemas/resources/EndTxnRequest.json +43 -0
- kafka/protocol/schemas/resources/EndTxnResponse.json +41 -0
- kafka/protocol/schemas/resources/FetchRequest.json +125 -0
- kafka/protocol/schemas/resources/FetchResponse.json +124 -0
- kafka/protocol/schemas/resources/FindCoordinatorRequest.json +43 -0
- kafka/protocol/schemas/resources/FindCoordinatorResponse.json +58 -0
- kafka/protocol/schemas/resources/HeartbeatRequest.json +39 -0
- kafka/protocol/schemas/resources/HeartbeatResponse.json +35 -0
- kafka/protocol/schemas/resources/IncrementalAlterConfigsRequest.json +44 -0
- kafka/protocol/schemas/resources/IncrementalAlterConfigsResponse.json +38 -0
- kafka/protocol/schemas/resources/InitProducerIdRequest.json +50 -0
- kafka/protocol/schemas/resources/InitProducerIdResponse.json +47 -0
- kafka/protocol/schemas/resources/JoinGroupRequest.json +63 -0
- kafka/protocol/schemas/resources/JoinGroupResponse.json +69 -0
- kafka/protocol/schemas/resources/LeaveGroupRequest.json +47 -0
- kafka/protocol/schemas/resources/LeaveGroupResponse.json +47 -0
- kafka/protocol/schemas/resources/ListConfigResourcesRequest.json +31 -0
- kafka/protocol/schemas/resources/ListConfigResourcesResponse.json +37 -0
- kafka/protocol/schemas/resources/ListGroupsRequest.json +36 -0
- kafka/protocol/schemas/resources/ListGroupsResponse.json +49 -0
- kafka/protocol/schemas/resources/ListOffsetsRequest.json +72 -0
- kafka/protocol/schemas/resources/ListOffsetsResponse.json +71 -0
- kafka/protocol/schemas/resources/ListPartitionReassignmentsRequest.json +34 -0
- kafka/protocol/schemas/resources/ListPartitionReassignmentsResponse.json +46 -0
- kafka/protocol/schemas/resources/ListTransactionsRequest.json +40 -0
- kafka/protocol/schemas/resources/ListTransactionsResponse.json +42 -0
- kafka/protocol/schemas/resources/MetadataRequest.json +56 -0
- kafka/protocol/schemas/resources/MetadataResponse.json +101 -0
- kafka/protocol/schemas/resources/OffsetCommitRequest.json +76 -0
- kafka/protocol/schemas/resources/OffsetCommitResponse.json +71 -0
- kafka/protocol/schemas/resources/OffsetDeleteRequest.json +39 -0
- kafka/protocol/schemas/resources/OffsetDeleteResponse.json +42 -0
- kafka/protocol/schemas/resources/OffsetFetchRequest.json +76 -0
- kafka/protocol/schemas/resources/OffsetFetchResponse.json +107 -0
- kafka/protocol/schemas/resources/OffsetForLeaderEpochRequest.json +52 -0
- kafka/protocol/schemas/resources/OffsetForLeaderEpochResponse.json +51 -0
- kafka/protocol/schemas/resources/ProduceRequest.json +73 -0
- kafka/protocol/schemas/resources/ProduceResponse.json +96 -0
- kafka/protocol/schemas/resources/RequestHeader.json +44 -0
- kafka/protocol/schemas/resources/ResponseHeader.json +26 -0
- kafka/protocol/schemas/resources/SaslAuthenticateRequest.json +29 -0
- kafka/protocol/schemas/resources/SaslAuthenticateResponse.json +34 -0
- kafka/protocol/schemas/resources/SaslHandshakeRequest.json +31 -0
- kafka/protocol/schemas/resources/SaslHandshakeResponse.json +32 -0
- kafka/protocol/schemas/resources/SyncGroupRequest.json +56 -0
- kafka/protocol/schemas/resources/SyncGroupResponse.json +46 -0
- kafka/protocol/schemas/resources/TxnOffsetCommitRequest.json +68 -0
- kafka/protocol/schemas/resources/TxnOffsetCommitResponse.json +47 -0
- kafka/protocol/schemas/resources/UpdateFeaturesRequest.json +43 -0
- kafka/protocol/schemas/resources/UpdateFeaturesResponse.json +39 -0
- kafka/protocol/schemas/resources/WriteTxnMarkersRequest.json +49 -0
- kafka/protocol/schemas/resources/WriteTxnMarkersResponse.json +45 -0
- kafka/protocol/schemas/resources/__init__.py +0 -0
- kafka/record/__init__.py +3 -0
- kafka/record/_crc32c.py +161 -0
- kafka/record/abc.py +144 -0
- kafka/record/default_records.py +782 -0
- kafka/record/legacy_records.py +587 -0
- kafka/record/memory_records.py +255 -0
- kafka/record/util.py +135 -0
- kafka/serializer/__init__.py +4 -0
- kafka/serializer/abstract.py +20 -0
- kafka/serializer/default.py +16 -0
- kafka/serializer/json.py +17 -0
- kafka/serializer/wrapper.py +21 -0
- kafka/structs.py +69 -0
- kafka/util.py +159 -0
- kafka/vendor/__init__.py +0 -0
- kafka/version.py +1 -0
- kafka_python-3.0.0.dist-info/METADATA +319 -0
- kafka_python-3.0.0.dist-info/RECORD +373 -0
- kafka_python-3.0.0.dist-info/WHEEL +5 -0
- kafka_python-3.0.0.dist-info/entry_points.txt +2 -0
- kafka_python-3.0.0.dist-info/licenses/LICENSE +202 -0
- kafka_python-3.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
from struct import error, pack, pack_into, unpack, unpack_from
|
|
2
|
+
import uuid
|
|
3
|
+
|
|
4
|
+
from .encode_buffer import EncodeBuffer
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FixedCodec:
|
|
8
|
+
"""Base class for fixed-size codecs. Subclasses define fmt and size.
|
|
9
|
+
|
|
10
|
+
The Kafka protocol uses big-endian ('>') byte order for all fixed-size
|
|
11
|
+
types. This prefix is applied here so subclasses only specify the type
|
|
12
|
+
format character (e.g., 'i' for 32-bit signed int).
|
|
13
|
+
"""
|
|
14
|
+
fmt = None # e.g., 'i' - set by subclass
|
|
15
|
+
size = None # e.g., 4 - set by subclass
|
|
16
|
+
batchable = True # Can be batched with adjacent FixedCodec fields
|
|
17
|
+
|
|
18
|
+
def __init_subclass__(cls, **kw):
|
|
19
|
+
super().__init_subclass__(**kw)
|
|
20
|
+
if cls.fmt is not None:
|
|
21
|
+
cls._be_fmt = '>' + cls.fmt
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def encode(cls, value, compact=False):
|
|
25
|
+
return pack(cls._be_fmt, value)
|
|
26
|
+
|
|
27
|
+
@classmethod
|
|
28
|
+
def encode_into(cls, out, value, compact=False):
|
|
29
|
+
pack_into(cls._be_fmt, out.buf, out.pos, value)
|
|
30
|
+
out.pos += cls.size
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def decode(cls, data, compact=False):
|
|
34
|
+
return unpack(cls._be_fmt, data.read(cls.size))[0]
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def decode_from(cls, data, pos):
|
|
38
|
+
"""Decode from a buffer at pos. Returns (value, new_pos)."""
|
|
39
|
+
return unpack_from(cls._be_fmt, data, pos)[0], pos + cls.size
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
|
|
43
|
+
ctx.emit(indent, "pack_into('%s', buf, pos, %s)" % (cls._be_fmt, val_expr))
|
|
44
|
+
ctx.emit(indent, 'pos += %d' % cls.size)
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def emit_decode_from(cls, ctx, var_name, indent, compact=False):
|
|
48
|
+
"""Emit decode for a single fixed field (unbatched fallback)."""
|
|
49
|
+
ctx.emit(indent, '%s = unpack_from("%s", data, pos)[0]' % (var_name, cls._be_fmt))
|
|
50
|
+
ctx.emit(indent, 'pos += %d' % cls.size)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Int8(FixedCodec):
|
|
54
|
+
fmt = 'b'
|
|
55
|
+
size = 1
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Int16(FixedCodec):
|
|
59
|
+
fmt = 'h'
|
|
60
|
+
size = 2
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class UnsignedInt16(FixedCodec):
|
|
64
|
+
fmt = 'H'
|
|
65
|
+
size = 2
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class Int32(FixedCodec):
|
|
69
|
+
fmt = 'i'
|
|
70
|
+
size = 4
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class Int64(FixedCodec):
|
|
74
|
+
fmt = 'q'
|
|
75
|
+
size = 8
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Float64(FixedCodec):
|
|
79
|
+
fmt = 'd'
|
|
80
|
+
size = 8
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class UUID:
|
|
84
|
+
fmt = '16B'
|
|
85
|
+
size = 16
|
|
86
|
+
ZERO_UUID = uuid.UUID(int=0)
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def encode(cls, value, compact=False):
|
|
90
|
+
if value is None:
|
|
91
|
+
value = cls.ZERO_UUID
|
|
92
|
+
if isinstance(value, uuid.UUID):
|
|
93
|
+
return value.bytes
|
|
94
|
+
return uuid.UUID(value).bytes
|
|
95
|
+
|
|
96
|
+
@classmethod
|
|
97
|
+
def encode_into(cls, out, value, compact=False):
|
|
98
|
+
if value is None:
|
|
99
|
+
value = cls.ZERO_UUID
|
|
100
|
+
if isinstance(value, uuid.UUID):
|
|
101
|
+
b = value.bytes
|
|
102
|
+
else:
|
|
103
|
+
b = uuid.UUID(value).bytes
|
|
104
|
+
pos = out.pos
|
|
105
|
+
out.buf[pos:pos+16] = b
|
|
106
|
+
out.pos = pos + 16
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
|
|
110
|
+
ctx.globs['_ZERO_UUID'] = cls.ZERO_UUID
|
|
111
|
+
v = ctx.next_var('uv')
|
|
112
|
+
ctx.emit(indent, '%s = %s' % (v, val_expr))
|
|
113
|
+
ctx.emit(indent, 'if %s is None: %s = _ZERO_UUID' % (v, v))
|
|
114
|
+
ctx.emit(indent, 'buf[pos:pos+16] = %s.bytes if hasattr(%s, "bytes") else __import__("uuid").UUID(%s).bytes' % (v, v, v))
|
|
115
|
+
ctx.emit(indent, 'pos += 16')
|
|
116
|
+
|
|
117
|
+
@classmethod
|
|
118
|
+
def emit_decode_from(cls, ctx, var_name, indent, compact=False):
|
|
119
|
+
ctx.globs['_ZERO_UUID'] = cls.ZERO_UUID
|
|
120
|
+
ctx.globs['_uuid_cls'] = uuid.UUID
|
|
121
|
+
ctx.emit(indent, '%s = _uuid_cls(bytes=bytes(data[pos:pos+16]))' % var_name)
|
|
122
|
+
ctx.emit(indent, 'if %s == _ZERO_UUID: %s = None' % (var_name, var_name))
|
|
123
|
+
ctx.emit(indent, 'pos += 16')
|
|
124
|
+
|
|
125
|
+
@classmethod
|
|
126
|
+
def decode(cls, data, compact=False):
|
|
127
|
+
val = uuid.UUID(bytes=data.read(16))
|
|
128
|
+
if val == cls.ZERO_UUID:
|
|
129
|
+
return None
|
|
130
|
+
return val
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class String:
|
|
134
|
+
fmt = None # 'B' for compact, 'h' for standard
|
|
135
|
+
size = 'variable'
|
|
136
|
+
|
|
137
|
+
def __init__(self, encoding='utf-8'):
|
|
138
|
+
self.encoding = encoding
|
|
139
|
+
|
|
140
|
+
def encode(self, value, compact=False):
|
|
141
|
+
if compact:
|
|
142
|
+
if value is None:
|
|
143
|
+
return UnsignedVarInt32.encode(0)
|
|
144
|
+
value = str(value).encode(self.encoding)
|
|
145
|
+
return UnsignedVarInt32.encode(len(value) + 1) + value
|
|
146
|
+
if value is None:
|
|
147
|
+
return Int16.encode(-1)
|
|
148
|
+
value = str(value).encode(self.encoding)
|
|
149
|
+
return Int16.encode(len(value)) + value
|
|
150
|
+
|
|
151
|
+
def encode_into(self, out, value, compact=False):
|
|
152
|
+
if compact:
|
|
153
|
+
if value is None:
|
|
154
|
+
UnsignedVarInt32.encode_into(out,0)
|
|
155
|
+
return
|
|
156
|
+
value = str(value).encode(self.encoding)
|
|
157
|
+
UnsignedVarInt32.encode_into(out,len(value) + 1)
|
|
158
|
+
else:
|
|
159
|
+
if value is None:
|
|
160
|
+
pack_into('>h', out.buf, out.pos, -1)
|
|
161
|
+
out.pos += 2
|
|
162
|
+
return
|
|
163
|
+
value = str(value).encode(self.encoding)
|
|
164
|
+
pack_into('>h', out.buf, out.pos, len(value))
|
|
165
|
+
out.pos += 2
|
|
166
|
+
n = len(value)
|
|
167
|
+
pos = out.pos
|
|
168
|
+
out.buf[pos:pos+n] = value
|
|
169
|
+
out.pos = pos + n
|
|
170
|
+
|
|
171
|
+
def emit_encode_into(self, ctx, val_expr, indent, compact=False):
|
|
172
|
+
sv = ctx.next_var('sv')
|
|
173
|
+
ctx.emit(indent, 'if %s is None:' % val_expr)
|
|
174
|
+
if compact:
|
|
175
|
+
ctx.emit(indent, ' buf[pos] = 0')
|
|
176
|
+
ctx.emit(indent, ' pos += 1')
|
|
177
|
+
ctx.emit(indent, 'else:')
|
|
178
|
+
sn = ctx.next_var('sn')
|
|
179
|
+
ctx.emit(indent, ' %s = str(%s).encode("utf-8")' % (sv, val_expr))
|
|
180
|
+
ctx.emit(indent, ' %s = len(%s) + 1' % (sn, sv))
|
|
181
|
+
UnsignedVarInt32.emit_encode_into(ctx, sn, indent + ' ')
|
|
182
|
+
ctx.emit(indent, ' buf[pos:pos+len(%s)] = %s' % (sv, sv))
|
|
183
|
+
ctx.emit(indent, ' pos += len(%s)' % sv)
|
|
184
|
+
else:
|
|
185
|
+
ctx.emit(indent, " pack_into('>h', buf, pos, -1)")
|
|
186
|
+
ctx.emit(indent, ' pos += 2')
|
|
187
|
+
ctx.emit(indent, 'else:')
|
|
188
|
+
ctx.emit(indent, ' %s = str(%s).encode("utf-8")' % (sv, val_expr))
|
|
189
|
+
ctx.emit(indent, " pack_into('>h', buf, pos, len(%s))" % sv)
|
|
190
|
+
ctx.emit(indent, ' pos += 2')
|
|
191
|
+
ctx.emit(indent, ' buf[pos:pos+len(%s)] = %s' % (sv, sv))
|
|
192
|
+
ctx.emit(indent, ' pos += len(%s)' % sv)
|
|
193
|
+
|
|
194
|
+
def emit_decode_from(self, ctx, var_name, indent, compact=False):
|
|
195
|
+
ln = ctx.next_var('ln')
|
|
196
|
+
if compact:
|
|
197
|
+
UnsignedVarInt32.emit_decode_from(ctx, ln, indent)
|
|
198
|
+
ctx.emit(indent, '%s -= 1' % ln)
|
|
199
|
+
else:
|
|
200
|
+
ctx.emit(indent, '%s = unpack_from(">h", data, pos)[0]' % ln)
|
|
201
|
+
ctx.emit(indent, 'pos += 2')
|
|
202
|
+
ctx.emit(indent, 'if %s < 0:' % ln)
|
|
203
|
+
ctx.emit(indent, ' %s = None' % var_name)
|
|
204
|
+
ctx.emit(indent, 'else:')
|
|
205
|
+
ctx.emit(indent, ' %s = str(data[pos:pos+%s], "utf-8")' % (var_name, ln))
|
|
206
|
+
ctx.emit(indent, ' pos += %s' % ln)
|
|
207
|
+
|
|
208
|
+
def decode(self, data, compact=False):
|
|
209
|
+
if compact:
|
|
210
|
+
length = UnsignedVarInt32.decode(data) - 1
|
|
211
|
+
else:
|
|
212
|
+
length = Int16.decode(data)
|
|
213
|
+
if length < 0:
|
|
214
|
+
return None
|
|
215
|
+
value = data.read(length)
|
|
216
|
+
if len(value) != length:
|
|
217
|
+
raise ValueError('Buffer underrun decoding string')
|
|
218
|
+
return value.decode(self.encoding)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
class Bytes:
|
|
222
|
+
fmt = Int32.fmt
|
|
223
|
+
size = 'variable'
|
|
224
|
+
|
|
225
|
+
@classmethod
|
|
226
|
+
def encode(cls, value, compact=False):
|
|
227
|
+
if value is not None and not isinstance(value, (bytes, bytearray, memoryview)):
|
|
228
|
+
value = value.encode()
|
|
229
|
+
if compact:
|
|
230
|
+
if value is None:
|
|
231
|
+
return UnsignedVarInt32.encode(0)
|
|
232
|
+
return UnsignedVarInt32.encode(len(value) + 1) + bytes(value)
|
|
233
|
+
if value is None:
|
|
234
|
+
return Int32.encode(-1)
|
|
235
|
+
return Int32.encode(len(value)) + bytes(value)
|
|
236
|
+
|
|
237
|
+
@classmethod
|
|
238
|
+
def encode_into(cls, out, value, compact=False):
|
|
239
|
+
if value is not None and not isinstance(value, (bytes, bytearray, memoryview)):
|
|
240
|
+
value = value.encode()
|
|
241
|
+
if compact:
|
|
242
|
+
if value is None:
|
|
243
|
+
UnsignedVarInt32.encode_into(out, 0)
|
|
244
|
+
return
|
|
245
|
+
UnsignedVarInt32.encode_into(out, len(value) + 1)
|
|
246
|
+
else:
|
|
247
|
+
if value is None:
|
|
248
|
+
pack_into('>i', out.buf, out.pos, -1)
|
|
249
|
+
out.pos += 4
|
|
250
|
+
return
|
|
251
|
+
pack_into('>i', out.buf, out.pos, len(value))
|
|
252
|
+
out.pos += 4
|
|
253
|
+
n = len(value)
|
|
254
|
+
out.ensure(n)
|
|
255
|
+
pos = out.pos
|
|
256
|
+
out.buf[pos:pos+n] = value
|
|
257
|
+
out.pos = pos + n
|
|
258
|
+
|
|
259
|
+
@classmethod
|
|
260
|
+
def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
|
|
261
|
+
bv = ctx.next_var('bv')
|
|
262
|
+
bn = ctx.next_var('bn')
|
|
263
|
+
if compact:
|
|
264
|
+
ctx.emit(indent, '%s = %s' % (bv, val_expr))
|
|
265
|
+
ctx.emit(indent, 'if %s is not None and not isinstance(%s, (bytes, bytearray, memoryview)): %s = %s.encode()' % (bv, bv, bv, bv))
|
|
266
|
+
ctx.emit(indent, 'if %s is None:' % bv)
|
|
267
|
+
ctx.emit(indent, ' buf[pos] = 0')
|
|
268
|
+
ctx.emit(indent, ' pos += 1')
|
|
269
|
+
ctx.emit(indent, 'else:')
|
|
270
|
+
ctx.emit(indent, ' %s = len(%s)' % (bn, bv))
|
|
271
|
+
sn = ctx.next_var('sn')
|
|
272
|
+
ctx.emit(indent, ' %s = %s + 1' % (sn, bn))
|
|
273
|
+
UnsignedVarInt32.emit_encode_into(ctx, sn, indent + ' ')
|
|
274
|
+
ctx.emit(indent, ' out.pos = pos')
|
|
275
|
+
ctx.emit(indent, ' out.ensure(%s)' % bn)
|
|
276
|
+
ctx.emit(indent, ' buf = out.buf')
|
|
277
|
+
ctx.emit(indent, ' buf[pos:pos+%s] = %s' % (bn, bv))
|
|
278
|
+
ctx.emit(indent, ' pos += %s' % bn)
|
|
279
|
+
else:
|
|
280
|
+
ctx.emit(indent, '%s = %s' % (bv, val_expr))
|
|
281
|
+
ctx.emit(indent, 'if %s is not None and not isinstance(%s, (bytes, bytearray, memoryview)): %s = %s.encode()' % (bv, bv, bv, bv))
|
|
282
|
+
ctx.emit(indent, 'if %s is None:' % bv)
|
|
283
|
+
ctx.emit(indent, " pack_into('>i', buf, pos, -1)")
|
|
284
|
+
ctx.emit(indent, ' pos += 4')
|
|
285
|
+
ctx.emit(indent, 'else:')
|
|
286
|
+
ctx.emit(indent, ' %s = len(%s)' % (bn, bv))
|
|
287
|
+
ctx.emit(indent, " pack_into('>i', buf, pos, %s)" % bn)
|
|
288
|
+
ctx.emit(indent, ' pos += 4')
|
|
289
|
+
ctx.emit(indent, ' out.pos = pos')
|
|
290
|
+
ctx.emit(indent, ' out.ensure(%s)' % bn)
|
|
291
|
+
ctx.emit(indent, ' buf = out.buf')
|
|
292
|
+
ctx.emit(indent, ' buf[pos:pos+%s] = %s' % (bn, bv))
|
|
293
|
+
ctx.emit(indent, ' pos += %s' % bn)
|
|
294
|
+
|
|
295
|
+
@classmethod
|
|
296
|
+
def emit_decode_from(cls, ctx, var_name, indent, compact=False):
|
|
297
|
+
ln = ctx.next_var('ln')
|
|
298
|
+
if compact:
|
|
299
|
+
UnsignedVarInt32.emit_decode_from(ctx, ln, indent)
|
|
300
|
+
ctx.emit(indent, '%s -= 1' % ln)
|
|
301
|
+
else:
|
|
302
|
+
ctx.emit(indent, '%s = unpack_from(">i", data, pos)[0]' % ln)
|
|
303
|
+
ctx.emit(indent, 'pos += 4')
|
|
304
|
+
ctx.emit(indent, 'if %s < 0:' % ln)
|
|
305
|
+
ctx.emit(indent, ' %s = None' % var_name)
|
|
306
|
+
ctx.emit(indent, 'else:')
|
|
307
|
+
ctx.emit(indent, ' %s = bytes(data[pos:pos+%s])' % (var_name, ln))
|
|
308
|
+
ctx.emit(indent, ' pos += %s' % ln)
|
|
309
|
+
|
|
310
|
+
@classmethod
|
|
311
|
+
def decode(cls, data, compact=False):
|
|
312
|
+
if compact:
|
|
313
|
+
length = UnsignedVarInt32.decode(data) - 1
|
|
314
|
+
else:
|
|
315
|
+
length = Int32.decode(data)
|
|
316
|
+
if length < 0:
|
|
317
|
+
return None
|
|
318
|
+
value = data.read(length)
|
|
319
|
+
if len(value) != length:
|
|
320
|
+
raise ValueError('Buffer underrun decoding Bytes')
|
|
321
|
+
return value
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
class Boolean(FixedCodec):
|
|
325
|
+
fmt = '?'
|
|
326
|
+
size = 1
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
class UnsignedVarInt32:
|
|
330
|
+
fmt = 'B'
|
|
331
|
+
size = 'variable'
|
|
332
|
+
|
|
333
|
+
@classmethod
|
|
334
|
+
def decode(cls, data, compact=False):
|
|
335
|
+
value = VarInt32.decode(data)
|
|
336
|
+
return (value << 1) ^ (value >> 31)
|
|
337
|
+
|
|
338
|
+
@classmethod
|
|
339
|
+
def encode(cls, value, compact=False):
|
|
340
|
+
return VarInt32.encode((value >> 1) ^ -(value & 1))
|
|
341
|
+
|
|
342
|
+
@classmethod
|
|
343
|
+
def encode_into(cls, out, value):
|
|
344
|
+
buf = out.buf
|
|
345
|
+
pos = out.pos
|
|
346
|
+
while (value & 0xffffff80) != 0:
|
|
347
|
+
buf[pos] = (value & 0x7f) | 0x80
|
|
348
|
+
value >>= 7
|
|
349
|
+
pos += 1
|
|
350
|
+
buf[pos] = value
|
|
351
|
+
out.pos = pos + 1
|
|
352
|
+
|
|
353
|
+
@classmethod
|
|
354
|
+
def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
|
|
355
|
+
ctx.emit(indent, 'while (%s & 0xffffff80) != 0:' % val_expr)
|
|
356
|
+
ctx.emit(indent, ' buf[pos] = (%s & 0x7f) | 0x80' % val_expr)
|
|
357
|
+
ctx.emit(indent, ' %s >>= 7' % val_expr)
|
|
358
|
+
ctx.emit(indent, ' pos += 1')
|
|
359
|
+
ctx.emit(indent, 'buf[pos] = %s' % val_expr)
|
|
360
|
+
ctx.emit(indent, 'pos += 1')
|
|
361
|
+
|
|
362
|
+
@classmethod
|
|
363
|
+
def emit_decode_from(cls, ctx, var_name, indent):
|
|
364
|
+
"""Emit inline unsigned varint decode. Result in var_name, pos advanced."""
|
|
365
|
+
b = ctx.next_var('b')
|
|
366
|
+
shift = ctx.next_var('sh')
|
|
367
|
+
ctx.emit(indent, '%s = 0' % var_name)
|
|
368
|
+
ctx.emit(indent, '%s = 0' % shift)
|
|
369
|
+
ctx.emit(indent, 'while True:')
|
|
370
|
+
ctx.emit(indent, ' %s = data[pos]' % b)
|
|
371
|
+
ctx.emit(indent, ' pos += 1')
|
|
372
|
+
ctx.emit(indent, ' %s |= (%s & 0x7f) << %s' % (var_name, b, shift))
|
|
373
|
+
ctx.emit(indent, ' if not (%s & 0x80): break' % b)
|
|
374
|
+
ctx.emit(indent, ' %s += 7' % shift)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
class VarInt32:
|
|
378
|
+
fmt = 'B'
|
|
379
|
+
size = 'variable'
|
|
380
|
+
|
|
381
|
+
@classmethod
|
|
382
|
+
def decode(cls, data, compact=False):
|
|
383
|
+
value, i = 0, 0
|
|
384
|
+
while True:
|
|
385
|
+
b, = unpack('B', data.read(1))
|
|
386
|
+
if not (b & 0x80):
|
|
387
|
+
break
|
|
388
|
+
value |= (b & 0x7f) << i
|
|
389
|
+
i += 7
|
|
390
|
+
if i > 28:
|
|
391
|
+
raise ValueError('Invalid value {}'.format(value))
|
|
392
|
+
value |= b << i
|
|
393
|
+
return (value >> 1) ^ -(value & 1)
|
|
394
|
+
|
|
395
|
+
@classmethod
|
|
396
|
+
def encode(cls, value, compact=False):
|
|
397
|
+
# bring it in line with the java binary repr
|
|
398
|
+
value = (value << 1) ^ (value >> 31)
|
|
399
|
+
value &= 0xffffffff
|
|
400
|
+
ret = b''
|
|
401
|
+
while (value & 0xffffff80) != 0:
|
|
402
|
+
b = (value & 0x7f) | 0x80
|
|
403
|
+
ret += pack('B', b)
|
|
404
|
+
value >>= 7
|
|
405
|
+
ret += pack('B', value)
|
|
406
|
+
return ret
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
class VarInt64:
|
|
410
|
+
fmt = 'B'
|
|
411
|
+
size = 'variable'
|
|
412
|
+
|
|
413
|
+
@classmethod
|
|
414
|
+
def decode(cls, data, compact=False):
|
|
415
|
+
value, i = 0, 0
|
|
416
|
+
while True:
|
|
417
|
+
b, = unpack('B', data.read(1))
|
|
418
|
+
if not (b & 0x80):
|
|
419
|
+
break
|
|
420
|
+
value |= (b & 0x7f) << i
|
|
421
|
+
i += 7
|
|
422
|
+
if i > 63:
|
|
423
|
+
raise ValueError('Invalid value {}'.format(value))
|
|
424
|
+
value |= b << i
|
|
425
|
+
return (value >> 1) ^ -(value & 1)
|
|
426
|
+
|
|
427
|
+
@classmethod
|
|
428
|
+
def encode(cls, value, compact=False):
|
|
429
|
+
# bring it in line with the java binary repr
|
|
430
|
+
value = (value << 1) ^ (value >> 63)
|
|
431
|
+
value &= 0xffffffffffffffff
|
|
432
|
+
ret = b''
|
|
433
|
+
while (value & 0xffffffffffffff80) != 0:
|
|
434
|
+
b = (value & 0x7f) | 0x80
|
|
435
|
+
ret += pack('B', b)
|
|
436
|
+
value >>= 7
|
|
437
|
+
ret += pack('B', value)
|
|
438
|
+
return ret
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
class BitField:
|
|
442
|
+
fmt = 'I'
|
|
443
|
+
size = 4
|
|
444
|
+
|
|
445
|
+
@classmethod
|
|
446
|
+
def decode(cls, data, compact=False):
|
|
447
|
+
vals = cls.from_32_bit_field(unpack('>I', data.read(4))[0])
|
|
448
|
+
if vals == {31}:
|
|
449
|
+
vals = None
|
|
450
|
+
return vals
|
|
451
|
+
|
|
452
|
+
@classmethod
|
|
453
|
+
def encode(cls, vals, compact=False):
|
|
454
|
+
if vals is None:
|
|
455
|
+
vals = {31}
|
|
456
|
+
# to_32_bit_field returns unsigned val, so we need to
|
|
457
|
+
# encode >I to avoid crash if/when byte 31 is set
|
|
458
|
+
# (note that decode as signed still works fine)
|
|
459
|
+
return pack('>I', cls.to_32_bit_field(vals))
|
|
460
|
+
|
|
461
|
+
@classmethod
|
|
462
|
+
def encode_into(cls, out, vals, compact=False):
|
|
463
|
+
if vals is None:
|
|
464
|
+
vals = {31}
|
|
465
|
+
pack_into('>I', out.buf, out.pos, cls.to_32_bit_field(vals))
|
|
466
|
+
out.pos += 4
|
|
467
|
+
|
|
468
|
+
@classmethod
|
|
469
|
+
def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
|
|
470
|
+
bf = ctx.next_var('bf')
|
|
471
|
+
bfi = ctx.next_var('bfi')
|
|
472
|
+
ctx.emit(indent, '%s = %s' % (bf, val_expr))
|
|
473
|
+
ctx.emit(indent, 'if %s is None: %s = {31}' % (bf, bf))
|
|
474
|
+
ctx.emit(indent, '%s = 0' % bfi)
|
|
475
|
+
ctx.emit(indent, 'for _b in %s: %s |= 1 << _b' % (bf, bfi))
|
|
476
|
+
ctx.emit(indent, "pack_into('>I', buf, pos, %s)" % bfi)
|
|
477
|
+
ctx.emit(indent, 'pos += 4')
|
|
478
|
+
|
|
479
|
+
@classmethod
|
|
480
|
+
def emit_decode_from(cls, ctx, var_name, indent, compact=False):
|
|
481
|
+
ctx.globs['_bitfield_from_32'] = cls.from_32_bit_field
|
|
482
|
+
raw = ctx.next_var('bfr')
|
|
483
|
+
ctx.emit(indent, '%s = unpack_from(">I", data, pos)[0]' % raw)
|
|
484
|
+
ctx.emit(indent, 'pos += 4')
|
|
485
|
+
ctx.emit(indent, '%s = _bitfield_from_32(%s)' % (var_name, raw))
|
|
486
|
+
ctx.emit(indent, 'if %s == {31}: %s = None' % (var_name, var_name))
|
|
487
|
+
|
|
488
|
+
@classmethod
|
|
489
|
+
def to_32_bit_field(cls, vals):
|
|
490
|
+
value = 0
|
|
491
|
+
for b in vals:
|
|
492
|
+
assert 0 <= b < 32
|
|
493
|
+
value |= 1 << b
|
|
494
|
+
return value
|
|
495
|
+
|
|
496
|
+
@classmethod
|
|
497
|
+
def from_32_bit_field(cls, value):
|
|
498
|
+
result = set()
|
|
499
|
+
count = 0
|
|
500
|
+
while value != 0:
|
|
501
|
+
if (value & 1) != 0:
|
|
502
|
+
result.add(count)
|
|
503
|
+
count += 1
|
|
504
|
+
value = (value & 0xFFFFFFFF) >> 1
|
|
505
|
+
return result
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Generate flat encode/decode functions for a StructField + version.
|
|
2
|
+
|
|
3
|
+
Given a StructField and a protocol version, generates Python functions
|
|
4
|
+
that encode/decode directly with zero dispatch overhead - no intermediate
|
|
5
|
+
SimpleField/ArrayField/StructField method calls.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
from kafka.protocol.schemas.fields.codegen import CodegenContext
|
|
9
|
+
# Encode: see StructField.compiled_encode_into()
|
|
10
|
+
# Decode: see StructField.compiled_decode_from()
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from struct import pack_into, unpack_from
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CodegenContext:
|
|
17
|
+
"""Shared state for code generation."""
|
|
18
|
+
|
|
19
|
+
def __init__(self):
|
|
20
|
+
self.lines = []
|
|
21
|
+
self.globs = {'pack_into': pack_into, 'unpack_from': unpack_from}
|
|
22
|
+
self._var_counter = 0
|
|
23
|
+
|
|
24
|
+
def next_var(self, prefix='v'):
|
|
25
|
+
self._var_counter += 1
|
|
26
|
+
return f'_{prefix}{self._var_counter}'
|
|
27
|
+
|
|
28
|
+
def emit(self, indent, line):
|
|
29
|
+
self.lines.append(f'{indent}{line}')
|
|
30
|
+
|
|
31
|
+
def source(self):
|
|
32
|
+
return '\n'.join(self.lines)
|
|
33
|
+
|
|
34
|
+
def print(self):
|
|
35
|
+
print('GLOBALS:')
|
|
36
|
+
for var in self.globs:
|
|
37
|
+
print(f' {var}={self.globs[var]}')
|
|
38
|
+
print('\nSOURCE:')
|
|
39
|
+
for i, line in enumerate(self.lines):
|
|
40
|
+
print(f'{i+1:<4} {line}')
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
|
|
3
|
+
from .base import BaseField
|
|
4
|
+
from .codecs import (
|
|
5
|
+
BitField, Boolean, Bytes,
|
|
6
|
+
Float64, Int8, Int16, Int32, Int64, String, UnsignedInt16, UUID
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SimpleField(BaseField):
|
|
11
|
+
TYPES = {
|
|
12
|
+
'int8': Int8,
|
|
13
|
+
'int16': Int16,
|
|
14
|
+
'uint16': UnsignedInt16,
|
|
15
|
+
'int32': Int32,
|
|
16
|
+
#'uint32': UnsignedInt32,
|
|
17
|
+
'int64': Int64,
|
|
18
|
+
'float64': Float64,
|
|
19
|
+
'bool': Boolean,
|
|
20
|
+
'uuid': UUID,
|
|
21
|
+
'string': String('utf-8'),
|
|
22
|
+
'bytes': Bytes,
|
|
23
|
+
'records': Bytes,
|
|
24
|
+
'bitfield': BitField, # patched only; does not exist in raw schemas
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
@classmethod
|
|
28
|
+
def parse_json(cls, json):
|
|
29
|
+
if 'fields' not in json and json['type'] in cls.TYPES:
|
|
30
|
+
return cls(json)
|
|
31
|
+
|
|
32
|
+
def __init__(self, json):
|
|
33
|
+
if 'fields' in json:
|
|
34
|
+
raise ValueError('Fields not allowed in SimpleField!')
|
|
35
|
+
super().__init__(json)
|
|
36
|
+
if self._type_str not in self.TYPES:
|
|
37
|
+
raise ValueError('Unrecognized type: %s' % self._type_str)
|
|
38
|
+
self._type = self.TYPES[self._type_str]
|
|
39
|
+
|
|
40
|
+
def is_batchable(self):
|
|
41
|
+
return getattr(self._type, 'batchable', False)
|
|
42
|
+
|
|
43
|
+
def _calculate_default(self, default):
|
|
44
|
+
if self._type is Boolean:
|
|
45
|
+
if not default:
|
|
46
|
+
return False
|
|
47
|
+
if isinstance(default, str):
|
|
48
|
+
if default.lower() == 'false':
|
|
49
|
+
return False
|
|
50
|
+
elif default.lower() == 'true':
|
|
51
|
+
return True
|
|
52
|
+
else:
|
|
53
|
+
raise ValueError('Invalid default for boolean field %s: %s' % (self._name, default))
|
|
54
|
+
return bool(default)
|
|
55
|
+
elif self._type in (Int8, Int16, Int32, Int64, UnsignedInt16):
|
|
56
|
+
if not default:
|
|
57
|
+
return 0
|
|
58
|
+
if isinstance(default, str):
|
|
59
|
+
if default.lower().startswith('0x'):
|
|
60
|
+
return int(default, 16)
|
|
61
|
+
else:
|
|
62
|
+
return int(default)
|
|
63
|
+
return int(default)
|
|
64
|
+
elif self._type is UUID:
|
|
65
|
+
if not default:
|
|
66
|
+
return None
|
|
67
|
+
else:
|
|
68
|
+
return uuid.UUID(default)
|
|
69
|
+
elif self._type is Float64:
|
|
70
|
+
if not default:
|
|
71
|
+
return 0.0
|
|
72
|
+
else:
|
|
73
|
+
return float(default)
|
|
74
|
+
elif self._type is BitField:
|
|
75
|
+
if not default:
|
|
76
|
+
return None
|
|
77
|
+
else:
|
|
78
|
+
default = BitField.from_32_bit_field(int(default))
|
|
79
|
+
if default == {31}:
|
|
80
|
+
return None
|
|
81
|
+
return default
|
|
82
|
+
elif default == 'null':
|
|
83
|
+
return None
|
|
84
|
+
elif isinstance(self._type, String):
|
|
85
|
+
return default
|
|
86
|
+
elif not default:
|
|
87
|
+
if self._type is Bytes:
|
|
88
|
+
return b''
|
|
89
|
+
else:
|
|
90
|
+
raise ValueError('Invalid default for field %s. The only valid default is empty or null.' % self._name)
|
|
91
|
+
|
|
92
|
+
def encode(self, value, version=None, compact=False, tagged=False):
|
|
93
|
+
return self._type.encode(value, compact=compact)
|
|
94
|
+
|
|
95
|
+
def encode_into(self, value, out, version=None, compact=False, tagged=False):
|
|
96
|
+
self._type.encode_into(out, value, compact=compact)
|
|
97
|
+
|
|
98
|
+
def emit_encode_into(self, ctx, val_expr, indent, version=None, compact=False, tagged=False):
|
|
99
|
+
self._type.emit_encode_into(ctx, val_expr, indent, compact=compact)
|
|
100
|
+
|
|
101
|
+
def emit_decode_from(self, ctx, var_name, indent, version=None, compact=False, tagged=False):
|
|
102
|
+
self._type.emit_decode_from(ctx, var_name, indent, compact=compact)
|
|
103
|
+
|
|
104
|
+
def decode(self, data, version=None, compact=False, tagged=False):
|
|
105
|
+
return self._type.decode(data, compact=compact)
|
|
106
|
+
|
|
107
|
+
def to_json(self, val):
|
|
108
|
+
if val is None:
|
|
109
|
+
return None
|
|
110
|
+
if self._type is UUID:
|
|
111
|
+
return str(val)
|
|
112
|
+
elif self._type is Bytes:
|
|
113
|
+
if isinstance(val, memoryview):
|
|
114
|
+
val = val.tobytes()
|
|
115
|
+
if hasattr(val, 'to_dict'):
|
|
116
|
+
return val.to_dict()
|
|
117
|
+
elif not isinstance(val, (bytes, bytearray)):
|
|
118
|
+
val = val.encode()
|
|
119
|
+
return val.decode(errors='backslashreplace')
|
|
120
|
+
|
|
121
|
+
elif self._type is BitField:
|
|
122
|
+
return list(val)
|
|
123
|
+
else:
|
|
124
|
+
return val
|
|
125
|
+
|
|
126
|
+
def __repr__(self):
|
|
127
|
+
return 'SimpleField(%s)' % self._json
|