kafka-python 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kafka/__init__.py +34 -0
- kafka/__main__.py +5 -0
- kafka/admin/__init__.py +29 -0
- kafka/admin/__main__.py +5 -0
- kafka/admin/_acls.py +355 -0
- kafka/admin/_cluster.py +359 -0
- kafka/admin/_configs.py +479 -0
- kafka/admin/_groups.py +754 -0
- kafka/admin/_partitions.py +595 -0
- kafka/admin/_topics.py +281 -0
- kafka/admin/_transactions.py +450 -0
- kafka/admin/_users.py +194 -0
- kafka/admin/client.py +373 -0
- kafka/benchmarks/__init__.py +0 -0
- kafka/benchmarks/consumer_performance.py +138 -0
- kafka/benchmarks/load_example.py +109 -0
- kafka/benchmarks/producer_encode_path.py +201 -0
- kafka/benchmarks/producer_performance.py +161 -0
- kafka/benchmarks/profile_protocol.py +138 -0
- kafka/benchmarks/protocol_old_vs_new.py +447 -0
- kafka/benchmarks/record_batch_compose.py +77 -0
- kafka/benchmarks/record_batch_read.py +82 -0
- kafka/benchmarks/varint_speed.py +426 -0
- kafka/cli/__init__.py +36 -0
- kafka/cli/admin/__init__.py +117 -0
- kafka/cli/admin/acls/__init__.py +9 -0
- kafka/cli/admin/acls/common.py +76 -0
- kafka/cli/admin/acls/create.py +19 -0
- kafka/cli/admin/acls/delete.py +23 -0
- kafka/cli/admin/acls/describe.py +16 -0
- kafka/cli/admin/cluster/__init__.py +14 -0
- kafka/cli/admin/cluster/describe.py +11 -0
- kafka/cli/admin/cluster/describe_quorum.py +11 -0
- kafka/cli/admin/cluster/features.py +52 -0
- kafka/cli/admin/cluster/log_dirs.py +43 -0
- kafka/cli/admin/cluster/versions.py +33 -0
- kafka/cli/admin/configs/__init__.py +10 -0
- kafka/cli/admin/configs/alter.py +43 -0
- kafka/cli/admin/configs/common.py +17 -0
- kafka/cli/admin/configs/describe.py +30 -0
- kafka/cli/admin/configs/list.py +16 -0
- kafka/cli/admin/configs/reset.py +20 -0
- kafka/cli/admin/groups/__init__.py +16 -0
- kafka/cli/admin/groups/alter_offsets.py +30 -0
- kafka/cli/admin/groups/delete.py +11 -0
- kafka/cli/admin/groups/delete_offsets.py +29 -0
- kafka/cli/admin/groups/describe.py +11 -0
- kafka/cli/admin/groups/list.py +28 -0
- kafka/cli/admin/groups/list_offsets.py +29 -0
- kafka/cli/admin/groups/remove_members.py +40 -0
- kafka/cli/admin/groups/reset_offsets.py +139 -0
- kafka/cli/admin/partitions/__init__.py +21 -0
- kafka/cli/admin/partitions/alter_reassignments.py +37 -0
- kafka/cli/admin/partitions/create.py +27 -0
- kafka/cli/admin/partitions/delete_records.py +31 -0
- kafka/cli/admin/partitions/describe.py +36 -0
- kafka/cli/admin/partitions/elect_leaders.py +53 -0
- kafka/cli/admin/partitions/list_offsets.py +88 -0
- kafka/cli/admin/partitions/list_reassignments.py +35 -0
- kafka/cli/admin/topics/__init__.py +10 -0
- kafka/cli/admin/topics/create.py +13 -0
- kafka/cli/admin/topics/delete.py +19 -0
- kafka/cli/admin/topics/describe.py +18 -0
- kafka/cli/admin/topics/list.py +11 -0
- kafka/cli/admin/transactions/__init__.py +17 -0
- kafka/cli/admin/transactions/abort.py +38 -0
- kafka/cli/admin/transactions/describe.py +24 -0
- kafka/cli/admin/transactions/describe_producers.py +29 -0
- kafka/cli/admin/transactions/find_hanging.py +26 -0
- kafka/cli/admin/transactions/list.py +37 -0
- kafka/cli/admin/users/__init__.py +8 -0
- kafka/cli/admin/users/alter_user_scram_credentials.py +34 -0
- kafka/cli/admin/users/describe_user_scram_credentials.py +15 -0
- kafka/cli/common.py +95 -0
- kafka/cli/consumer/__init__.py +63 -0
- kafka/cli/producer/__init__.py +57 -0
- kafka/cluster.py +824 -0
- kafka/codec.py +325 -0
- kafka/consumer/__init__.py +5 -0
- kafka/consumer/__main__.py +5 -0
- kafka/consumer/fetcher.py +2012 -0
- kafka/consumer/group.py +1347 -0
- kafka/consumer/subscription_state.py +897 -0
- kafka/coordinator/__init__.py +0 -0
- kafka/coordinator/assignors/__init__.py +0 -0
- kafka/coordinator/assignors/abstract.py +90 -0
- kafka/coordinator/assignors/cooperative_sticky.py +167 -0
- kafka/coordinator/assignors/range.py +81 -0
- kafka/coordinator/assignors/roundrobin.py +101 -0
- kafka/coordinator/assignors/sticky/StickyAssignorUserData.json +37 -0
- kafka/coordinator/assignors/sticky/__init__.py +0 -0
- kafka/coordinator/assignors/sticky/partition_movements.py +149 -0
- kafka/coordinator/assignors/sticky/sorted_set.py +63 -0
- kafka/coordinator/assignors/sticky/sticky_assignor.py +665 -0
- kafka/coordinator/assignors/sticky/user_data.py +8 -0
- kafka/coordinator/base.py +1215 -0
- kafka/coordinator/consumer.py +1224 -0
- kafka/coordinator/heartbeat.py +82 -0
- kafka/coordinator/subscription.py +34 -0
- kafka/errors.py +1004 -0
- kafka/future.py +166 -0
- kafka/metrics/__init__.py +13 -0
- kafka/metrics/compound_stat.py +33 -0
- kafka/metrics/dict_reporter.py +81 -0
- kafka/metrics/kafka_metric.py +36 -0
- kafka/metrics/measurable.py +27 -0
- kafka/metrics/measurable_stat.py +13 -0
- kafka/metrics/metric_config.py +33 -0
- kafka/metrics/metric_name.py +105 -0
- kafka/metrics/metrics.py +261 -0
- kafka/metrics/metrics_reporter.py +53 -0
- kafka/metrics/quota.py +41 -0
- kafka/metrics/stat.py +19 -0
- kafka/metrics/stats/__init__.py +15 -0
- kafka/metrics/stats/avg.py +24 -0
- kafka/metrics/stats/count.py +17 -0
- kafka/metrics/stats/histogram.py +99 -0
- kafka/metrics/stats/max_stat.py +17 -0
- kafka/metrics/stats/min_stat.py +19 -0
- kafka/metrics/stats/percentile.py +14 -0
- kafka/metrics/stats/percentiles.py +75 -0
- kafka/metrics/stats/rate.py +118 -0
- kafka/metrics/stats/sampled_stat.py +99 -0
- kafka/metrics/stats/sensor.py +136 -0
- kafka/metrics/stats/total.py +15 -0
- kafka/net/__init__.py +19 -0
- kafka/net/compat.py +165 -0
- kafka/net/connection.py +593 -0
- kafka/net/http_connect.py +144 -0
- kafka/net/inet.py +122 -0
- kafka/net/manager.py +451 -0
- kafka/net/metrics.py +149 -0
- kafka/net/sasl/__init__.py +32 -0
- kafka/net/sasl/abc.py +28 -0
- kafka/net/sasl/gssapi.py +95 -0
- kafka/net/sasl/msk.py +245 -0
- kafka/net/sasl/oauth.py +98 -0
- kafka/net/sasl/plain.py +42 -0
- kafka/net/sasl/scram.py +135 -0
- kafka/net/sasl/sspi.py +111 -0
- kafka/net/selector.py +644 -0
- kafka/net/socks5.py +262 -0
- kafka/net/transport.py +415 -0
- kafka/net/wakeup_notifier.py +72 -0
- kafka/partitioner/__init__.py +8 -0
- kafka/partitioner/abc.py +8 -0
- kafka/partitioner/default.py +89 -0
- kafka/partitioner/sticky.py +109 -0
- kafka/producer/__init__.py +5 -0
- kafka/producer/__main__.py +5 -0
- kafka/producer/future.py +101 -0
- kafka/producer/kafka.py +1123 -0
- kafka/producer/producer_batch.py +192 -0
- kafka/producer/record_accumulator.py +647 -0
- kafka/producer/sender.py +884 -0
- kafka/producer/transaction_manager.py +1326 -0
- kafka/protocol/__init__.py +0 -0
- kafka/protocol/admin/__init__.py +29 -0
- kafka/protocol/admin/acl.py +83 -0
- kafka/protocol/admin/acl.pyi +375 -0
- kafka/protocol/admin/client_quotas.py +14 -0
- kafka/protocol/admin/client_quotas.pyi +265 -0
- kafka/protocol/admin/cluster.py +31 -0
- kafka/protocol/admin/cluster.pyi +620 -0
- kafka/protocol/admin/configs.py +22 -0
- kafka/protocol/admin/configs.pyi +437 -0
- kafka/protocol/admin/groups.py +24 -0
- kafka/protocol/admin/groups.pyi +261 -0
- kafka/protocol/admin/topics.py +53 -0
- kafka/protocol/admin/topics.pyi +982 -0
- kafka/protocol/admin/transactions.py +18 -0
- kafka/protocol/admin/transactions.pyi +311 -0
- kafka/protocol/admin/users.py +14 -0
- kafka/protocol/admin/users.pyi +223 -0
- kafka/protocol/api_data.py +125 -0
- kafka/protocol/api_header.py +55 -0
- kafka/protocol/api_key.py +97 -0
- kafka/protocol/api_message.py +277 -0
- kafka/protocol/broker_version_data.py +246 -0
- kafka/protocol/consumer/__init__.py +13 -0
- kafka/protocol/consumer/fetch.py +16 -0
- kafka/protocol/consumer/fetch.pyi +298 -0
- kafka/protocol/consumer/group.py +38 -0
- kafka/protocol/consumer/group.pyi +824 -0
- kafka/protocol/consumer/metadata.py +30 -0
- kafka/protocol/consumer/metadata.pyi +89 -0
- kafka/protocol/consumer/offsets.py +75 -0
- kafka/protocol/consumer/offsets.pyi +288 -0
- kafka/protocol/data_container.py +166 -0
- kafka/protocol/frame.py +30 -0
- kafka/protocol/generate_stubs.py +468 -0
- kafka/protocol/metadata/__init__.py +10 -0
- kafka/protocol/metadata/api_versions.py +41 -0
- kafka/protocol/metadata/api_versions.pyi +128 -0
- kafka/protocol/metadata/find_coordinator.py +19 -0
- kafka/protocol/metadata/find_coordinator.pyi +105 -0
- kafka/protocol/metadata/metadata.py +34 -0
- kafka/protocol/metadata/metadata.pyi +160 -0
- kafka/protocol/old/__init__.py +0 -0
- kafka/protocol/old/abstract.py +17 -0
- kafka/protocol/old/add_offsets_to_txn.py +54 -0
- kafka/protocol/old/add_partitions_to_txn.py +71 -0
- kafka/protocol/old/admin.py +1086 -0
- kafka/protocol/old/api.py +205 -0
- kafka/protocol/old/api_versions.py +133 -0
- kafka/protocol/old/commit.py +355 -0
- kafka/protocol/old/consumer_protocol.py +36 -0
- kafka/protocol/old/end_txn.py +53 -0
- kafka/protocol/old/fetch.py +408 -0
- kafka/protocol/old/find_coordinator.py +72 -0
- kafka/protocol/old/group.py +451 -0
- kafka/protocol/old/init_producer_id.py +42 -0
- kafka/protocol/old/list_offsets.py +186 -0
- kafka/protocol/old/metadata.py +290 -0
- kafka/protocol/old/offset_for_leader_epoch.py +133 -0
- kafka/protocol/old/produce.py +247 -0
- kafka/protocol/old/sasl_authenticate.py +38 -0
- kafka/protocol/old/sasl_handshake.py +39 -0
- kafka/protocol/old/struct.py +87 -0
- kafka/protocol/old/txn_offset_commit.py +73 -0
- kafka/protocol/old/types.py +440 -0
- kafka/protocol/parser.py +191 -0
- kafka/protocol/producer/__init__.py +7 -0
- kafka/protocol/producer/produce.py +17 -0
- kafka/protocol/producer/produce.pyi +197 -0
- kafka/protocol/producer/transaction.py +30 -0
- kafka/protocol/producer/transaction.pyi +663 -0
- kafka/protocol/sasl.py +52 -0
- kafka/protocol/sasl.pyi +126 -0
- kafka/protocol/schemas/__init__.py +7 -0
- kafka/protocol/schemas/fields/__init__.py +7 -0
- kafka/protocol/schemas/fields/array.py +127 -0
- kafka/protocol/schemas/fields/base.py +156 -0
- kafka/protocol/schemas/fields/codecs/__init__.py +12 -0
- kafka/protocol/schemas/fields/codecs/encode_buffer.py +82 -0
- kafka/protocol/schemas/fields/codecs/tagged_fields.py +109 -0
- kafka/protocol/schemas/fields/codecs/types.py +505 -0
- kafka/protocol/schemas/fields/codegen.py +40 -0
- kafka/protocol/schemas/fields/simple.py +127 -0
- kafka/protocol/schemas/fields/struct.py +357 -0
- kafka/protocol/schemas/fields/struct_array.py +142 -0
- kafka/protocol/schemas/load_json.py +42 -0
- kafka/protocol/schemas/resources/AddOffsetsToTxnRequest.json +40 -0
- kafka/protocol/schemas/resources/AddOffsetsToTxnResponse.json +35 -0
- kafka/protocol/schemas/resources/AddPartitionsToTxnRequest.json +65 -0
- kafka/protocol/schemas/resources/AddPartitionsToTxnResponse.json +60 -0
- kafka/protocol/schemas/resources/AlterClientQuotasRequest.json +47 -0
- kafka/protocol/schemas/resources/AlterClientQuotasResponse.json +41 -0
- kafka/protocol/schemas/resources/AlterConfigsRequest.json +43 -0
- kafka/protocol/schemas/resources/AlterConfigsResponse.json +39 -0
- kafka/protocol/schemas/resources/AlterPartitionReassignmentsRequest.json +42 -0
- kafka/protocol/schemas/resources/AlterPartitionReassignmentsResponse.json +47 -0
- kafka/protocol/schemas/resources/AlterReplicaLogDirsRequest.json +41 -0
- kafka/protocol/schemas/resources/AlterReplicaLogDirsResponse.json +41 -0
- kafka/protocol/schemas/resources/AlterUserScramCredentialsRequest.json +45 -0
- kafka/protocol/schemas/resources/AlterUserScramCredentialsResponse.json +35 -0
- kafka/protocol/schemas/resources/ApiVersionsRequest.json +34 -0
- kafka/protocol/schemas/resources/ApiVersionsResponse.json +79 -0
- kafka/protocol/schemas/resources/ConsumerProtocolAssignment.json +42 -0
- kafka/protocol/schemas/resources/ConsumerProtocolSubscription.json +49 -0
- kafka/protocol/schemas/resources/CreateAclsRequest.json +46 -0
- kafka/protocol/schemas/resources/CreateAclsResponse.json +37 -0
- kafka/protocol/schemas/resources/CreatePartitionsRequest.json +47 -0
- kafka/protocol/schemas/resources/CreatePartitionsResponse.json +41 -0
- kafka/protocol/schemas/resources/CreateTopicsRequest.json +65 -0
- kafka/protocol/schemas/resources/CreateTopicsResponse.json +72 -0
- kafka/protocol/schemas/resources/DeleteAclsRequest.json +46 -0
- kafka/protocol/schemas/resources/DeleteAclsResponse.json +59 -0
- kafka/protocol/schemas/resources/DeleteGroupsRequest.json +30 -0
- kafka/protocol/schemas/resources/DeleteGroupsResponse.json +36 -0
- kafka/protocol/schemas/resources/DeleteRecordsRequest.json +42 -0
- kafka/protocol/schemas/resources/DeleteRecordsResponse.json +43 -0
- kafka/protocol/schemas/resources/DeleteTopicsRequest.json +43 -0
- kafka/protocol/schemas/resources/DeleteTopicsResponse.json +52 -0
- kafka/protocol/schemas/resources/DescribeAclsRequest.json +43 -0
- kafka/protocol/schemas/resources/DescribeAclsResponse.json +55 -0
- kafka/protocol/schemas/resources/DescribeClientQuotasRequest.json +37 -0
- kafka/protocol/schemas/resources/DescribeClientQuotasResponse.json +47 -0
- kafka/protocol/schemas/resources/DescribeClusterRequest.json +35 -0
- kafka/protocol/schemas/resources/DescribeClusterResponse.json +56 -0
- kafka/protocol/schemas/resources/DescribeConfigsRequest.json +42 -0
- kafka/protocol/schemas/resources/DescribeConfigsResponse.json +69 -0
- kafka/protocol/schemas/resources/DescribeGroupsRequest.json +38 -0
- kafka/protocol/schemas/resources/DescribeGroupsResponse.json +74 -0
- kafka/protocol/schemas/resources/DescribeLogDirsRequest.json +38 -0
- kafka/protocol/schemas/resources/DescribeLogDirsResponse.json +65 -0
- kafka/protocol/schemas/resources/DescribeProducersRequest.json +32 -0
- kafka/protocol/schemas/resources/DescribeProducersResponse.json +55 -0
- kafka/protocol/schemas/resources/DescribeQuorumRequest.json +39 -0
- kafka/protocol/schemas/resources/DescribeQuorumResponse.json +82 -0
- kafka/protocol/schemas/resources/DescribeTopicPartitionsRequest.json +40 -0
- kafka/protocol/schemas/resources/DescribeTopicPartitionsResponse.json +66 -0
- kafka/protocol/schemas/resources/DescribeTransactionsRequest.json +27 -0
- kafka/protocol/schemas/resources/DescribeTransactionsResponse.json +52 -0
- kafka/protocol/schemas/resources/DescribeUserScramCredentialsRequest.json +30 -0
- kafka/protocol/schemas/resources/DescribeUserScramCredentialsResponse.json +45 -0
- kafka/protocol/schemas/resources/ElectLeadersRequest.json +41 -0
- kafka/protocol/schemas/resources/ElectLeadersResponse.json +45 -0
- kafka/protocol/schemas/resources/EndTxnRequest.json +43 -0
- kafka/protocol/schemas/resources/EndTxnResponse.json +41 -0
- kafka/protocol/schemas/resources/FetchRequest.json +125 -0
- kafka/protocol/schemas/resources/FetchResponse.json +124 -0
- kafka/protocol/schemas/resources/FindCoordinatorRequest.json +43 -0
- kafka/protocol/schemas/resources/FindCoordinatorResponse.json +58 -0
- kafka/protocol/schemas/resources/HeartbeatRequest.json +39 -0
- kafka/protocol/schemas/resources/HeartbeatResponse.json +35 -0
- kafka/protocol/schemas/resources/IncrementalAlterConfigsRequest.json +44 -0
- kafka/protocol/schemas/resources/IncrementalAlterConfigsResponse.json +38 -0
- kafka/protocol/schemas/resources/InitProducerIdRequest.json +50 -0
- kafka/protocol/schemas/resources/InitProducerIdResponse.json +47 -0
- kafka/protocol/schemas/resources/JoinGroupRequest.json +63 -0
- kafka/protocol/schemas/resources/JoinGroupResponse.json +69 -0
- kafka/protocol/schemas/resources/LeaveGroupRequest.json +47 -0
- kafka/protocol/schemas/resources/LeaveGroupResponse.json +47 -0
- kafka/protocol/schemas/resources/ListConfigResourcesRequest.json +31 -0
- kafka/protocol/schemas/resources/ListConfigResourcesResponse.json +37 -0
- kafka/protocol/schemas/resources/ListGroupsRequest.json +36 -0
- kafka/protocol/schemas/resources/ListGroupsResponse.json +49 -0
- kafka/protocol/schemas/resources/ListOffsetsRequest.json +72 -0
- kafka/protocol/schemas/resources/ListOffsetsResponse.json +71 -0
- kafka/protocol/schemas/resources/ListPartitionReassignmentsRequest.json +34 -0
- kafka/protocol/schemas/resources/ListPartitionReassignmentsResponse.json +46 -0
- kafka/protocol/schemas/resources/ListTransactionsRequest.json +40 -0
- kafka/protocol/schemas/resources/ListTransactionsResponse.json +42 -0
- kafka/protocol/schemas/resources/MetadataRequest.json +56 -0
- kafka/protocol/schemas/resources/MetadataResponse.json +101 -0
- kafka/protocol/schemas/resources/OffsetCommitRequest.json +76 -0
- kafka/protocol/schemas/resources/OffsetCommitResponse.json +71 -0
- kafka/protocol/schemas/resources/OffsetDeleteRequest.json +39 -0
- kafka/protocol/schemas/resources/OffsetDeleteResponse.json +42 -0
- kafka/protocol/schemas/resources/OffsetFetchRequest.json +76 -0
- kafka/protocol/schemas/resources/OffsetFetchResponse.json +107 -0
- kafka/protocol/schemas/resources/OffsetForLeaderEpochRequest.json +52 -0
- kafka/protocol/schemas/resources/OffsetForLeaderEpochResponse.json +51 -0
- kafka/protocol/schemas/resources/ProduceRequest.json +73 -0
- kafka/protocol/schemas/resources/ProduceResponse.json +96 -0
- kafka/protocol/schemas/resources/RequestHeader.json +44 -0
- kafka/protocol/schemas/resources/ResponseHeader.json +26 -0
- kafka/protocol/schemas/resources/SaslAuthenticateRequest.json +29 -0
- kafka/protocol/schemas/resources/SaslAuthenticateResponse.json +34 -0
- kafka/protocol/schemas/resources/SaslHandshakeRequest.json +31 -0
- kafka/protocol/schemas/resources/SaslHandshakeResponse.json +32 -0
- kafka/protocol/schemas/resources/SyncGroupRequest.json +56 -0
- kafka/protocol/schemas/resources/SyncGroupResponse.json +46 -0
- kafka/protocol/schemas/resources/TxnOffsetCommitRequest.json +68 -0
- kafka/protocol/schemas/resources/TxnOffsetCommitResponse.json +47 -0
- kafka/protocol/schemas/resources/UpdateFeaturesRequest.json +43 -0
- kafka/protocol/schemas/resources/UpdateFeaturesResponse.json +39 -0
- kafka/protocol/schemas/resources/WriteTxnMarkersRequest.json +49 -0
- kafka/protocol/schemas/resources/WriteTxnMarkersResponse.json +45 -0
- kafka/protocol/schemas/resources/__init__.py +0 -0
- kafka/record/__init__.py +3 -0
- kafka/record/_crc32c.py +161 -0
- kafka/record/abc.py +144 -0
- kafka/record/default_records.py +782 -0
- kafka/record/legacy_records.py +587 -0
- kafka/record/memory_records.py +255 -0
- kafka/record/util.py +135 -0
- kafka/serializer/__init__.py +4 -0
- kafka/serializer/abstract.py +20 -0
- kafka/serializer/default.py +16 -0
- kafka/serializer/json.py +17 -0
- kafka/serializer/wrapper.py +21 -0
- kafka/structs.py +69 -0
- kafka/util.py +159 -0
- kafka/vendor/__init__.py +0 -0
- kafka/version.py +1 -0
- kafka_python-3.0.0.dist-info/METADATA +319 -0
- kafka_python-3.0.0.dist-info/RECORD +373 -0
- kafka_python-3.0.0.dist-info/WHEEL +5 -0
- kafka_python-3.0.0.dist-info/entry_points.txt +2 -0
- kafka_python-3.0.0.dist-info/licenses/LICENSE +202 -0
- kafka_python-3.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
from .base import BaseField
|
|
2
|
+
from .codecs.tagged_fields import TaggedFields
|
|
3
|
+
from .codecs.types import UnsignedVarInt32
|
|
4
|
+
from .codegen import CodegenContext
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class StructField(BaseField):
|
|
8
|
+
@classmethod
|
|
9
|
+
def parse_json(cls, json):
|
|
10
|
+
if 'type' not in json or json['type'].startswith('[]'):
|
|
11
|
+
return
|
|
12
|
+
if 'fields' in json:
|
|
13
|
+
return cls(json)
|
|
14
|
+
|
|
15
|
+
# Cases
|
|
16
|
+
# oldschool - standard types, no tagged fields
|
|
17
|
+
# newschool - compact types, tagged fields
|
|
18
|
+
# nested tag - compact types, no (nested) tagged fields
|
|
19
|
+
def __init__(self, json):
|
|
20
|
+
super().__init__(json)
|
|
21
|
+
self._field_map = {field.name: field for field in self._fields}
|
|
22
|
+
self._data_class = None
|
|
23
|
+
self._untagged_fields_cache = {}
|
|
24
|
+
self._tagged_fields_cache = {}
|
|
25
|
+
self._compiled_encoders = {}
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def fields(self):
|
|
29
|
+
return self._field_map
|
|
30
|
+
|
|
31
|
+
def is_struct(self):
|
|
32
|
+
return True
|
|
33
|
+
|
|
34
|
+
def has_data_class(self):
|
|
35
|
+
return self._data_class is not None
|
|
36
|
+
|
|
37
|
+
def set_data_class(self, data_class):
|
|
38
|
+
assert self._data_class is None
|
|
39
|
+
self._data_class = data_class
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def data_class(self):
|
|
43
|
+
return self._data_class
|
|
44
|
+
|
|
45
|
+
def __call__(self, *args, **kw):
|
|
46
|
+
return self.data_class(*args, **kw) # pylint: disable=E1102
|
|
47
|
+
|
|
48
|
+
def _calculate_default(self, default):
|
|
49
|
+
if default == 'null':
|
|
50
|
+
return None
|
|
51
|
+
if self._tag is not None:
|
|
52
|
+
return None
|
|
53
|
+
elif not default:
|
|
54
|
+
raise NotImplementedError(f"Default value not implemented for struct field '{self._name}'")
|
|
55
|
+
else:
|
|
56
|
+
raise ValueError('Invalid default for struct field %s. The only valid default is null.' % self._name)
|
|
57
|
+
|
|
58
|
+
def tagged_fields(self, version):
|
|
59
|
+
if version not in self._tagged_fields_cache:
|
|
60
|
+
self._tagged_fields_cache[version] = TaggedFields(
|
|
61
|
+
[field for field in self._fields
|
|
62
|
+
if field.for_version_q(version)
|
|
63
|
+
and field.tagged_field_q(version)])
|
|
64
|
+
return self._tagged_fields_cache[version]
|
|
65
|
+
|
|
66
|
+
def untagged_fields(self, version):
|
|
67
|
+
if version not in self._untagged_fields_cache:
|
|
68
|
+
self._untagged_fields_cache[version] = [
|
|
69
|
+
field for field in self._fields
|
|
70
|
+
if field.for_version_q(version)
|
|
71
|
+
and not field.tagged_field_q(version)]
|
|
72
|
+
return self._untagged_fields_cache[version]
|
|
73
|
+
|
|
74
|
+
def encode(self, item, version=None, compact=False, tagged=False):
|
|
75
|
+
# Nested nullable struct (KIP-893): 1-byte INT8 prefix, -1 = null,
|
|
76
|
+
# 1 = present. Top-level message structs never have nullableVersions
|
|
77
|
+
# set, so this check is safe without a top-level guard.
|
|
78
|
+
if self.nullable_for_version_q(version):
|
|
79
|
+
if item is None:
|
|
80
|
+
return b'\xff'
|
|
81
|
+
prefix = b'\x01'
|
|
82
|
+
else:
|
|
83
|
+
prefix = b''
|
|
84
|
+
fields = self.untagged_fields(version)
|
|
85
|
+
if isinstance(item, tuple):
|
|
86
|
+
getter = lambda item, i, field: item[i]
|
|
87
|
+
tags = {} if len(item) == len(fields) else item[-1]
|
|
88
|
+
elif isinstance(item, dict):
|
|
89
|
+
getter = lambda item, i, field: item.get(field.name) # defaults?
|
|
90
|
+
tags = item
|
|
91
|
+
elif isinstance(item, (str, int, float)):
|
|
92
|
+
assert len(fields) == 1, "Encoding single value item (str/int/float) requires single field struct"
|
|
93
|
+
getter = lambda item, i, field: item
|
|
94
|
+
tags = {}
|
|
95
|
+
else:
|
|
96
|
+
getter = lambda item, i, field: getattr(item, field.name)
|
|
97
|
+
tags = item
|
|
98
|
+
encoded = [field.encode(getter(item, i, field),
|
|
99
|
+
version=version, compact=compact, tagged=tagged)
|
|
100
|
+
for i, field in enumerate(fields)]
|
|
101
|
+
if tagged:
|
|
102
|
+
# TaggedFields are always compact and never include nested tagged fields
|
|
103
|
+
encoded.append(self.tagged_fields(version).encode(tags, version=version))
|
|
104
|
+
elif tagged is None:
|
|
105
|
+
encoded.append(TaggedFields.encode_empty())
|
|
106
|
+
return prefix + b''.join(encoded)
|
|
107
|
+
|
|
108
|
+
def emit_encode_into(self, ctx, item_expr, indent, version=None, compact=False,
|
|
109
|
+
tagged=False, tuple_access=False):
|
|
110
|
+
# Top-level struct (item_expr == 'item') has its nullability handled
|
|
111
|
+
# by the parent struct; only inline null-prefix when this is a nested
|
|
112
|
+
# nullable struct field.
|
|
113
|
+
inline_nullable = (
|
|
114
|
+
self.nullable_for_version_q(version)
|
|
115
|
+
and item_expr != 'item'
|
|
116
|
+
and not tuple_access
|
|
117
|
+
)
|
|
118
|
+
if inline_nullable:
|
|
119
|
+
ctx.emit(indent, 'if %s is None:' % item_expr)
|
|
120
|
+
ctx.emit(indent, ' buf[pos] = 0xff')
|
|
121
|
+
ctx.emit(indent, ' pos += 1')
|
|
122
|
+
ctx.emit(indent, 'else:')
|
|
123
|
+
ctx.emit(indent, ' buf[pos] = 1')
|
|
124
|
+
ctx.emit(indent, ' pos += 1')
|
|
125
|
+
indent = indent + ' '
|
|
126
|
+
fields = self.untagged_fields(version)
|
|
127
|
+
for i, field in enumerate(fields):
|
|
128
|
+
if tuple_access:
|
|
129
|
+
val_expr = '%s[%d]' % (item_expr, i)
|
|
130
|
+
else:
|
|
131
|
+
val_expr = '%s.%s' % (item_expr, field.name)
|
|
132
|
+
field.emit_encode_into(ctx, val_expr, indent, version=version,
|
|
133
|
+
compact=compact, tagged=tagged)
|
|
134
|
+
if tagged:
|
|
135
|
+
tf_var = ctx.next_var('tf')
|
|
136
|
+
ctx.globs[tf_var] = self.tagged_fields(version)
|
|
137
|
+
ctx.emit(indent, 'out.pos = pos')
|
|
138
|
+
ctx.emit(indent, '%s.encode_into(%s, out, version=%d)' % (tf_var, item_expr, version))
|
|
139
|
+
ctx.emit(indent, 'pos = out.pos')
|
|
140
|
+
elif tagged is None:
|
|
141
|
+
ctx.emit(indent, 'buf[pos] = 0')
|
|
142
|
+
ctx.emit(indent, 'pos += 1')
|
|
143
|
+
|
|
144
|
+
def encode_into(self, item, out, version=None, compact=False, tagged=False):
|
|
145
|
+
if self.nullable_for_version_q(version):
|
|
146
|
+
out.ensure(1)
|
|
147
|
+
if item is None:
|
|
148
|
+
out.buf[out.pos] = 0xff
|
|
149
|
+
out.pos += 1
|
|
150
|
+
return
|
|
151
|
+
out.buf[out.pos] = 1
|
|
152
|
+
out.pos += 1
|
|
153
|
+
fields = self.untagged_fields(version)
|
|
154
|
+
if isinstance(item, tuple):
|
|
155
|
+
for i, field in enumerate(fields):
|
|
156
|
+
field.encode_into(item[i], out, version=version, compact=compact, tagged=tagged)
|
|
157
|
+
elif isinstance(item, dict):
|
|
158
|
+
for field in fields:
|
|
159
|
+
field.encode_into(item.get(field.name), out, version=version, compact=compact, tagged=tagged)
|
|
160
|
+
elif isinstance(item, (str, int, float)):
|
|
161
|
+
fields[0].encode_into(item, out, version=version, compact=compact, tagged=tagged)
|
|
162
|
+
else:
|
|
163
|
+
for field in fields:
|
|
164
|
+
field.encode_into(getattr(item, field.name), out, version=version, compact=compact, tagged=tagged)
|
|
165
|
+
if tagged:
|
|
166
|
+
self.tagged_fields(version).encode_into(item, out, version=version)
|
|
167
|
+
elif tagged is None:
|
|
168
|
+
UnsignedVarInt32.encode_into(out, 0)
|
|
169
|
+
|
|
170
|
+
def encode_into__optimized_context(self, version, compact=False, tagged=False):
|
|
171
|
+
ctx = CodegenContext()
|
|
172
|
+
indent = ' '
|
|
173
|
+
ctx.lines.append('def _encode(item, out):')
|
|
174
|
+
ctx.emit(indent, 'buf = out.buf')
|
|
175
|
+
ctx.emit(indent, 'pos = out.pos')
|
|
176
|
+
self.emit_encode_into(ctx, 'item', indent, version=version, compact=compact, tagged=tagged)
|
|
177
|
+
ctx.emit(indent, 'out.pos = pos')
|
|
178
|
+
return ctx
|
|
179
|
+
|
|
180
|
+
def compiled_encode_into(self, version, compact=False, tagged=False):
|
|
181
|
+
"""Return a compiled flat encode function for this struct+version.
|
|
182
|
+
|
|
183
|
+
Lazily compiled on first call and cached. The returned function has
|
|
184
|
+
signature: f(item, out) where out is an EncodeBuffer.
|
|
185
|
+
"""
|
|
186
|
+
key = (version, compact, tagged)
|
|
187
|
+
if key not in self._compiled_encoders:
|
|
188
|
+
ctx = self.encode_into__optimized_context(version, compact=compact, tagged=tagged)
|
|
189
|
+
exec(compile(ctx.source(), '<codegen:%s_v%d>' % (self.name, version), 'exec'), ctx.globs)
|
|
190
|
+
self._compiled_encoders[key] = ctx.globs['_encode']
|
|
191
|
+
return self._compiled_encoders[key]
|
|
192
|
+
|
|
193
|
+
def emit_decode_from(self, ctx, var_name, indent, version=None, compact=False, tagged=False):
|
|
194
|
+
"""Emit decode code that creates a DataContainer via __new__ + direct slot assignment.
|
|
195
|
+
|
|
196
|
+
Batches adjacent batchable fields into single unpack_from calls.
|
|
197
|
+
"""
|
|
198
|
+
# Top-level struct decode (var_name == 'obj') has no outer null-prefix;
|
|
199
|
+
# only a nested nullable struct field consumes one.
|
|
200
|
+
inline_nullable = (
|
|
201
|
+
self.nullable_for_version_q(version)
|
|
202
|
+
and var_name != 'obj'
|
|
203
|
+
)
|
|
204
|
+
if inline_nullable:
|
|
205
|
+
ctx.emit(indent, 'if data[pos] == 0xff:')
|
|
206
|
+
ctx.emit(indent, ' pos += 1')
|
|
207
|
+
ctx.emit(indent, ' %s = None' % var_name)
|
|
208
|
+
ctx.emit(indent, 'else:')
|
|
209
|
+
ctx.emit(indent, ' pos += 1')
|
|
210
|
+
indent = indent + ' '
|
|
211
|
+
fields = self.untagged_fields(version)
|
|
212
|
+
data_class = self.data_class
|
|
213
|
+
|
|
214
|
+
if data_class is not None:
|
|
215
|
+
dc_var = ctx.next_var('dc')
|
|
216
|
+
ctx.globs[dc_var] = data_class
|
|
217
|
+
ctx.emit(indent, '%s = object.__new__(%s)' % (var_name, dc_var))
|
|
218
|
+
ctx.emit(indent, '%s._version = %d' % (var_name, version))
|
|
219
|
+
ctx.emit(indent, '%s.tags = None' % var_name)
|
|
220
|
+
ctx.emit(indent, '%s.unknown_tags = None' % var_name)
|
|
221
|
+
|
|
222
|
+
# Walk fields, batching adjacent batchable fields
|
|
223
|
+
i = 0
|
|
224
|
+
while i < len(fields):
|
|
225
|
+
field = fields[i]
|
|
226
|
+
|
|
227
|
+
# Try to batch consecutive batchable fields.
|
|
228
|
+
if field.is_batchable():
|
|
229
|
+
# Collect the run
|
|
230
|
+
batch_fields = []
|
|
231
|
+
batch_fmt = '>'
|
|
232
|
+
batch_size = 0
|
|
233
|
+
while i < len(fields):
|
|
234
|
+
f = fields[i]
|
|
235
|
+
if f.is_batchable():
|
|
236
|
+
batch_fields.append(f)
|
|
237
|
+
batch_fmt += f._type.fmt
|
|
238
|
+
batch_size += f._type.size
|
|
239
|
+
i += 1
|
|
240
|
+
else:
|
|
241
|
+
break
|
|
242
|
+
|
|
243
|
+
if len(batch_fields) == 1:
|
|
244
|
+
# Single field - no batching benefit
|
|
245
|
+
f = batch_fields[0]
|
|
246
|
+
v = ctx.next_var('val')
|
|
247
|
+
f.emit_decode_from(ctx, v, indent, version=version,
|
|
248
|
+
compact=compact, tagged=tagged)
|
|
249
|
+
if data_class is not None:
|
|
250
|
+
ctx.emit(indent, '%s.%s = %s' % (var_name, f.name, v))
|
|
251
|
+
else:
|
|
252
|
+
# Batched unpack
|
|
253
|
+
var_names = [ctx.next_var('val') for _ in batch_fields]
|
|
254
|
+
ctx.emit(indent, '%s = unpack_from("%s", data, pos)' % (
|
|
255
|
+
', '.join(var_names), batch_fmt))
|
|
256
|
+
ctx.emit(indent, 'pos += %d' % batch_size)
|
|
257
|
+
if data_class is not None:
|
|
258
|
+
for f, v in zip(batch_fields, var_names):
|
|
259
|
+
ctx.emit(indent, '%s.%s = %s' % (var_name, f.name, v))
|
|
260
|
+
else:
|
|
261
|
+
# Non-batchable field (String, Bytes, Array, Struct, etc.)
|
|
262
|
+
v = ctx.next_var('val')
|
|
263
|
+
field.emit_decode_from(ctx, v, indent, version=version,
|
|
264
|
+
compact=compact, tagged=tagged)
|
|
265
|
+
if data_class is not None:
|
|
266
|
+
ctx.emit(indent, '%s.%s = %s' % (var_name, field.name, v))
|
|
267
|
+
i += 1
|
|
268
|
+
|
|
269
|
+
if tagged:
|
|
270
|
+
# Fast path: most messages have zero tagged fields (single 0x00 byte).
|
|
271
|
+
# Only fall back to full TaggedFields.decode when tags are present.
|
|
272
|
+
ctx.emit(indent, 'if data[pos] == 0:')
|
|
273
|
+
ctx.emit(indent, ' pos += 1')
|
|
274
|
+
ctx.emit(indent, 'else:')
|
|
275
|
+
self._emit_tagged_decode(ctx, var_name, indent + ' ', version)
|
|
276
|
+
elif tagged is None:
|
|
277
|
+
# Empty tagged fields: single 0x00 byte
|
|
278
|
+
ctx.emit(indent, 'pos += 1 # empty tagged fields')
|
|
279
|
+
|
|
280
|
+
def _emit_tagged_decode(self, ctx, var_name, indent, version):
|
|
281
|
+
"""Emit tagged fields decode - falls back to method-based decode."""
|
|
282
|
+
tf_var = ctx.next_var('tf')
|
|
283
|
+
ctx.globs[tf_var] = self.tagged_fields(version)
|
|
284
|
+
ctx.globs['_BytesIO'] = __import__('io').BytesIO
|
|
285
|
+
bio_var = ctx.next_var('bio')
|
|
286
|
+
ctx.emit(indent, '%s = _BytesIO(bytes(data[pos:]))' % bio_var)
|
|
287
|
+
ctx.emit(indent, '_tfd = %s.decode(%s, version=%d)' % (tf_var, bio_var, version))
|
|
288
|
+
ctx.emit(indent, 'pos += %s.tell()' % bio_var)
|
|
289
|
+
ctx.emit(indent, 'if _tfd:')
|
|
290
|
+
ctx.emit(indent, ' for _tk, _tv in _tfd.items():')
|
|
291
|
+
ctx.emit(indent, ' if _tk.startswith("_"):')
|
|
292
|
+
ctx.emit(indent, ' if %s.unknown_tags is None: %s.unknown_tags = {}' % (var_name, var_name))
|
|
293
|
+
ctx.emit(indent, ' %s.unknown_tags[_tk] = _tv' % var_name)
|
|
294
|
+
ctx.emit(indent, ' else:')
|
|
295
|
+
ctx.emit(indent, ' if %s.tags is None: %s.tags = set()' % (var_name, var_name))
|
|
296
|
+
ctx.emit(indent, ' %s.tags.add(_tk)' % var_name)
|
|
297
|
+
ctx.emit(indent, ' setattr(%s, _tk, _tv)' % var_name)
|
|
298
|
+
|
|
299
|
+
def compiled_decode_from(self, version, compact=False, tagged=False, data_class=None):
|
|
300
|
+
"""Return a compiled flat decode function for this struct+version.
|
|
301
|
+
|
|
302
|
+
Lazily compiled on first call and cached. The returned function has
|
|
303
|
+
signature: f(data, pos) -> (obj, pos) where data is a memoryview/bytes.
|
|
304
|
+
data_class overrides self.data_class for the top-level object (needed
|
|
305
|
+
to resolve weakref proxies on ApiMessage classes).
|
|
306
|
+
"""
|
|
307
|
+
key = ('decode', version, compact, tagged, data_class)
|
|
308
|
+
if key not in self._compiled_encoders:
|
|
309
|
+
# Temporarily override data_class for codegen if provided
|
|
310
|
+
saved = self._data_class
|
|
311
|
+
if data_class is not None:
|
|
312
|
+
self._data_class = data_class
|
|
313
|
+
try:
|
|
314
|
+
ctx = CodegenContext()
|
|
315
|
+
indent = ' '
|
|
316
|
+
ctx.lines.append('def _decode(data, pos):')
|
|
317
|
+
self.emit_decode_from(ctx, 'obj', indent, version=version,
|
|
318
|
+
compact=compact, tagged=tagged)
|
|
319
|
+
ctx.emit(indent, 'return obj, pos')
|
|
320
|
+
code = ctx.source()
|
|
321
|
+
exec(compile(code, '<codegen_decode:%s_v%d>' % (self.name, version), 'exec'), ctx.globs)
|
|
322
|
+
self._compiled_encoders[key] = ctx.globs['_decode']
|
|
323
|
+
finally:
|
|
324
|
+
self._data_class = saved
|
|
325
|
+
return self._compiled_encoders[key]
|
|
326
|
+
|
|
327
|
+
def decode(self, data, version=None, compact=False, tagged=False, data_class=None):
|
|
328
|
+
if self.nullable_for_version_q(version):
|
|
329
|
+
if data.read(1) == b'\xff':
|
|
330
|
+
return None
|
|
331
|
+
if data_class is None:
|
|
332
|
+
data_class = self.data_class
|
|
333
|
+
decoded = {
|
|
334
|
+
field.name: field.decode(data, version=version, compact=compact, tagged=tagged)
|
|
335
|
+
for field in self.untagged_fields(version)
|
|
336
|
+
}
|
|
337
|
+
if tagged:
|
|
338
|
+
decoded.update(self.tagged_fields(version).decode(data, version=version))
|
|
339
|
+
elif tagged is None:
|
|
340
|
+
TaggedFields.decode_empty(data)
|
|
341
|
+
|
|
342
|
+
if data_class is not None:
|
|
343
|
+
return data_class(version=version, **decoded)
|
|
344
|
+
return decoded
|
|
345
|
+
|
|
346
|
+
def __len__(self):
|
|
347
|
+
return len(self._fields)
|
|
348
|
+
|
|
349
|
+
def __eq__(self, other):
|
|
350
|
+
if not super().__eq__(other):
|
|
351
|
+
return False
|
|
352
|
+
if self._fields != other._fields:
|
|
353
|
+
return False
|
|
354
|
+
return True
|
|
355
|
+
|
|
356
|
+
def __repr__(self):
|
|
357
|
+
return 'StructField(%s)' % self._json
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from .array import ArrayField
|
|
2
|
+
from .struct import StructField
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class StructArrayField(ArrayField):
|
|
6
|
+
@classmethod
|
|
7
|
+
def parse_inner_type(cls, json):
|
|
8
|
+
# StructArrayField requires non-empty fields
|
|
9
|
+
if 'fields' not in json:
|
|
10
|
+
return
|
|
11
|
+
assert len(json['fields']) > 0, 'Unexpected empty fields in json'
|
|
12
|
+
type_str = cls.parse_array_type(json)
|
|
13
|
+
if type_str is None:
|
|
14
|
+
return
|
|
15
|
+
inner_json = {**json, 'type': type_str}
|
|
16
|
+
return StructField.parse_json(inner_json)
|
|
17
|
+
|
|
18
|
+
@classmethod
|
|
19
|
+
def parse_json(cls, json):
|
|
20
|
+
inner_type = cls.parse_inner_type(json)
|
|
21
|
+
if inner_type is not None:
|
|
22
|
+
return cls(json, array_of=inner_type)
|
|
23
|
+
|
|
24
|
+
def __init__(self, json, array_of=None):
|
|
25
|
+
if array_of is None:
|
|
26
|
+
array_of = self.parse_inner_type(json)
|
|
27
|
+
assert array_of is not None, 'json does not contain a StructArray!'
|
|
28
|
+
super().__init__(json, array_of=array_of)
|
|
29
|
+
# nullableVersions on the JSON describes the array's nullability, not
|
|
30
|
+
# the inner struct's. Clear it on the inner struct so StructField does
|
|
31
|
+
# not try to emit a per-element null-prefix when encoding/decoding.
|
|
32
|
+
array_of._nullable_versions = None
|
|
33
|
+
# map_key will be (idx, field) of the mapKey field if found
|
|
34
|
+
self.map_key = next(filter(lambda x: x[1]._json.get('mapKey'), enumerate(self._fields)), None)
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def type_str(self):
|
|
38
|
+
return self._type_str[2:]
|
|
39
|
+
|
|
40
|
+
def is_struct_array(self):
|
|
41
|
+
return True
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def fields(self):
|
|
45
|
+
return self.array_of.fields
|
|
46
|
+
|
|
47
|
+
def tagged_fields(self, version):
|
|
48
|
+
return self.array_of.tagged_fields(version)
|
|
49
|
+
|
|
50
|
+
def untagged_fields(self, version):
|
|
51
|
+
return self.array_of.untagged_fields(version)
|
|
52
|
+
|
|
53
|
+
def has_data_class(self):
|
|
54
|
+
return self.array_of.has_data_class()
|
|
55
|
+
|
|
56
|
+
def set_data_class(self, data_class):
|
|
57
|
+
return self.array_of.set_data_class(data_class)
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def data_class(self):
|
|
61
|
+
return self.array_of.data_class
|
|
62
|
+
|
|
63
|
+
def __call__(self, *args, **kw):
|
|
64
|
+
return self.data_class(*args, **kw) # pylint: disable=E1102
|
|
65
|
+
|
|
66
|
+
def emit_decode_from(self, ctx, var_name, indent, version=None, compact=False, tagged=False):
|
|
67
|
+
from .codecs import UnsignedVarInt32
|
|
68
|
+
inner_struct = self.array_of
|
|
69
|
+
n = ctx.next_var('n')
|
|
70
|
+
if compact:
|
|
71
|
+
UnsignedVarInt32.emit_decode_from(ctx, n, indent)
|
|
72
|
+
ctx.emit(indent, '%s -= 1' % n)
|
|
73
|
+
else:
|
|
74
|
+
ctx.emit(indent, '%s = unpack_from(">i", data, pos)[0]' % n)
|
|
75
|
+
ctx.emit(indent, 'pos += 4')
|
|
76
|
+
ctx.emit(indent, 'if %s == -1:' % n)
|
|
77
|
+
ctx.emit(indent, ' %s = None' % var_name)
|
|
78
|
+
ctx.emit(indent, 'else:')
|
|
79
|
+
inner_indent = indent + ' '
|
|
80
|
+
ctx.emit(inner_indent, '%s = []' % var_name)
|
|
81
|
+
idx = ctx.next_var('idx')
|
|
82
|
+
item = ctx.next_var('obj')
|
|
83
|
+
ctx.emit(inner_indent, 'for %s in range(%s):' % (idx, n))
|
|
84
|
+
inner_struct.emit_decode_from(ctx, item, inner_indent + ' ',
|
|
85
|
+
version=version, compact=compact, tagged=tagged)
|
|
86
|
+
ctx.emit(inner_indent, ' %s.append(%s)' % (var_name, item))
|
|
87
|
+
|
|
88
|
+
def emit_encode_into(self, ctx, val_expr, indent, version=None, compact=False, tagged=False):
|
|
89
|
+
from .codecs import UnsignedVarInt32
|
|
90
|
+
inner_struct = self.array_of
|
|
91
|
+
fields = inner_struct.untagged_fields(version)
|
|
92
|
+
if compact:
|
|
93
|
+
an = ctx.next_var('an')
|
|
94
|
+
ctx.emit(indent, '%s = len(%s) + 1 if %s is not None else 0' % (an, val_expr, val_expr))
|
|
95
|
+
UnsignedVarInt32.emit_encode_into(ctx, an, indent)
|
|
96
|
+
ctx.emit(indent, 'if %s is not None:' % val_expr)
|
|
97
|
+
else:
|
|
98
|
+
ctx.emit(indent, 'if %s is None:' % val_expr)
|
|
99
|
+
ctx.emit(indent, " pack_into('>i', buf, pos, -1)")
|
|
100
|
+
ctx.emit(indent, ' pos += 4')
|
|
101
|
+
ctx.emit(indent, 'else:')
|
|
102
|
+
ctx.emit(indent, " pack_into('>i', buf, pos, len(%s))" % val_expr)
|
|
103
|
+
ctx.emit(indent, ' pos += 4')
|
|
104
|
+
guard = indent + ' '
|
|
105
|
+
item_var = ctx.next_var('si')
|
|
106
|
+
if len(fields) == 1:
|
|
107
|
+
# Single-field struct: items may be scalars (str, int, etc.)
|
|
108
|
+
ctx.emit(guard, 'for %s in %s:' % (item_var, val_expr))
|
|
109
|
+
scalar_indent = guard + ' '
|
|
110
|
+
ctx.emit(scalar_indent, 'if isinstance(%s, tuple): %s = %s[0]' % (item_var, item_var, item_var))
|
|
111
|
+
ctx.emit(scalar_indent, 'elif hasattr(%s, "%s"): %s = %s.%s' % (
|
|
112
|
+
item_var, fields[0].name, item_var, item_var, fields[0].name))
|
|
113
|
+
fields[0].emit_encode_into(ctx, item_var, scalar_indent,
|
|
114
|
+
version=version, compact=compact, tagged=tagged)
|
|
115
|
+
if tagged:
|
|
116
|
+
tf_var = ctx.next_var('tf')
|
|
117
|
+
ctx.globs[tf_var] = inner_struct.tagged_fields(version)
|
|
118
|
+
ctx.emit(scalar_indent, 'out.pos = pos')
|
|
119
|
+
ctx.emit(scalar_indent, '# tagged fields for single-field struct')
|
|
120
|
+
ctx.emit(scalar_indent, '%s.encode_into(%s, out, version=%d)' % (tf_var, item_var, version))
|
|
121
|
+
ctx.emit(scalar_indent, 'pos = out.pos')
|
|
122
|
+
elif tagged is None:
|
|
123
|
+
ctx.emit(scalar_indent, 'buf[pos] = 0')
|
|
124
|
+
ctx.emit(scalar_indent, 'pos += 1')
|
|
125
|
+
else:
|
|
126
|
+
# Multi-field struct: emit both tuple and attribute access paths.
|
|
127
|
+
# Use list() to handle non-subscriptable iterables (e.g., dict_items).
|
|
128
|
+
list_var = ctx.next_var('lst')
|
|
129
|
+
ctx.emit(guard, '%s = %s if isinstance(%s, list) else list(%s)' % (list_var, val_expr, val_expr, val_expr))
|
|
130
|
+
ctx.emit(guard, 'if %s and isinstance(%s[0], tuple):' % (list_var, list_var))
|
|
131
|
+
ctx.emit(guard, ' for %s in %s:' % (item_var, list_var))
|
|
132
|
+
inner_struct.emit_encode_into(ctx, item_var, guard + ' ',
|
|
133
|
+
version=version, compact=compact, tagged=tagged,
|
|
134
|
+
tuple_access=True)
|
|
135
|
+
ctx.emit(guard, 'else:')
|
|
136
|
+
ctx.emit(guard, ' for %s in %s:' % (item_var, list_var))
|
|
137
|
+
inner_struct.emit_encode_into(ctx, item_var, guard + ' ',
|
|
138
|
+
version=version, compact=compact, tagged=tagged,
|
|
139
|
+
tuple_access=False)
|
|
140
|
+
|
|
141
|
+
def __repr__(self):
|
|
142
|
+
return 'StructArrayField(%s)' % self._json
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import importlib.resources
|
|
2
|
+
import inspect
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def load_json(msg_type, package=None):
|
|
8
|
+
if package is None:
|
|
9
|
+
package = __package__ + '.resources'
|
|
10
|
+
elif inspect.ismodule(package):
|
|
11
|
+
package = package.__package__
|
|
12
|
+
COMMENTS_REGEX = r"(?m)((?:^\s*//.*\n?)+)"
|
|
13
|
+
# Raises FileNotFoundError if not found
|
|
14
|
+
msg_json = importlib.resources.read_text(package, msg_type + '.json')
|
|
15
|
+
data = json.loads(re.sub(COMMENTS_REGEX, '', msg_json))
|
|
16
|
+
comments = re.findall(COMMENTS_REGEX, msg_json)
|
|
17
|
+
if comments:
|
|
18
|
+
data['license'] = comments[0]
|
|
19
|
+
if len(comments) > 1:
|
|
20
|
+
data['doc'] = comments[1]
|
|
21
|
+
common_structs = {s['name']: s['fields'] for s in data.get('commonStructs', [])}
|
|
22
|
+
if common_structs:
|
|
23
|
+
_resolve_common_structs(data.get('fields', []), common_structs)
|
|
24
|
+
return data
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _resolve_common_structs(fields, common_structs):
|
|
28
|
+
for field in fields:
|
|
29
|
+
field_type = field['type']
|
|
30
|
+
struct_name = None
|
|
31
|
+
if field_type.startswith('[]'):
|
|
32
|
+
inner_type = field_type[2:]
|
|
33
|
+
if inner_type and inner_type[0].isupper():
|
|
34
|
+
struct_name = inner_type
|
|
35
|
+
elif field_type and field_type[0].isupper():
|
|
36
|
+
struct_name = field_type
|
|
37
|
+
|
|
38
|
+
if struct_name and struct_name in common_structs and 'fields' not in field:
|
|
39
|
+
field['fields'] = common_structs[struct_name]
|
|
40
|
+
|
|
41
|
+
if 'fields' in field:
|
|
42
|
+
_resolve_common_structs(field['fields'], common_structs)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one or more
|
|
2
|
+
// contributor license agreements. See the NOTICE file distributed with
|
|
3
|
+
// this work for additional information regarding copyright ownership.
|
|
4
|
+
// The ASF licenses this file to You under the Apache License, Version 2.0
|
|
5
|
+
// (the "License"); you may not use this file except in compliance with
|
|
6
|
+
// the License. You may obtain a copy of the License at
|
|
7
|
+
//
|
|
8
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
//
|
|
10
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
// See the License for the specific language governing permissions and
|
|
14
|
+
// limitations under the License.
|
|
15
|
+
|
|
16
|
+
{
|
|
17
|
+
"apiKey": 25,
|
|
18
|
+
"type": "request",
|
|
19
|
+
"listeners": ["broker"],
|
|
20
|
+
"name": "AddOffsetsToTxnRequest",
|
|
21
|
+
// Version 1 is the same as version 0.
|
|
22
|
+
//
|
|
23
|
+
// Version 2 adds the support for new error code PRODUCER_FENCED.
|
|
24
|
+
//
|
|
25
|
+
// Version 3 enables flexible versions.
|
|
26
|
+
//
|
|
27
|
+
// Version 4 adds support for new error code TRANSACTION_ABORTABLE (KIP-890).
|
|
28
|
+
"validVersions": "0-4",
|
|
29
|
+
"flexibleVersions": "3+",
|
|
30
|
+
"fields": [
|
|
31
|
+
{ "name": "TransactionalId", "type": "string", "versions": "0+", "entityType": "transactionalId",
|
|
32
|
+
"about": "The transactional id corresponding to the transaction."},
|
|
33
|
+
{ "name": "ProducerId", "type": "int64", "versions": "0+", "entityType": "producerId",
|
|
34
|
+
"about": "Current producer id in use by the transactional id." },
|
|
35
|
+
{ "name": "ProducerEpoch", "type": "int16", "versions": "0+",
|
|
36
|
+
"about": "Current epoch associated with the producer id." },
|
|
37
|
+
{ "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
|
|
38
|
+
"about": "The unique group identifier." }
|
|
39
|
+
]
|
|
40
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one or more
|
|
2
|
+
// contributor license agreements. See the NOTICE file distributed with
|
|
3
|
+
// this work for additional information regarding copyright ownership.
|
|
4
|
+
// The ASF licenses this file to You under the Apache License, Version 2.0
|
|
5
|
+
// (the "License"); you may not use this file except in compliance with
|
|
6
|
+
// the License. You may obtain a copy of the License at
|
|
7
|
+
//
|
|
8
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
//
|
|
10
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
// See the License for the specific language governing permissions and
|
|
14
|
+
// limitations under the License.
|
|
15
|
+
|
|
16
|
+
{
|
|
17
|
+
"apiKey": 25,
|
|
18
|
+
"type": "response",
|
|
19
|
+
"name": "AddOffsetsToTxnResponse",
|
|
20
|
+
// Starting in version 1, on quota violation brokers send out responses before throttling.
|
|
21
|
+
//
|
|
22
|
+
// Version 2 adds the support for new error code PRODUCER_FENCED.
|
|
23
|
+
//
|
|
24
|
+
// Version 3 enables flexible versions.
|
|
25
|
+
//
|
|
26
|
+
// Version 4 adds support for new error code TRANSACTION_ABORTABLE (KIP-890).
|
|
27
|
+
"validVersions": "0-4",
|
|
28
|
+
"flexibleVersions": "3+",
|
|
29
|
+
"fields": [
|
|
30
|
+
{ "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
|
|
31
|
+
"about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
|
|
32
|
+
{ "name": "ErrorCode", "type": "int16", "versions": "0+",
|
|
33
|
+
"about": "The response error code, or 0 if there was no error." }
|
|
34
|
+
]
|
|
35
|
+
}
|