kafka-python 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (373) hide show
  1. kafka/__init__.py +34 -0
  2. kafka/__main__.py +5 -0
  3. kafka/admin/__init__.py +29 -0
  4. kafka/admin/__main__.py +5 -0
  5. kafka/admin/_acls.py +355 -0
  6. kafka/admin/_cluster.py +359 -0
  7. kafka/admin/_configs.py +479 -0
  8. kafka/admin/_groups.py +754 -0
  9. kafka/admin/_partitions.py +595 -0
  10. kafka/admin/_topics.py +281 -0
  11. kafka/admin/_transactions.py +450 -0
  12. kafka/admin/_users.py +194 -0
  13. kafka/admin/client.py +373 -0
  14. kafka/benchmarks/__init__.py +0 -0
  15. kafka/benchmarks/consumer_performance.py +138 -0
  16. kafka/benchmarks/load_example.py +109 -0
  17. kafka/benchmarks/producer_encode_path.py +201 -0
  18. kafka/benchmarks/producer_performance.py +161 -0
  19. kafka/benchmarks/profile_protocol.py +138 -0
  20. kafka/benchmarks/protocol_old_vs_new.py +447 -0
  21. kafka/benchmarks/record_batch_compose.py +77 -0
  22. kafka/benchmarks/record_batch_read.py +82 -0
  23. kafka/benchmarks/varint_speed.py +426 -0
  24. kafka/cli/__init__.py +36 -0
  25. kafka/cli/admin/__init__.py +117 -0
  26. kafka/cli/admin/acls/__init__.py +9 -0
  27. kafka/cli/admin/acls/common.py +76 -0
  28. kafka/cli/admin/acls/create.py +19 -0
  29. kafka/cli/admin/acls/delete.py +23 -0
  30. kafka/cli/admin/acls/describe.py +16 -0
  31. kafka/cli/admin/cluster/__init__.py +14 -0
  32. kafka/cli/admin/cluster/describe.py +11 -0
  33. kafka/cli/admin/cluster/describe_quorum.py +11 -0
  34. kafka/cli/admin/cluster/features.py +52 -0
  35. kafka/cli/admin/cluster/log_dirs.py +43 -0
  36. kafka/cli/admin/cluster/versions.py +33 -0
  37. kafka/cli/admin/configs/__init__.py +10 -0
  38. kafka/cli/admin/configs/alter.py +43 -0
  39. kafka/cli/admin/configs/common.py +17 -0
  40. kafka/cli/admin/configs/describe.py +30 -0
  41. kafka/cli/admin/configs/list.py +16 -0
  42. kafka/cli/admin/configs/reset.py +20 -0
  43. kafka/cli/admin/groups/__init__.py +16 -0
  44. kafka/cli/admin/groups/alter_offsets.py +30 -0
  45. kafka/cli/admin/groups/delete.py +11 -0
  46. kafka/cli/admin/groups/delete_offsets.py +29 -0
  47. kafka/cli/admin/groups/describe.py +11 -0
  48. kafka/cli/admin/groups/list.py +28 -0
  49. kafka/cli/admin/groups/list_offsets.py +29 -0
  50. kafka/cli/admin/groups/remove_members.py +40 -0
  51. kafka/cli/admin/groups/reset_offsets.py +139 -0
  52. kafka/cli/admin/partitions/__init__.py +21 -0
  53. kafka/cli/admin/partitions/alter_reassignments.py +37 -0
  54. kafka/cli/admin/partitions/create.py +27 -0
  55. kafka/cli/admin/partitions/delete_records.py +31 -0
  56. kafka/cli/admin/partitions/describe.py +36 -0
  57. kafka/cli/admin/partitions/elect_leaders.py +53 -0
  58. kafka/cli/admin/partitions/list_offsets.py +88 -0
  59. kafka/cli/admin/partitions/list_reassignments.py +35 -0
  60. kafka/cli/admin/topics/__init__.py +10 -0
  61. kafka/cli/admin/topics/create.py +13 -0
  62. kafka/cli/admin/topics/delete.py +19 -0
  63. kafka/cli/admin/topics/describe.py +18 -0
  64. kafka/cli/admin/topics/list.py +11 -0
  65. kafka/cli/admin/transactions/__init__.py +17 -0
  66. kafka/cli/admin/transactions/abort.py +38 -0
  67. kafka/cli/admin/transactions/describe.py +24 -0
  68. kafka/cli/admin/transactions/describe_producers.py +29 -0
  69. kafka/cli/admin/transactions/find_hanging.py +26 -0
  70. kafka/cli/admin/transactions/list.py +37 -0
  71. kafka/cli/admin/users/__init__.py +8 -0
  72. kafka/cli/admin/users/alter_user_scram_credentials.py +34 -0
  73. kafka/cli/admin/users/describe_user_scram_credentials.py +15 -0
  74. kafka/cli/common.py +95 -0
  75. kafka/cli/consumer/__init__.py +63 -0
  76. kafka/cli/producer/__init__.py +57 -0
  77. kafka/cluster.py +824 -0
  78. kafka/codec.py +325 -0
  79. kafka/consumer/__init__.py +5 -0
  80. kafka/consumer/__main__.py +5 -0
  81. kafka/consumer/fetcher.py +2012 -0
  82. kafka/consumer/group.py +1347 -0
  83. kafka/consumer/subscription_state.py +897 -0
  84. kafka/coordinator/__init__.py +0 -0
  85. kafka/coordinator/assignors/__init__.py +0 -0
  86. kafka/coordinator/assignors/abstract.py +90 -0
  87. kafka/coordinator/assignors/cooperative_sticky.py +167 -0
  88. kafka/coordinator/assignors/range.py +81 -0
  89. kafka/coordinator/assignors/roundrobin.py +101 -0
  90. kafka/coordinator/assignors/sticky/StickyAssignorUserData.json +37 -0
  91. kafka/coordinator/assignors/sticky/__init__.py +0 -0
  92. kafka/coordinator/assignors/sticky/partition_movements.py +149 -0
  93. kafka/coordinator/assignors/sticky/sorted_set.py +63 -0
  94. kafka/coordinator/assignors/sticky/sticky_assignor.py +665 -0
  95. kafka/coordinator/assignors/sticky/user_data.py +8 -0
  96. kafka/coordinator/base.py +1215 -0
  97. kafka/coordinator/consumer.py +1224 -0
  98. kafka/coordinator/heartbeat.py +82 -0
  99. kafka/coordinator/subscription.py +34 -0
  100. kafka/errors.py +1004 -0
  101. kafka/future.py +166 -0
  102. kafka/metrics/__init__.py +13 -0
  103. kafka/metrics/compound_stat.py +33 -0
  104. kafka/metrics/dict_reporter.py +81 -0
  105. kafka/metrics/kafka_metric.py +36 -0
  106. kafka/metrics/measurable.py +27 -0
  107. kafka/metrics/measurable_stat.py +13 -0
  108. kafka/metrics/metric_config.py +33 -0
  109. kafka/metrics/metric_name.py +105 -0
  110. kafka/metrics/metrics.py +261 -0
  111. kafka/metrics/metrics_reporter.py +53 -0
  112. kafka/metrics/quota.py +41 -0
  113. kafka/metrics/stat.py +19 -0
  114. kafka/metrics/stats/__init__.py +15 -0
  115. kafka/metrics/stats/avg.py +24 -0
  116. kafka/metrics/stats/count.py +17 -0
  117. kafka/metrics/stats/histogram.py +99 -0
  118. kafka/metrics/stats/max_stat.py +17 -0
  119. kafka/metrics/stats/min_stat.py +19 -0
  120. kafka/metrics/stats/percentile.py +14 -0
  121. kafka/metrics/stats/percentiles.py +75 -0
  122. kafka/metrics/stats/rate.py +118 -0
  123. kafka/metrics/stats/sampled_stat.py +99 -0
  124. kafka/metrics/stats/sensor.py +136 -0
  125. kafka/metrics/stats/total.py +15 -0
  126. kafka/net/__init__.py +19 -0
  127. kafka/net/compat.py +165 -0
  128. kafka/net/connection.py +593 -0
  129. kafka/net/http_connect.py +144 -0
  130. kafka/net/inet.py +122 -0
  131. kafka/net/manager.py +451 -0
  132. kafka/net/metrics.py +149 -0
  133. kafka/net/sasl/__init__.py +32 -0
  134. kafka/net/sasl/abc.py +28 -0
  135. kafka/net/sasl/gssapi.py +95 -0
  136. kafka/net/sasl/msk.py +245 -0
  137. kafka/net/sasl/oauth.py +98 -0
  138. kafka/net/sasl/plain.py +42 -0
  139. kafka/net/sasl/scram.py +135 -0
  140. kafka/net/sasl/sspi.py +111 -0
  141. kafka/net/selector.py +644 -0
  142. kafka/net/socks5.py +262 -0
  143. kafka/net/transport.py +415 -0
  144. kafka/net/wakeup_notifier.py +72 -0
  145. kafka/partitioner/__init__.py +8 -0
  146. kafka/partitioner/abc.py +8 -0
  147. kafka/partitioner/default.py +89 -0
  148. kafka/partitioner/sticky.py +109 -0
  149. kafka/producer/__init__.py +5 -0
  150. kafka/producer/__main__.py +5 -0
  151. kafka/producer/future.py +101 -0
  152. kafka/producer/kafka.py +1123 -0
  153. kafka/producer/producer_batch.py +192 -0
  154. kafka/producer/record_accumulator.py +647 -0
  155. kafka/producer/sender.py +884 -0
  156. kafka/producer/transaction_manager.py +1326 -0
  157. kafka/protocol/__init__.py +0 -0
  158. kafka/protocol/admin/__init__.py +29 -0
  159. kafka/protocol/admin/acl.py +83 -0
  160. kafka/protocol/admin/acl.pyi +375 -0
  161. kafka/protocol/admin/client_quotas.py +14 -0
  162. kafka/protocol/admin/client_quotas.pyi +265 -0
  163. kafka/protocol/admin/cluster.py +31 -0
  164. kafka/protocol/admin/cluster.pyi +620 -0
  165. kafka/protocol/admin/configs.py +22 -0
  166. kafka/protocol/admin/configs.pyi +437 -0
  167. kafka/protocol/admin/groups.py +24 -0
  168. kafka/protocol/admin/groups.pyi +261 -0
  169. kafka/protocol/admin/topics.py +53 -0
  170. kafka/protocol/admin/topics.pyi +982 -0
  171. kafka/protocol/admin/transactions.py +18 -0
  172. kafka/protocol/admin/transactions.pyi +311 -0
  173. kafka/protocol/admin/users.py +14 -0
  174. kafka/protocol/admin/users.pyi +223 -0
  175. kafka/protocol/api_data.py +125 -0
  176. kafka/protocol/api_header.py +55 -0
  177. kafka/protocol/api_key.py +97 -0
  178. kafka/protocol/api_message.py +277 -0
  179. kafka/protocol/broker_version_data.py +246 -0
  180. kafka/protocol/consumer/__init__.py +13 -0
  181. kafka/protocol/consumer/fetch.py +16 -0
  182. kafka/protocol/consumer/fetch.pyi +298 -0
  183. kafka/protocol/consumer/group.py +38 -0
  184. kafka/protocol/consumer/group.pyi +824 -0
  185. kafka/protocol/consumer/metadata.py +30 -0
  186. kafka/protocol/consumer/metadata.pyi +89 -0
  187. kafka/protocol/consumer/offsets.py +75 -0
  188. kafka/protocol/consumer/offsets.pyi +288 -0
  189. kafka/protocol/data_container.py +166 -0
  190. kafka/protocol/frame.py +30 -0
  191. kafka/protocol/generate_stubs.py +468 -0
  192. kafka/protocol/metadata/__init__.py +10 -0
  193. kafka/protocol/metadata/api_versions.py +41 -0
  194. kafka/protocol/metadata/api_versions.pyi +128 -0
  195. kafka/protocol/metadata/find_coordinator.py +19 -0
  196. kafka/protocol/metadata/find_coordinator.pyi +105 -0
  197. kafka/protocol/metadata/metadata.py +34 -0
  198. kafka/protocol/metadata/metadata.pyi +160 -0
  199. kafka/protocol/old/__init__.py +0 -0
  200. kafka/protocol/old/abstract.py +17 -0
  201. kafka/protocol/old/add_offsets_to_txn.py +54 -0
  202. kafka/protocol/old/add_partitions_to_txn.py +71 -0
  203. kafka/protocol/old/admin.py +1086 -0
  204. kafka/protocol/old/api.py +205 -0
  205. kafka/protocol/old/api_versions.py +133 -0
  206. kafka/protocol/old/commit.py +355 -0
  207. kafka/protocol/old/consumer_protocol.py +36 -0
  208. kafka/protocol/old/end_txn.py +53 -0
  209. kafka/protocol/old/fetch.py +408 -0
  210. kafka/protocol/old/find_coordinator.py +72 -0
  211. kafka/protocol/old/group.py +451 -0
  212. kafka/protocol/old/init_producer_id.py +42 -0
  213. kafka/protocol/old/list_offsets.py +186 -0
  214. kafka/protocol/old/metadata.py +290 -0
  215. kafka/protocol/old/offset_for_leader_epoch.py +133 -0
  216. kafka/protocol/old/produce.py +247 -0
  217. kafka/protocol/old/sasl_authenticate.py +38 -0
  218. kafka/protocol/old/sasl_handshake.py +39 -0
  219. kafka/protocol/old/struct.py +87 -0
  220. kafka/protocol/old/txn_offset_commit.py +73 -0
  221. kafka/protocol/old/types.py +440 -0
  222. kafka/protocol/parser.py +191 -0
  223. kafka/protocol/producer/__init__.py +7 -0
  224. kafka/protocol/producer/produce.py +17 -0
  225. kafka/protocol/producer/produce.pyi +197 -0
  226. kafka/protocol/producer/transaction.py +30 -0
  227. kafka/protocol/producer/transaction.pyi +663 -0
  228. kafka/protocol/sasl.py +52 -0
  229. kafka/protocol/sasl.pyi +126 -0
  230. kafka/protocol/schemas/__init__.py +7 -0
  231. kafka/protocol/schemas/fields/__init__.py +7 -0
  232. kafka/protocol/schemas/fields/array.py +127 -0
  233. kafka/protocol/schemas/fields/base.py +156 -0
  234. kafka/protocol/schemas/fields/codecs/__init__.py +12 -0
  235. kafka/protocol/schemas/fields/codecs/encode_buffer.py +82 -0
  236. kafka/protocol/schemas/fields/codecs/tagged_fields.py +109 -0
  237. kafka/protocol/schemas/fields/codecs/types.py +505 -0
  238. kafka/protocol/schemas/fields/codegen.py +40 -0
  239. kafka/protocol/schemas/fields/simple.py +127 -0
  240. kafka/protocol/schemas/fields/struct.py +357 -0
  241. kafka/protocol/schemas/fields/struct_array.py +142 -0
  242. kafka/protocol/schemas/load_json.py +42 -0
  243. kafka/protocol/schemas/resources/AddOffsetsToTxnRequest.json +40 -0
  244. kafka/protocol/schemas/resources/AddOffsetsToTxnResponse.json +35 -0
  245. kafka/protocol/schemas/resources/AddPartitionsToTxnRequest.json +65 -0
  246. kafka/protocol/schemas/resources/AddPartitionsToTxnResponse.json +60 -0
  247. kafka/protocol/schemas/resources/AlterClientQuotasRequest.json +47 -0
  248. kafka/protocol/schemas/resources/AlterClientQuotasResponse.json +41 -0
  249. kafka/protocol/schemas/resources/AlterConfigsRequest.json +43 -0
  250. kafka/protocol/schemas/resources/AlterConfigsResponse.json +39 -0
  251. kafka/protocol/schemas/resources/AlterPartitionReassignmentsRequest.json +42 -0
  252. kafka/protocol/schemas/resources/AlterPartitionReassignmentsResponse.json +47 -0
  253. kafka/protocol/schemas/resources/AlterReplicaLogDirsRequest.json +41 -0
  254. kafka/protocol/schemas/resources/AlterReplicaLogDirsResponse.json +41 -0
  255. kafka/protocol/schemas/resources/AlterUserScramCredentialsRequest.json +45 -0
  256. kafka/protocol/schemas/resources/AlterUserScramCredentialsResponse.json +35 -0
  257. kafka/protocol/schemas/resources/ApiVersionsRequest.json +34 -0
  258. kafka/protocol/schemas/resources/ApiVersionsResponse.json +79 -0
  259. kafka/protocol/schemas/resources/ConsumerProtocolAssignment.json +42 -0
  260. kafka/protocol/schemas/resources/ConsumerProtocolSubscription.json +49 -0
  261. kafka/protocol/schemas/resources/CreateAclsRequest.json +46 -0
  262. kafka/protocol/schemas/resources/CreateAclsResponse.json +37 -0
  263. kafka/protocol/schemas/resources/CreatePartitionsRequest.json +47 -0
  264. kafka/protocol/schemas/resources/CreatePartitionsResponse.json +41 -0
  265. kafka/protocol/schemas/resources/CreateTopicsRequest.json +65 -0
  266. kafka/protocol/schemas/resources/CreateTopicsResponse.json +72 -0
  267. kafka/protocol/schemas/resources/DeleteAclsRequest.json +46 -0
  268. kafka/protocol/schemas/resources/DeleteAclsResponse.json +59 -0
  269. kafka/protocol/schemas/resources/DeleteGroupsRequest.json +30 -0
  270. kafka/protocol/schemas/resources/DeleteGroupsResponse.json +36 -0
  271. kafka/protocol/schemas/resources/DeleteRecordsRequest.json +42 -0
  272. kafka/protocol/schemas/resources/DeleteRecordsResponse.json +43 -0
  273. kafka/protocol/schemas/resources/DeleteTopicsRequest.json +43 -0
  274. kafka/protocol/schemas/resources/DeleteTopicsResponse.json +52 -0
  275. kafka/protocol/schemas/resources/DescribeAclsRequest.json +43 -0
  276. kafka/protocol/schemas/resources/DescribeAclsResponse.json +55 -0
  277. kafka/protocol/schemas/resources/DescribeClientQuotasRequest.json +37 -0
  278. kafka/protocol/schemas/resources/DescribeClientQuotasResponse.json +47 -0
  279. kafka/protocol/schemas/resources/DescribeClusterRequest.json +35 -0
  280. kafka/protocol/schemas/resources/DescribeClusterResponse.json +56 -0
  281. kafka/protocol/schemas/resources/DescribeConfigsRequest.json +42 -0
  282. kafka/protocol/schemas/resources/DescribeConfigsResponse.json +69 -0
  283. kafka/protocol/schemas/resources/DescribeGroupsRequest.json +38 -0
  284. kafka/protocol/schemas/resources/DescribeGroupsResponse.json +74 -0
  285. kafka/protocol/schemas/resources/DescribeLogDirsRequest.json +38 -0
  286. kafka/protocol/schemas/resources/DescribeLogDirsResponse.json +65 -0
  287. kafka/protocol/schemas/resources/DescribeProducersRequest.json +32 -0
  288. kafka/protocol/schemas/resources/DescribeProducersResponse.json +55 -0
  289. kafka/protocol/schemas/resources/DescribeQuorumRequest.json +39 -0
  290. kafka/protocol/schemas/resources/DescribeQuorumResponse.json +82 -0
  291. kafka/protocol/schemas/resources/DescribeTopicPartitionsRequest.json +40 -0
  292. kafka/protocol/schemas/resources/DescribeTopicPartitionsResponse.json +66 -0
  293. kafka/protocol/schemas/resources/DescribeTransactionsRequest.json +27 -0
  294. kafka/protocol/schemas/resources/DescribeTransactionsResponse.json +52 -0
  295. kafka/protocol/schemas/resources/DescribeUserScramCredentialsRequest.json +30 -0
  296. kafka/protocol/schemas/resources/DescribeUserScramCredentialsResponse.json +45 -0
  297. kafka/protocol/schemas/resources/ElectLeadersRequest.json +41 -0
  298. kafka/protocol/schemas/resources/ElectLeadersResponse.json +45 -0
  299. kafka/protocol/schemas/resources/EndTxnRequest.json +43 -0
  300. kafka/protocol/schemas/resources/EndTxnResponse.json +41 -0
  301. kafka/protocol/schemas/resources/FetchRequest.json +125 -0
  302. kafka/protocol/schemas/resources/FetchResponse.json +124 -0
  303. kafka/protocol/schemas/resources/FindCoordinatorRequest.json +43 -0
  304. kafka/protocol/schemas/resources/FindCoordinatorResponse.json +58 -0
  305. kafka/protocol/schemas/resources/HeartbeatRequest.json +39 -0
  306. kafka/protocol/schemas/resources/HeartbeatResponse.json +35 -0
  307. kafka/protocol/schemas/resources/IncrementalAlterConfigsRequest.json +44 -0
  308. kafka/protocol/schemas/resources/IncrementalAlterConfigsResponse.json +38 -0
  309. kafka/protocol/schemas/resources/InitProducerIdRequest.json +50 -0
  310. kafka/protocol/schemas/resources/InitProducerIdResponse.json +47 -0
  311. kafka/protocol/schemas/resources/JoinGroupRequest.json +63 -0
  312. kafka/protocol/schemas/resources/JoinGroupResponse.json +69 -0
  313. kafka/protocol/schemas/resources/LeaveGroupRequest.json +47 -0
  314. kafka/protocol/schemas/resources/LeaveGroupResponse.json +47 -0
  315. kafka/protocol/schemas/resources/ListConfigResourcesRequest.json +31 -0
  316. kafka/protocol/schemas/resources/ListConfigResourcesResponse.json +37 -0
  317. kafka/protocol/schemas/resources/ListGroupsRequest.json +36 -0
  318. kafka/protocol/schemas/resources/ListGroupsResponse.json +49 -0
  319. kafka/protocol/schemas/resources/ListOffsetsRequest.json +72 -0
  320. kafka/protocol/schemas/resources/ListOffsetsResponse.json +71 -0
  321. kafka/protocol/schemas/resources/ListPartitionReassignmentsRequest.json +34 -0
  322. kafka/protocol/schemas/resources/ListPartitionReassignmentsResponse.json +46 -0
  323. kafka/protocol/schemas/resources/ListTransactionsRequest.json +40 -0
  324. kafka/protocol/schemas/resources/ListTransactionsResponse.json +42 -0
  325. kafka/protocol/schemas/resources/MetadataRequest.json +56 -0
  326. kafka/protocol/schemas/resources/MetadataResponse.json +101 -0
  327. kafka/protocol/schemas/resources/OffsetCommitRequest.json +76 -0
  328. kafka/protocol/schemas/resources/OffsetCommitResponse.json +71 -0
  329. kafka/protocol/schemas/resources/OffsetDeleteRequest.json +39 -0
  330. kafka/protocol/schemas/resources/OffsetDeleteResponse.json +42 -0
  331. kafka/protocol/schemas/resources/OffsetFetchRequest.json +76 -0
  332. kafka/protocol/schemas/resources/OffsetFetchResponse.json +107 -0
  333. kafka/protocol/schemas/resources/OffsetForLeaderEpochRequest.json +52 -0
  334. kafka/protocol/schemas/resources/OffsetForLeaderEpochResponse.json +51 -0
  335. kafka/protocol/schemas/resources/ProduceRequest.json +73 -0
  336. kafka/protocol/schemas/resources/ProduceResponse.json +96 -0
  337. kafka/protocol/schemas/resources/RequestHeader.json +44 -0
  338. kafka/protocol/schemas/resources/ResponseHeader.json +26 -0
  339. kafka/protocol/schemas/resources/SaslAuthenticateRequest.json +29 -0
  340. kafka/protocol/schemas/resources/SaslAuthenticateResponse.json +34 -0
  341. kafka/protocol/schemas/resources/SaslHandshakeRequest.json +31 -0
  342. kafka/protocol/schemas/resources/SaslHandshakeResponse.json +32 -0
  343. kafka/protocol/schemas/resources/SyncGroupRequest.json +56 -0
  344. kafka/protocol/schemas/resources/SyncGroupResponse.json +46 -0
  345. kafka/protocol/schemas/resources/TxnOffsetCommitRequest.json +68 -0
  346. kafka/protocol/schemas/resources/TxnOffsetCommitResponse.json +47 -0
  347. kafka/protocol/schemas/resources/UpdateFeaturesRequest.json +43 -0
  348. kafka/protocol/schemas/resources/UpdateFeaturesResponse.json +39 -0
  349. kafka/protocol/schemas/resources/WriteTxnMarkersRequest.json +49 -0
  350. kafka/protocol/schemas/resources/WriteTxnMarkersResponse.json +45 -0
  351. kafka/protocol/schemas/resources/__init__.py +0 -0
  352. kafka/record/__init__.py +3 -0
  353. kafka/record/_crc32c.py +161 -0
  354. kafka/record/abc.py +144 -0
  355. kafka/record/default_records.py +782 -0
  356. kafka/record/legacy_records.py +587 -0
  357. kafka/record/memory_records.py +255 -0
  358. kafka/record/util.py +135 -0
  359. kafka/serializer/__init__.py +4 -0
  360. kafka/serializer/abstract.py +20 -0
  361. kafka/serializer/default.py +16 -0
  362. kafka/serializer/json.py +17 -0
  363. kafka/serializer/wrapper.py +21 -0
  364. kafka/structs.py +69 -0
  365. kafka/util.py +159 -0
  366. kafka/vendor/__init__.py +0 -0
  367. kafka/version.py +1 -0
  368. kafka_python-3.0.0.dist-info/METADATA +319 -0
  369. kafka_python-3.0.0.dist-info/RECORD +373 -0
  370. kafka_python-3.0.0.dist-info/WHEEL +5 -0
  371. kafka_python-3.0.0.dist-info/entry_points.txt +2 -0
  372. kafka_python-3.0.0.dist-info/licenses/LICENSE +202 -0
  373. kafka_python-3.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,505 @@
1
+ from struct import error, pack, pack_into, unpack, unpack_from
2
+ import uuid
3
+
4
+ from .encode_buffer import EncodeBuffer
5
+
6
+
7
+ class FixedCodec:
8
+ """Base class for fixed-size codecs. Subclasses define fmt and size.
9
+
10
+ The Kafka protocol uses big-endian ('>') byte order for all fixed-size
11
+ types. This prefix is applied here so subclasses only specify the type
12
+ format character (e.g., 'i' for 32-bit signed int).
13
+ """
14
+ fmt = None # e.g., 'i' - set by subclass
15
+ size = None # e.g., 4 - set by subclass
16
+ batchable = True # Can be batched with adjacent FixedCodec fields
17
+
18
+ def __init_subclass__(cls, **kw):
19
+ super().__init_subclass__(**kw)
20
+ if cls.fmt is not None:
21
+ cls._be_fmt = '>' + cls.fmt
22
+
23
+ @classmethod
24
+ def encode(cls, value, compact=False):
25
+ return pack(cls._be_fmt, value)
26
+
27
+ @classmethod
28
+ def encode_into(cls, out, value, compact=False):
29
+ pack_into(cls._be_fmt, out.buf, out.pos, value)
30
+ out.pos += cls.size
31
+
32
+ @classmethod
33
+ def decode(cls, data, compact=False):
34
+ return unpack(cls._be_fmt, data.read(cls.size))[0]
35
+
36
+ @classmethod
37
+ def decode_from(cls, data, pos):
38
+ """Decode from a buffer at pos. Returns (value, new_pos)."""
39
+ return unpack_from(cls._be_fmt, data, pos)[0], pos + cls.size
40
+
41
+ @classmethod
42
+ def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
43
+ ctx.emit(indent, "pack_into('%s', buf, pos, %s)" % (cls._be_fmt, val_expr))
44
+ ctx.emit(indent, 'pos += %d' % cls.size)
45
+
46
+ @classmethod
47
+ def emit_decode_from(cls, ctx, var_name, indent, compact=False):
48
+ """Emit decode for a single fixed field (unbatched fallback)."""
49
+ ctx.emit(indent, '%s = unpack_from("%s", data, pos)[0]' % (var_name, cls._be_fmt))
50
+ ctx.emit(indent, 'pos += %d' % cls.size)
51
+
52
+
53
+ class Int8(FixedCodec):
54
+ fmt = 'b'
55
+ size = 1
56
+
57
+
58
+ class Int16(FixedCodec):
59
+ fmt = 'h'
60
+ size = 2
61
+
62
+
63
+ class UnsignedInt16(FixedCodec):
64
+ fmt = 'H'
65
+ size = 2
66
+
67
+
68
+ class Int32(FixedCodec):
69
+ fmt = 'i'
70
+ size = 4
71
+
72
+
73
+ class Int64(FixedCodec):
74
+ fmt = 'q'
75
+ size = 8
76
+
77
+
78
+ class Float64(FixedCodec):
79
+ fmt = 'd'
80
+ size = 8
81
+
82
+
83
+ class UUID:
84
+ fmt = '16B'
85
+ size = 16
86
+ ZERO_UUID = uuid.UUID(int=0)
87
+
88
+ @classmethod
89
+ def encode(cls, value, compact=False):
90
+ if value is None:
91
+ value = cls.ZERO_UUID
92
+ if isinstance(value, uuid.UUID):
93
+ return value.bytes
94
+ return uuid.UUID(value).bytes
95
+
96
+ @classmethod
97
+ def encode_into(cls, out, value, compact=False):
98
+ if value is None:
99
+ value = cls.ZERO_UUID
100
+ if isinstance(value, uuid.UUID):
101
+ b = value.bytes
102
+ else:
103
+ b = uuid.UUID(value).bytes
104
+ pos = out.pos
105
+ out.buf[pos:pos+16] = b
106
+ out.pos = pos + 16
107
+
108
+ @classmethod
109
+ def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
110
+ ctx.globs['_ZERO_UUID'] = cls.ZERO_UUID
111
+ v = ctx.next_var('uv')
112
+ ctx.emit(indent, '%s = %s' % (v, val_expr))
113
+ ctx.emit(indent, 'if %s is None: %s = _ZERO_UUID' % (v, v))
114
+ ctx.emit(indent, 'buf[pos:pos+16] = %s.bytes if hasattr(%s, "bytes") else __import__("uuid").UUID(%s).bytes' % (v, v, v))
115
+ ctx.emit(indent, 'pos += 16')
116
+
117
+ @classmethod
118
+ def emit_decode_from(cls, ctx, var_name, indent, compact=False):
119
+ ctx.globs['_ZERO_UUID'] = cls.ZERO_UUID
120
+ ctx.globs['_uuid_cls'] = uuid.UUID
121
+ ctx.emit(indent, '%s = _uuid_cls(bytes=bytes(data[pos:pos+16]))' % var_name)
122
+ ctx.emit(indent, 'if %s == _ZERO_UUID: %s = None' % (var_name, var_name))
123
+ ctx.emit(indent, 'pos += 16')
124
+
125
+ @classmethod
126
+ def decode(cls, data, compact=False):
127
+ val = uuid.UUID(bytes=data.read(16))
128
+ if val == cls.ZERO_UUID:
129
+ return None
130
+ return val
131
+
132
+
133
+ class String:
134
+ fmt = None # 'B' for compact, 'h' for standard
135
+ size = 'variable'
136
+
137
+ def __init__(self, encoding='utf-8'):
138
+ self.encoding = encoding
139
+
140
+ def encode(self, value, compact=False):
141
+ if compact:
142
+ if value is None:
143
+ return UnsignedVarInt32.encode(0)
144
+ value = str(value).encode(self.encoding)
145
+ return UnsignedVarInt32.encode(len(value) + 1) + value
146
+ if value is None:
147
+ return Int16.encode(-1)
148
+ value = str(value).encode(self.encoding)
149
+ return Int16.encode(len(value)) + value
150
+
151
+ def encode_into(self, out, value, compact=False):
152
+ if compact:
153
+ if value is None:
154
+ UnsignedVarInt32.encode_into(out,0)
155
+ return
156
+ value = str(value).encode(self.encoding)
157
+ UnsignedVarInt32.encode_into(out,len(value) + 1)
158
+ else:
159
+ if value is None:
160
+ pack_into('>h', out.buf, out.pos, -1)
161
+ out.pos += 2
162
+ return
163
+ value = str(value).encode(self.encoding)
164
+ pack_into('>h', out.buf, out.pos, len(value))
165
+ out.pos += 2
166
+ n = len(value)
167
+ pos = out.pos
168
+ out.buf[pos:pos+n] = value
169
+ out.pos = pos + n
170
+
171
+ def emit_encode_into(self, ctx, val_expr, indent, compact=False):
172
+ sv = ctx.next_var('sv')
173
+ ctx.emit(indent, 'if %s is None:' % val_expr)
174
+ if compact:
175
+ ctx.emit(indent, ' buf[pos] = 0')
176
+ ctx.emit(indent, ' pos += 1')
177
+ ctx.emit(indent, 'else:')
178
+ sn = ctx.next_var('sn')
179
+ ctx.emit(indent, ' %s = str(%s).encode("utf-8")' % (sv, val_expr))
180
+ ctx.emit(indent, ' %s = len(%s) + 1' % (sn, sv))
181
+ UnsignedVarInt32.emit_encode_into(ctx, sn, indent + ' ')
182
+ ctx.emit(indent, ' buf[pos:pos+len(%s)] = %s' % (sv, sv))
183
+ ctx.emit(indent, ' pos += len(%s)' % sv)
184
+ else:
185
+ ctx.emit(indent, " pack_into('>h', buf, pos, -1)")
186
+ ctx.emit(indent, ' pos += 2')
187
+ ctx.emit(indent, 'else:')
188
+ ctx.emit(indent, ' %s = str(%s).encode("utf-8")' % (sv, val_expr))
189
+ ctx.emit(indent, " pack_into('>h', buf, pos, len(%s))" % sv)
190
+ ctx.emit(indent, ' pos += 2')
191
+ ctx.emit(indent, ' buf[pos:pos+len(%s)] = %s' % (sv, sv))
192
+ ctx.emit(indent, ' pos += len(%s)' % sv)
193
+
194
+ def emit_decode_from(self, ctx, var_name, indent, compact=False):
195
+ ln = ctx.next_var('ln')
196
+ if compact:
197
+ UnsignedVarInt32.emit_decode_from(ctx, ln, indent)
198
+ ctx.emit(indent, '%s -= 1' % ln)
199
+ else:
200
+ ctx.emit(indent, '%s = unpack_from(">h", data, pos)[0]' % ln)
201
+ ctx.emit(indent, 'pos += 2')
202
+ ctx.emit(indent, 'if %s < 0:' % ln)
203
+ ctx.emit(indent, ' %s = None' % var_name)
204
+ ctx.emit(indent, 'else:')
205
+ ctx.emit(indent, ' %s = str(data[pos:pos+%s], "utf-8")' % (var_name, ln))
206
+ ctx.emit(indent, ' pos += %s' % ln)
207
+
208
+ def decode(self, data, compact=False):
209
+ if compact:
210
+ length = UnsignedVarInt32.decode(data) - 1
211
+ else:
212
+ length = Int16.decode(data)
213
+ if length < 0:
214
+ return None
215
+ value = data.read(length)
216
+ if len(value) != length:
217
+ raise ValueError('Buffer underrun decoding string')
218
+ return value.decode(self.encoding)
219
+
220
+
221
+ class Bytes:
222
+ fmt = Int32.fmt
223
+ size = 'variable'
224
+
225
+ @classmethod
226
+ def encode(cls, value, compact=False):
227
+ if value is not None and not isinstance(value, (bytes, bytearray, memoryview)):
228
+ value = value.encode()
229
+ if compact:
230
+ if value is None:
231
+ return UnsignedVarInt32.encode(0)
232
+ return UnsignedVarInt32.encode(len(value) + 1) + bytes(value)
233
+ if value is None:
234
+ return Int32.encode(-1)
235
+ return Int32.encode(len(value)) + bytes(value)
236
+
237
+ @classmethod
238
+ def encode_into(cls, out, value, compact=False):
239
+ if value is not None and not isinstance(value, (bytes, bytearray, memoryview)):
240
+ value = value.encode()
241
+ if compact:
242
+ if value is None:
243
+ UnsignedVarInt32.encode_into(out, 0)
244
+ return
245
+ UnsignedVarInt32.encode_into(out, len(value) + 1)
246
+ else:
247
+ if value is None:
248
+ pack_into('>i', out.buf, out.pos, -1)
249
+ out.pos += 4
250
+ return
251
+ pack_into('>i', out.buf, out.pos, len(value))
252
+ out.pos += 4
253
+ n = len(value)
254
+ out.ensure(n)
255
+ pos = out.pos
256
+ out.buf[pos:pos+n] = value
257
+ out.pos = pos + n
258
+
259
+ @classmethod
260
+ def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
261
+ bv = ctx.next_var('bv')
262
+ bn = ctx.next_var('bn')
263
+ if compact:
264
+ ctx.emit(indent, '%s = %s' % (bv, val_expr))
265
+ ctx.emit(indent, 'if %s is not None and not isinstance(%s, (bytes, bytearray, memoryview)): %s = %s.encode()' % (bv, bv, bv, bv))
266
+ ctx.emit(indent, 'if %s is None:' % bv)
267
+ ctx.emit(indent, ' buf[pos] = 0')
268
+ ctx.emit(indent, ' pos += 1')
269
+ ctx.emit(indent, 'else:')
270
+ ctx.emit(indent, ' %s = len(%s)' % (bn, bv))
271
+ sn = ctx.next_var('sn')
272
+ ctx.emit(indent, ' %s = %s + 1' % (sn, bn))
273
+ UnsignedVarInt32.emit_encode_into(ctx, sn, indent + ' ')
274
+ ctx.emit(indent, ' out.pos = pos')
275
+ ctx.emit(indent, ' out.ensure(%s)' % bn)
276
+ ctx.emit(indent, ' buf = out.buf')
277
+ ctx.emit(indent, ' buf[pos:pos+%s] = %s' % (bn, bv))
278
+ ctx.emit(indent, ' pos += %s' % bn)
279
+ else:
280
+ ctx.emit(indent, '%s = %s' % (bv, val_expr))
281
+ ctx.emit(indent, 'if %s is not None and not isinstance(%s, (bytes, bytearray, memoryview)): %s = %s.encode()' % (bv, bv, bv, bv))
282
+ ctx.emit(indent, 'if %s is None:' % bv)
283
+ ctx.emit(indent, " pack_into('>i', buf, pos, -1)")
284
+ ctx.emit(indent, ' pos += 4')
285
+ ctx.emit(indent, 'else:')
286
+ ctx.emit(indent, ' %s = len(%s)' % (bn, bv))
287
+ ctx.emit(indent, " pack_into('>i', buf, pos, %s)" % bn)
288
+ ctx.emit(indent, ' pos += 4')
289
+ ctx.emit(indent, ' out.pos = pos')
290
+ ctx.emit(indent, ' out.ensure(%s)' % bn)
291
+ ctx.emit(indent, ' buf = out.buf')
292
+ ctx.emit(indent, ' buf[pos:pos+%s] = %s' % (bn, bv))
293
+ ctx.emit(indent, ' pos += %s' % bn)
294
+
295
+ @classmethod
296
+ def emit_decode_from(cls, ctx, var_name, indent, compact=False):
297
+ ln = ctx.next_var('ln')
298
+ if compact:
299
+ UnsignedVarInt32.emit_decode_from(ctx, ln, indent)
300
+ ctx.emit(indent, '%s -= 1' % ln)
301
+ else:
302
+ ctx.emit(indent, '%s = unpack_from(">i", data, pos)[0]' % ln)
303
+ ctx.emit(indent, 'pos += 4')
304
+ ctx.emit(indent, 'if %s < 0:' % ln)
305
+ ctx.emit(indent, ' %s = None' % var_name)
306
+ ctx.emit(indent, 'else:')
307
+ ctx.emit(indent, ' %s = bytes(data[pos:pos+%s])' % (var_name, ln))
308
+ ctx.emit(indent, ' pos += %s' % ln)
309
+
310
+ @classmethod
311
+ def decode(cls, data, compact=False):
312
+ if compact:
313
+ length = UnsignedVarInt32.decode(data) - 1
314
+ else:
315
+ length = Int32.decode(data)
316
+ if length < 0:
317
+ return None
318
+ value = data.read(length)
319
+ if len(value) != length:
320
+ raise ValueError('Buffer underrun decoding Bytes')
321
+ return value
322
+
323
+
324
+ class Boolean(FixedCodec):
325
+ fmt = '?'
326
+ size = 1
327
+
328
+
329
+ class UnsignedVarInt32:
330
+ fmt = 'B'
331
+ size = 'variable'
332
+
333
+ @classmethod
334
+ def decode(cls, data, compact=False):
335
+ value = VarInt32.decode(data)
336
+ return (value << 1) ^ (value >> 31)
337
+
338
+ @classmethod
339
+ def encode(cls, value, compact=False):
340
+ return VarInt32.encode((value >> 1) ^ -(value & 1))
341
+
342
+ @classmethod
343
+ def encode_into(cls, out, value):
344
+ buf = out.buf
345
+ pos = out.pos
346
+ while (value & 0xffffff80) != 0:
347
+ buf[pos] = (value & 0x7f) | 0x80
348
+ value >>= 7
349
+ pos += 1
350
+ buf[pos] = value
351
+ out.pos = pos + 1
352
+
353
+ @classmethod
354
+ def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
355
+ ctx.emit(indent, 'while (%s & 0xffffff80) != 0:' % val_expr)
356
+ ctx.emit(indent, ' buf[pos] = (%s & 0x7f) | 0x80' % val_expr)
357
+ ctx.emit(indent, ' %s >>= 7' % val_expr)
358
+ ctx.emit(indent, ' pos += 1')
359
+ ctx.emit(indent, 'buf[pos] = %s' % val_expr)
360
+ ctx.emit(indent, 'pos += 1')
361
+
362
+ @classmethod
363
+ def emit_decode_from(cls, ctx, var_name, indent):
364
+ """Emit inline unsigned varint decode. Result in var_name, pos advanced."""
365
+ b = ctx.next_var('b')
366
+ shift = ctx.next_var('sh')
367
+ ctx.emit(indent, '%s = 0' % var_name)
368
+ ctx.emit(indent, '%s = 0' % shift)
369
+ ctx.emit(indent, 'while True:')
370
+ ctx.emit(indent, ' %s = data[pos]' % b)
371
+ ctx.emit(indent, ' pos += 1')
372
+ ctx.emit(indent, ' %s |= (%s & 0x7f) << %s' % (var_name, b, shift))
373
+ ctx.emit(indent, ' if not (%s & 0x80): break' % b)
374
+ ctx.emit(indent, ' %s += 7' % shift)
375
+
376
+
377
+ class VarInt32:
378
+ fmt = 'B'
379
+ size = 'variable'
380
+
381
+ @classmethod
382
+ def decode(cls, data, compact=False):
383
+ value, i = 0, 0
384
+ while True:
385
+ b, = unpack('B', data.read(1))
386
+ if not (b & 0x80):
387
+ break
388
+ value |= (b & 0x7f) << i
389
+ i += 7
390
+ if i > 28:
391
+ raise ValueError('Invalid value {}'.format(value))
392
+ value |= b << i
393
+ return (value >> 1) ^ -(value & 1)
394
+
395
+ @classmethod
396
+ def encode(cls, value, compact=False):
397
+ # bring it in line with the java binary repr
398
+ value = (value << 1) ^ (value >> 31)
399
+ value &= 0xffffffff
400
+ ret = b''
401
+ while (value & 0xffffff80) != 0:
402
+ b = (value & 0x7f) | 0x80
403
+ ret += pack('B', b)
404
+ value >>= 7
405
+ ret += pack('B', value)
406
+ return ret
407
+
408
+
409
+ class VarInt64:
410
+ fmt = 'B'
411
+ size = 'variable'
412
+
413
+ @classmethod
414
+ def decode(cls, data, compact=False):
415
+ value, i = 0, 0
416
+ while True:
417
+ b, = unpack('B', data.read(1))
418
+ if not (b & 0x80):
419
+ break
420
+ value |= (b & 0x7f) << i
421
+ i += 7
422
+ if i > 63:
423
+ raise ValueError('Invalid value {}'.format(value))
424
+ value |= b << i
425
+ return (value >> 1) ^ -(value & 1)
426
+
427
+ @classmethod
428
+ def encode(cls, value, compact=False):
429
+ # bring it in line with the java binary repr
430
+ value = (value << 1) ^ (value >> 63)
431
+ value &= 0xffffffffffffffff
432
+ ret = b''
433
+ while (value & 0xffffffffffffff80) != 0:
434
+ b = (value & 0x7f) | 0x80
435
+ ret += pack('B', b)
436
+ value >>= 7
437
+ ret += pack('B', value)
438
+ return ret
439
+
440
+
441
+ class BitField:
442
+ fmt = 'I'
443
+ size = 4
444
+
445
+ @classmethod
446
+ def decode(cls, data, compact=False):
447
+ vals = cls.from_32_bit_field(unpack('>I', data.read(4))[0])
448
+ if vals == {31}:
449
+ vals = None
450
+ return vals
451
+
452
+ @classmethod
453
+ def encode(cls, vals, compact=False):
454
+ if vals is None:
455
+ vals = {31}
456
+ # to_32_bit_field returns unsigned val, so we need to
457
+ # encode >I to avoid crash if/when byte 31 is set
458
+ # (note that decode as signed still works fine)
459
+ return pack('>I', cls.to_32_bit_field(vals))
460
+
461
+ @classmethod
462
+ def encode_into(cls, out, vals, compact=False):
463
+ if vals is None:
464
+ vals = {31}
465
+ pack_into('>I', out.buf, out.pos, cls.to_32_bit_field(vals))
466
+ out.pos += 4
467
+
468
+ @classmethod
469
+ def emit_encode_into(cls, ctx, val_expr, indent, compact=False):
470
+ bf = ctx.next_var('bf')
471
+ bfi = ctx.next_var('bfi')
472
+ ctx.emit(indent, '%s = %s' % (bf, val_expr))
473
+ ctx.emit(indent, 'if %s is None: %s = {31}' % (bf, bf))
474
+ ctx.emit(indent, '%s = 0' % bfi)
475
+ ctx.emit(indent, 'for _b in %s: %s |= 1 << _b' % (bf, bfi))
476
+ ctx.emit(indent, "pack_into('>I', buf, pos, %s)" % bfi)
477
+ ctx.emit(indent, 'pos += 4')
478
+
479
+ @classmethod
480
+ def emit_decode_from(cls, ctx, var_name, indent, compact=False):
481
+ ctx.globs['_bitfield_from_32'] = cls.from_32_bit_field
482
+ raw = ctx.next_var('bfr')
483
+ ctx.emit(indent, '%s = unpack_from(">I", data, pos)[0]' % raw)
484
+ ctx.emit(indent, 'pos += 4')
485
+ ctx.emit(indent, '%s = _bitfield_from_32(%s)' % (var_name, raw))
486
+ ctx.emit(indent, 'if %s == {31}: %s = None' % (var_name, var_name))
487
+
488
+ @classmethod
489
+ def to_32_bit_field(cls, vals):
490
+ value = 0
491
+ for b in vals:
492
+ assert 0 <= b < 32
493
+ value |= 1 << b
494
+ return value
495
+
496
+ @classmethod
497
+ def from_32_bit_field(cls, value):
498
+ result = set()
499
+ count = 0
500
+ while value != 0:
501
+ if (value & 1) != 0:
502
+ result.add(count)
503
+ count += 1
504
+ value = (value & 0xFFFFFFFF) >> 1
505
+ return result
@@ -0,0 +1,40 @@
1
+ """Generate flat encode/decode functions for a StructField + version.
2
+
3
+ Given a StructField and a protocol version, generates Python functions
4
+ that encode/decode directly with zero dispatch overhead - no intermediate
5
+ SimpleField/ArrayField/StructField method calls.
6
+
7
+ Usage:
8
+ from kafka.protocol.schemas.fields.codegen import CodegenContext
9
+ # Encode: see StructField.compiled_encode_into()
10
+ # Decode: see StructField.compiled_decode_from()
11
+ """
12
+
13
+ from struct import pack_into, unpack_from
14
+
15
+
16
+ class CodegenContext:
17
+ """Shared state for code generation."""
18
+
19
+ def __init__(self):
20
+ self.lines = []
21
+ self.globs = {'pack_into': pack_into, 'unpack_from': unpack_from}
22
+ self._var_counter = 0
23
+
24
+ def next_var(self, prefix='v'):
25
+ self._var_counter += 1
26
+ return f'_{prefix}{self._var_counter}'
27
+
28
+ def emit(self, indent, line):
29
+ self.lines.append(f'{indent}{line}')
30
+
31
+ def source(self):
32
+ return '\n'.join(self.lines)
33
+
34
+ def print(self):
35
+ print('GLOBALS:')
36
+ for var in self.globs:
37
+ print(f' {var}={self.globs[var]}')
38
+ print('\nSOURCE:')
39
+ for i, line in enumerate(self.lines):
40
+ print(f'{i+1:<4} {line}')
@@ -0,0 +1,127 @@
1
+ import uuid
2
+
3
+ from .base import BaseField
4
+ from .codecs import (
5
+ BitField, Boolean, Bytes,
6
+ Float64, Int8, Int16, Int32, Int64, String, UnsignedInt16, UUID
7
+ )
8
+
9
+
10
+ class SimpleField(BaseField):
11
+ TYPES = {
12
+ 'int8': Int8,
13
+ 'int16': Int16,
14
+ 'uint16': UnsignedInt16,
15
+ 'int32': Int32,
16
+ #'uint32': UnsignedInt32,
17
+ 'int64': Int64,
18
+ 'float64': Float64,
19
+ 'bool': Boolean,
20
+ 'uuid': UUID,
21
+ 'string': String('utf-8'),
22
+ 'bytes': Bytes,
23
+ 'records': Bytes,
24
+ 'bitfield': BitField, # patched only; does not exist in raw schemas
25
+ }
26
+
27
+ @classmethod
28
+ def parse_json(cls, json):
29
+ if 'fields' not in json and json['type'] in cls.TYPES:
30
+ return cls(json)
31
+
32
+ def __init__(self, json):
33
+ if 'fields' in json:
34
+ raise ValueError('Fields not allowed in SimpleField!')
35
+ super().__init__(json)
36
+ if self._type_str not in self.TYPES:
37
+ raise ValueError('Unrecognized type: %s' % self._type_str)
38
+ self._type = self.TYPES[self._type_str]
39
+
40
+ def is_batchable(self):
41
+ return getattr(self._type, 'batchable', False)
42
+
43
+ def _calculate_default(self, default):
44
+ if self._type is Boolean:
45
+ if not default:
46
+ return False
47
+ if isinstance(default, str):
48
+ if default.lower() == 'false':
49
+ return False
50
+ elif default.lower() == 'true':
51
+ return True
52
+ else:
53
+ raise ValueError('Invalid default for boolean field %s: %s' % (self._name, default))
54
+ return bool(default)
55
+ elif self._type in (Int8, Int16, Int32, Int64, UnsignedInt16):
56
+ if not default:
57
+ return 0
58
+ if isinstance(default, str):
59
+ if default.lower().startswith('0x'):
60
+ return int(default, 16)
61
+ else:
62
+ return int(default)
63
+ return int(default)
64
+ elif self._type is UUID:
65
+ if not default:
66
+ return None
67
+ else:
68
+ return uuid.UUID(default)
69
+ elif self._type is Float64:
70
+ if not default:
71
+ return 0.0
72
+ else:
73
+ return float(default)
74
+ elif self._type is BitField:
75
+ if not default:
76
+ return None
77
+ else:
78
+ default = BitField.from_32_bit_field(int(default))
79
+ if default == {31}:
80
+ return None
81
+ return default
82
+ elif default == 'null':
83
+ return None
84
+ elif isinstance(self._type, String):
85
+ return default
86
+ elif not default:
87
+ if self._type is Bytes:
88
+ return b''
89
+ else:
90
+ raise ValueError('Invalid default for field %s. The only valid default is empty or null.' % self._name)
91
+
92
+ def encode(self, value, version=None, compact=False, tagged=False):
93
+ return self._type.encode(value, compact=compact)
94
+
95
+ def encode_into(self, value, out, version=None, compact=False, tagged=False):
96
+ self._type.encode_into(out, value, compact=compact)
97
+
98
+ def emit_encode_into(self, ctx, val_expr, indent, version=None, compact=False, tagged=False):
99
+ self._type.emit_encode_into(ctx, val_expr, indent, compact=compact)
100
+
101
+ def emit_decode_from(self, ctx, var_name, indent, version=None, compact=False, tagged=False):
102
+ self._type.emit_decode_from(ctx, var_name, indent, compact=compact)
103
+
104
+ def decode(self, data, version=None, compact=False, tagged=False):
105
+ return self._type.decode(data, compact=compact)
106
+
107
+ def to_json(self, val):
108
+ if val is None:
109
+ return None
110
+ if self._type is UUID:
111
+ return str(val)
112
+ elif self._type is Bytes:
113
+ if isinstance(val, memoryview):
114
+ val = val.tobytes()
115
+ if hasattr(val, 'to_dict'):
116
+ return val.to_dict()
117
+ elif not isinstance(val, (bytes, bytearray)):
118
+ val = val.encode()
119
+ return val.decode(errors='backslashreplace')
120
+
121
+ elif self._type is BitField:
122
+ return list(val)
123
+ else:
124
+ return val
125
+
126
+ def __repr__(self):
127
+ return 'SimpleField(%s)' % self._json