midstream-platform 1.0.0rc1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (245) hide show
  1. midstream_platform-1.0.0rc1/PKG-INFO +38 -0
  2. midstream_platform-1.0.0rc1/README.md +11 -0
  3. midstream_platform-1.0.0rc1/pyproject.toml +47 -0
  4. midstream_platform-1.0.0rc1/setup.cfg +4 -0
  5. midstream_platform-1.0.0rc1/src/midstream/__init__.py +5 -0
  6. midstream_platform-1.0.0rc1/src/midstream/config.py +577 -0
  7. midstream_platform-1.0.0rc1/src/midstream/handlers/__init__.py +34 -0
  8. midstream_platform-1.0.0rc1/src/midstream/handlers/identity.py +60 -0
  9. midstream_platform-1.0.0rc1/src/midstream/handlers/maintenance.py +207 -0
  10. midstream_platform-1.0.0rc1/src/midstream/handlers/request_intake.py +51 -0
  11. midstream_platform-1.0.0rc1/src/midstream/handlers/resource_observation.py +272 -0
  12. midstream_platform-1.0.0rc1/src/midstream/handlers/runtime_status.py +14 -0
  13. midstream_platform-1.0.0rc1/src/midstream/handlers/settings.py +29 -0
  14. midstream_platform-1.0.0rc1/src/midstream/handlers/table_events.py +34 -0
  15. midstream_platform-1.0.0rc1/src/midstream/handlers/workflow_apply.py +276 -0
  16. midstream_platform-1.0.0rc1/src/midstream/identity/__init__.py +72 -0
  17. midstream_platform-1.0.0rc1/src/midstream/identity/access_tokens.py +72 -0
  18. midstream_platform-1.0.0rc1/src/midstream/identity/authorization.py +296 -0
  19. midstream_platform-1.0.0rc1/src/midstream/identity/directory.py +144 -0
  20. midstream_platform-1.0.0rc1/src/midstream/identity/passwords.py +169 -0
  21. midstream_platform-1.0.0rc1/src/midstream/identity/permissions.py +519 -0
  22. midstream_platform-1.0.0rc1/src/midstream/identity/principals.py +375 -0
  23. midstream_platform-1.0.0rc1/src/midstream/identity/secrets.py +119 -0
  24. midstream_platform-1.0.0rc1/src/midstream/identity/tokens.py +167 -0
  25. midstream_platform-1.0.0rc1/src/midstream/identity/validation.py +121 -0
  26. midstream_platform-1.0.0rc1/src/midstream/kafka/__init__.py +25 -0
  27. midstream_platform-1.0.0rc1/src/midstream/kafka/admin.py +403 -0
  28. midstream_platform-1.0.0rc1/src/midstream/kafka/metadata.py +429 -0
  29. midstream_platform-1.0.0rc1/src/midstream/kubernetes/__init__.py +5 -0
  30. midstream_platform-1.0.0rc1/src/midstream/kubernetes/client.py +5 -0
  31. midstream_platform-1.0.0rc1/src/midstream/kubernetes/kaspr_apps.py +341 -0
  32. midstream_platform-1.0.0rc1/src/midstream/lakehouse/__init__.py +5 -0
  33. midstream_platform-1.0.0rc1/src/midstream/lakehouse/catalog.py +406 -0
  34. midstream_platform-1.0.0rc1/src/midstream/lakehouse/commits.py +595 -0
  35. midstream_platform-1.0.0rc1/src/midstream/lakehouse/files.py +241 -0
  36. midstream_platform-1.0.0rc1/src/midstream/lakehouse/gcs.py +116 -0
  37. midstream_platform-1.0.0rc1/src/midstream/lakehouse/io.py +180 -0
  38. midstream_platform-1.0.0rc1/src/midstream/lakehouse/lifecycle.py +81 -0
  39. midstream_platform-1.0.0rc1/src/midstream/lakehouse/maintenance.py +604 -0
  40. midstream_platform-1.0.0rc1/src/midstream/lakehouse/offsets.py +147 -0
  41. midstream_platform-1.0.0rc1/src/midstream/lakehouse/parquet.py +103 -0
  42. midstream_platform-1.0.0rc1/src/midstream/lakehouse/pointer.py +228 -0
  43. midstream_platform-1.0.0rc1/src/midstream/lakehouse/rest.py +137 -0
  44. midstream_platform-1.0.0rc1/src/midstream/lakehouse/rows.py +82 -0
  45. midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/__init__.py +1 -0
  46. midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/conform.py +225 -0
  47. midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/forkmap.py +111 -0
  48. midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/index.py +304 -0
  49. midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/infer.py +124 -0
  50. midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/policy.py +867 -0
  51. midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/render.py +166 -0
  52. midstream_platform-1.0.0rc1/src/midstream/lakehouse/tables.py +280 -0
  53. midstream_platform-1.0.0rc1/src/midstream/maintenance_app/__init__.py +1 -0
  54. midstream_platform-1.0.0rc1/src/midstream/maintenance_app/compact_worker.py +234 -0
  55. midstream_platform-1.0.0rc1/src/midstream/maintenance_app/config.py +54 -0
  56. midstream_platform-1.0.0rc1/src/midstream/maintenance_app/handlers.py +42 -0
  57. midstream_platform-1.0.0rc1/src/midstream/maintenance_app/worker.py +981 -0
  58. midstream_platform-1.0.0rc1/src/midstream/models/__init__.py +163 -0
  59. midstream_platform-1.0.0rc1/src/midstream/models/autoscale.py +85 -0
  60. midstream_platform-1.0.0rc1/src/midstream/models/binding.py +98 -0
  61. midstream_platform-1.0.0rc1/src/midstream/models/envelopes.py +44 -0
  62. midstream_platform-1.0.0rc1/src/midstream/models/events.py +624 -0
  63. midstream_platform-1.0.0rc1/src/midstream/models/identity.py +893 -0
  64. midstream_platform-1.0.0rc1/src/midstream/models/live_tail.py +162 -0
  65. midstream_platform-1.0.0rc1/src/midstream/models/maintenance.py +139 -0
  66. midstream_platform-1.0.0rc1/src/midstream/models/materialization.py +429 -0
  67. midstream_platform-1.0.0rc1/src/midstream/models/offsets.py +43 -0
  68. midstream_platform-1.0.0rc1/src/midstream/models/settings.py +374 -0
  69. midstream_platform-1.0.0rc1/src/midstream/models/status.py +641 -0
  70. midstream_platform-1.0.0rc1/src/midstream/models/table.py +299 -0
  71. midstream_platform-1.0.0rc1/src/midstream/observation/__init__.py +41 -0
  72. midstream_platform-1.0.0rc1/src/midstream/observation/mapping.py +332 -0
  73. midstream_platform-1.0.0rc1/src/midstream/observation/policy.py +73 -0
  74. midstream_platform-1.0.0rc1/src/midstream/observation/progress.py +328 -0
  75. midstream_platform-1.0.0rc1/src/midstream/observation/reader.py +46 -0
  76. midstream_platform-1.0.0rc1/src/midstream/observation/restarts.py +101 -0
  77. midstream_platform-1.0.0rc1/src/midstream/protocol_logging.py +66 -0
  78. midstream_platform-1.0.0rc1/src/midstream/runtime/__init__.py +51 -0
  79. midstream_platform-1.0.0rc1/src/midstream/runtime/bindings.py +33 -0
  80. midstream_platform-1.0.0rc1/src/midstream/runtime/naming.py +86 -0
  81. midstream_platform-1.0.0rc1/src/midstream/runtime/templates.py +434 -0
  82. midstream_platform-1.0.0rc1/src/midstream/runtime_app/__init__.py +1 -0
  83. midstream_platform-1.0.0rc1/src/midstream/runtime_app/acks.py +329 -0
  84. midstream_platform-1.0.0rc1/src/midstream/runtime_app/committer.py +885 -0
  85. midstream_platform-1.0.0rc1/src/midstream/runtime_app/config.py +207 -0
  86. midstream_platform-1.0.0rc1/src/midstream/runtime_app/flow.py +100 -0
  87. midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/__init__.py +11 -0
  88. midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/runtime_settings.py +124 -0
  89. midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/status.py +207 -0
  90. midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/writer.py +366 -0
  91. midstream_platform-1.0.0rc1/src/midstream/runtime_app/live_settings.py +222 -0
  92. midstream_platform-1.0.0rc1/src/midstream/runtime_app/models/__init__.py +23 -0
  93. midstream_platform-1.0.0rc1/src/midstream/runtime_app/models/processing_state.py +402 -0
  94. midstream_platform-1.0.0rc1/src/midstream/runtime_app/outbox.py +74 -0
  95. midstream_platform-1.0.0rc1/src/midstream/runtime_app/performance.py +180 -0
  96. midstream_platform-1.0.0rc1/src/midstream/runtime_app/pipeline.py +995 -0
  97. midstream_platform-1.0.0rc1/src/midstream/runtime_app/schema/__init__.py +1 -0
  98. midstream_platform-1.0.0rc1/src/midstream/runtime_app/schema/drift.py +146 -0
  99. midstream_platform-1.0.0rc1/src/midstream/runtime_app/state.py +478 -0
  100. midstream_platform-1.0.0rc1/src/midstream/runtime_app/table_state.py +366 -0
  101. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/commit-timer-task.yaml.j2 +40 -0
  102. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/committer-agent.yaml.j2 +68 -0
  103. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/kaspr-app.yaml.j2 +11 -0
  104. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/pipeline-task.yaml.j2 +45 -0
  105. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/settings-agent.yaml.j2 +39 -0
  106. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/state-table.yaml.j2 +15 -0
  107. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/status-task.yaml.j2 +57 -0
  108. midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/writer-agent.yaml.j2 +58 -0
  109. midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/__init__.py +19 -0
  110. midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/batches.py +230 -0
  111. midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/context.py +145 -0
  112. midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/seal.py +261 -0
  113. midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/seal_worker.py +45 -0
  114. midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/upload.py +58 -0
  115. midstream_platform-1.0.0rc1/src/midstream/scale.py +180 -0
  116. midstream_platform-1.0.0rc1/src/midstream/scheduler/__init__.py +81 -0
  117. midstream_platform-1.0.0rc1/src/midstream/schema/__init__.py +8 -0
  118. midstream_platform-1.0.0rc1/src/midstream/schema/discovery.py +423 -0
  119. midstream_platform-1.0.0rc1/src/midstream/services/__init__.py +51 -0
  120. midstream_platform-1.0.0rc1/src/midstream/services/autoscaling.py +294 -0
  121. midstream_platform-1.0.0rc1/src/midstream/services/consumer_lag.py +349 -0
  122. midstream_platform-1.0.0rc1/src/midstream/services/identity.py +1998 -0
  123. midstream_platform-1.0.0rc1/src/midstream/services/live_tail.py +139 -0
  124. midstream_platform-1.0.0rc1/src/midstream/services/observation.py +550 -0
  125. midstream_platform-1.0.0rc1/src/midstream/services/request_intake.py +608 -0
  126. midstream_platform-1.0.0rc1/src/midstream/services/retention.py +216 -0
  127. midstream_platform-1.0.0rc1/src/midstream/services/schema_projection.py +43 -0
  128. midstream_platform-1.0.0rc1/src/midstream/services/settings.py +456 -0
  129. midstream_platform-1.0.0rc1/src/midstream/services/sweep_cursor.py +39 -0
  130. midstream_platform-1.0.0rc1/src/midstream/services/table_claims.py +153 -0
  131. midstream_platform-1.0.0rc1/src/midstream/services/table_events.py +384 -0
  132. midstream_platform-1.0.0rc1/src/midstream/services/table_maintenance.py +155 -0
  133. midstream_platform-1.0.0rc1/src/midstream/services/topic_analysis.py +121 -0
  134. midstream_platform-1.0.0rc1/src/midstream/services/topic_catalog.py +174 -0
  135. midstream_platform-1.0.0rc1/src/midstream/services/topic_changes.py +210 -0
  136. midstream_platform-1.0.0rc1/src/midstream/services/workflow.py +3231 -0
  137. midstream_platform-1.0.0rc1/src/midstream/settings/__init__.py +49 -0
  138. midstream_platform-1.0.0rc1/src/midstream/settings/catalog.py +1332 -0
  139. midstream_platform-1.0.0rc1/src/midstream/settings/effective.py +221 -0
  140. midstream_platform-1.0.0rc1/src/midstream/storage/__init__.py +139 -0
  141. midstream_platform-1.0.0rc1/src/midstream/storage/autoscale.py +38 -0
  142. midstream_platform-1.0.0rc1/src/midstream/storage/bindings.py +47 -0
  143. midstream_platform-1.0.0rc1/src/midstream/storage/envelopes.py +72 -0
  144. midstream_platform-1.0.0rc1/src/midstream/storage/identity.py +749 -0
  145. midstream_platform-1.0.0rc1/src/midstream/storage/keys.py +79 -0
  146. midstream_platform-1.0.0rc1/src/midstream/storage/materializations.py +230 -0
  147. midstream_platform-1.0.0rc1/src/midstream/storage/objects.py +550 -0
  148. midstream_platform-1.0.0rc1/src/midstream/storage/operations.py +72 -0
  149. midstream_platform-1.0.0rc1/src/midstream/storage/resolve.py +42 -0
  150. midstream_platform-1.0.0rc1/src/midstream/storage/schemas.py +180 -0
  151. midstream_platform-1.0.0rc1/src/midstream/storage/settings.py +119 -0
  152. midstream_platform-1.0.0rc1/src/midstream/storage/status.py +49 -0
  153. midstream_platform-1.0.0rc1/src/midstream/storage/table.py +20 -0
  154. midstream_platform-1.0.0rc1/src/midstream/storage/tables.py +66 -0
  155. midstream_platform-1.0.0rc1/src/midstream/table_names.py +251 -0
  156. midstream_platform-1.0.0rc1/src/midstream/tail_app/__init__.py +1 -0
  157. midstream_platform-1.0.0rc1/src/midstream/tail_app/auth.py +149 -0
  158. midstream_platform-1.0.0rc1/src/midstream/tail_app/buffer.py +445 -0
  159. midstream_platform-1.0.0rc1/src/midstream/tail_app/config.py +191 -0
  160. midstream_platform-1.0.0rc1/src/midstream/tail_app/consumer.py +238 -0
  161. midstream_platform-1.0.0rc1/src/midstream/tail_app/control_plane.py +90 -0
  162. midstream_platform-1.0.0rc1/src/midstream/tail_app/events.py +62 -0
  163. midstream_platform-1.0.0rc1/src/midstream/tail_app/handlers.py +97 -0
  164. midstream_platform-1.0.0rc1/src/midstream/tail_app/http.py +71 -0
  165. midstream_platform-1.0.0rc1/src/midstream/tail_app/ingest.py +127 -0
  166. midstream_platform-1.0.0rc1/src/midstream/tail_app/peek.py +218 -0
  167. midstream_platform-1.0.0rc1/src/midstream/tail_app/routing.py +213 -0
  168. midstream_platform-1.0.0rc1/src/midstream/tail_app/serve.py +233 -0
  169. midstream_platform-1.0.0rc1/src/midstream/tail_app/table_tail.py +625 -0
  170. midstream_platform-1.0.0rc1/src/midstream/tail_app/views.py +396 -0
  171. midstream_platform-1.0.0rc1/src/midstream/tail_app/worker.py +808 -0
  172. midstream_platform-1.0.0rc1/src/midstream/utils/__init__.py +1 -0
  173. midstream_platform-1.0.0rc1/src/midstream/utils/background_loop.py +127 -0
  174. midstream_platform-1.0.0rc1/src/midstream/version.py +33 -0
  175. midstream_platform-1.0.0rc1/src/midstream/web/__init__.py +52 -0
  176. midstream_platform-1.0.0rc1/src/midstream/web/auth.py +554 -0
  177. midstream_platform-1.0.0rc1/src/midstream/web/catalog.py +299 -0
  178. midstream_platform-1.0.0rc1/src/midstream/web/datasets.py +455 -0
  179. midstream_platform-1.0.0rc1/src/midstream/web/errors.py +136 -0
  180. midstream_platform-1.0.0rc1/src/midstream/web/http.py +29 -0
  181. midstream_platform-1.0.0rc1/src/midstream/web/identity.py +870 -0
  182. midstream_platform-1.0.0rc1/src/midstream/web/materializations.py +1287 -0
  183. midstream_platform-1.0.0rc1/src/midstream/web/responses.py +327 -0
  184. midstream_platform-1.0.0rc1/src/midstream/web/settings.py +424 -0
  185. midstream_platform-1.0.0rc1/src/midstream/web/system.py +75 -0
  186. midstream_platform-1.0.0rc1/src/midstream/web/tokens.py +251 -0
  187. midstream_platform-1.0.0rc1/src/midstream/web/topics.py +122 -0
  188. midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/PKG-INFO +38 -0
  189. midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/SOURCES.txt +243 -0
  190. midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/dependency_links.txt +1 -0
  191. midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/requires.txt +7 -0
  192. midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/top_level.txt +1 -0
  193. midstream_platform-1.0.0rc1/tests/test_agent_manifests.py +1106 -0
  194. midstream_platform-1.0.0rc1/tests/test_autoscaling.py +446 -0
  195. midstream_platform-1.0.0rc1/tests/test_batched_destination_deletes.py +363 -0
  196. midstream_platform-1.0.0rc1/tests/test_catalog.py +423 -0
  197. midstream_platform-1.0.0rc1/tests/test_config.py +208 -0
  198. midstream_platform-1.0.0rc1/tests/test_datasets.py +703 -0
  199. midstream_platform-1.0.0rc1/tests/test_identity.py +1476 -0
  200. midstream_platform-1.0.0rc1/tests/test_kafka_admin.py +481 -0
  201. midstream_platform-1.0.0rc1/tests/test_kubernetes_client.py +324 -0
  202. midstream_platform-1.0.0rc1/tests/test_lakehouse_catalog.py +197 -0
  203. midstream_platform-1.0.0rc1/tests/test_lakehouse_commits.py +402 -0
  204. midstream_platform-1.0.0rc1/tests/test_lakehouse_conform.py +165 -0
  205. midstream_platform-1.0.0rc1/tests/test_lakehouse_maintenance.py +597 -0
  206. midstream_platform-1.0.0rc1/tests/test_lakehouse_schema_policy.py +247 -0
  207. midstream_platform-1.0.0rc1/tests/test_live_tail_scheduler.py +238 -0
  208. midstream_platform-1.0.0rc1/tests/test_maintenance.py +1997 -0
  209. midstream_platform-1.0.0rc1/tests/test_maintenance_app.py +665 -0
  210. midstream_platform-1.0.0rc1/tests/test_models.py +806 -0
  211. midstream_platform-1.0.0rc1/tests/test_observation.py +1661 -0
  212. midstream_platform-1.0.0rc1/tests/test_package_layout.py +40 -0
  213. midstream_platform-1.0.0rc1/tests/test_request_intake.py +511 -0
  214. midstream_platform-1.0.0rc1/tests/test_row_builder.py +128 -0
  215. midstream_platform-1.0.0rc1/tests/test_runtime_acks.py +627 -0
  216. midstream_platform-1.0.0rc1/tests/test_runtime_committer.py +385 -0
  217. midstream_platform-1.0.0rc1/tests/test_runtime_config.py +61 -0
  218. midstream_platform-1.0.0rc1/tests/test_runtime_flow.py +311 -0
  219. midstream_platform-1.0.0rc1/tests/test_runtime_multi_topic.py +335 -0
  220. midstream_platform-1.0.0rc1/tests/test_runtime_seal.py +218 -0
  221. midstream_platform-1.0.0rc1/tests/test_runtime_settings.py +412 -0
  222. midstream_platform-1.0.0rc1/tests/test_runtime_status.py +100 -0
  223. midstream_platform-1.0.0rc1/tests/test_runtime_templates.py +516 -0
  224. midstream_platform-1.0.0rc1/tests/test_runtime_writer.py +171 -0
  225. midstream_platform-1.0.0rc1/tests/test_scheduler_routing.py +530 -0
  226. midstream_platform-1.0.0rc1/tests/test_schema_discovery.py +438 -0
  227. midstream_platform-1.0.0rc1/tests/test_settings.py +934 -0
  228. midstream_platform-1.0.0rc1/tests/test_storage.py +689 -0
  229. midstream_platform-1.0.0rc1/tests/test_table_events.py +466 -0
  230. midstream_platform-1.0.0rc1/tests/test_table_maintenance.py +196 -0
  231. midstream_platform-1.0.0rc1/tests/test_table_names.py +148 -0
  232. midstream_platform-1.0.0rc1/tests/test_tail_app.py +725 -0
  233. midstream_platform-1.0.0rc1/tests/test_tail_buffer.py +294 -0
  234. midstream_platform-1.0.0rc1/tests/test_tail_exactly_once.py +258 -0
  235. midstream_platform-1.0.0rc1/tests/test_tail_failures.py +276 -0
  236. midstream_platform-1.0.0rc1/tests/test_tail_http.py +88 -0
  237. midstream_platform-1.0.0rc1/tests/test_tail_parity.py +329 -0
  238. midstream_platform-1.0.0rc1/tests/test_tail_routing.py +265 -0
  239. midstream_platform-1.0.0rc1/tests/test_tail_security.py +230 -0
  240. midstream_platform-1.0.0rc1/tests/test_topic_catalog.py +576 -0
  241. midstream_platform-1.0.0rc1/tests/test_topic_changes.py +247 -0
  242. midstream_platform-1.0.0rc1/tests/test_versioning.py +164 -0
  243. midstream_platform-1.0.0rc1/tests/test_web.py +1681 -0
  244. midstream_platform-1.0.0rc1/tests/test_workflow.py +763 -0
  245. midstream_platform-1.0.0rc1/tests/test_workflow_topic_changes.py +527 -0
@@ -0,0 +1,38 @@
1
+ Metadata-Version: 2.4
2
+ Name: midstream-platform
3
+ Version: 1.0.0rc1
4
+ Summary: Midstream: a streaming platform on Kaspr that materializes Kafka topics into Iceberg tables
5
+ License: MIT
6
+ Project-URL: Homepage, https://github.com/vmirz/midstream
7
+ Project-URL: Repository, https://github.com/vmirz/midstream
8
+ Project-URL: Documentation, https://github.com/vmirz/midstream/tree/main/docs/handbook
9
+ Project-URL: Changelog, https://github.com/vmirz/midstream/releases
10
+ Keywords: kafka,iceberg,kaspr,streaming,lakehouse
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: System :: Distributed Computing
18
+ Requires-Python: >=3.12
19
+ Description-Content-Type: text/markdown
20
+ Requires-Dist: jinja2>=3.1.0
21
+ Requires-Dist: google-cloud-storage>=2.18.0
22
+ Requires-Dist: kubernetes-asyncio>=33.1.0
23
+ Requires-Dist: pyarrow>=18.1.0
24
+ Requires-Dist: pyiceberg[pyarrow]<0.13,>=0.12
25
+ Requires-Dist: pyyaml>=6.0.0
26
+ Requires-Dist: tzdata>=2024.1
27
+
28
+ # midstream-platform
29
+
30
+ The Python package of [Midstream](https://github.com/vmirz/midstream), a streaming platform on
31
+ [Kaspr](https://kaspr.dev) that materializes Kafka topics into Apache Iceberg tables, exactly once,
32
+ with schemas that evolve by themselves, an Iceberg REST catalog, accounts and permissions, and a web
33
+ console.
34
+
35
+ One distribution holds every server-side part: the control plane, the per-materialization runtime,
36
+ the maintenance and tail apps, and the Iceberg library they share. It is not a library to import into
37
+ your own code: Kaspr applications install it in their pods, and the Helm chart in the repository
38
+ deploys them. See the repository's `docs/install.md` to run it, and `docs/handbook/` for how it works.
@@ -0,0 +1,11 @@
1
+ # midstream-platform
2
+
3
+ The Python package of [Midstream](https://github.com/vmirz/midstream), a streaming platform on
4
+ [Kaspr](https://kaspr.dev) that materializes Kafka topics into Apache Iceberg tables, exactly once,
5
+ with schemas that evolve by themselves, an Iceberg REST catalog, accounts and permissions, and a web
6
+ console.
7
+
8
+ One distribution holds every server-side part: the control plane, the per-materialization runtime,
9
+ the maintenance and tail apps, and the Iceberg library they share. It is not a library to import into
10
+ your own code: Kaspr applications install it in their pods, and the Helm chart in the repository
11
+ deploys them. See the repository's `docs/install.md` to run it, and `docs/handbook/` for how it works.
@@ -0,0 +1,47 @@
1
+ [build-system]
2
+ requires = ["setuptools>=69"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "midstream-platform"
7
+ version = "1.0.0rc1"
8
+ description = "Midstream: a streaming platform on Kaspr that materializes Kafka topics into Iceberg tables"
9
+ readme = "README.md"
10
+ license = {text = "MIT"}
11
+ requires-python = ">=3.12"
12
+ keywords = ["kafka", "iceberg", "kaspr", "streaming", "lakehouse"]
13
+ classifiers = [
14
+ "Development Status :: 4 - Beta",
15
+ "Intended Audience :: Developers",
16
+ "License :: OSI Approved :: MIT License",
17
+ "Programming Language :: Python :: 3",
18
+ "Programming Language :: Python :: 3.12",
19
+ "Programming Language :: Python :: 3.13",
20
+ "Topic :: System :: Distributed Computing",
21
+ ]
22
+ dependencies = [
23
+ "jinja2>=3.1.0",
24
+ "google-cloud-storage>=2.18.0",
25
+ "kubernetes-asyncio>=33.1.0",
26
+ "pyarrow>=18.1.0",
27
+ # The minor is pinned: the committer uses documented-but-internal update APIs (lakehouse/catalog.py
28
+ # `commit_once`, lakehouse/commits.py). The tests exercise them, so a new minor is a deliberate bump.
29
+ "pyiceberg[pyarrow]>=0.12,<0.13",
30
+ "pyyaml>=6.0.0",
31
+ "tzdata>=2024.1",
32
+ ]
33
+
34
+ [tool.setuptools]
35
+ package-dir = {"" = "src"}
36
+
37
+ [tool.setuptools.packages.find]
38
+ where = ["src"]
39
+
40
+ [tool.setuptools.package-data]
41
+ "midstream.runtime_app" = ["templates/*.j2"]
42
+
43
+ [project.urls]
44
+ Homepage = "https://github.com/vmirz/midstream"
45
+ Repository = "https://github.com/vmirz/midstream"
46
+ Documentation = "https://github.com/vmirz/midstream/tree/main/docs/handbook"
47
+ Changelog = "https://github.com/vmirz/midstream/releases"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,5 @@
1
+ """Kaspr control-plane package for Kafka topic materialization."""
2
+
3
+ from midstream.config import ControlPlaneConfig, get_config, normalize_list_limit
4
+
5
+ __all__ = ["ControlPlaneConfig", "get_config", "normalize_list_limit"]