midstream-platform 1.0.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- midstream_platform-1.0.0rc1/PKG-INFO +38 -0
- midstream_platform-1.0.0rc1/README.md +11 -0
- midstream_platform-1.0.0rc1/pyproject.toml +47 -0
- midstream_platform-1.0.0rc1/setup.cfg +4 -0
- midstream_platform-1.0.0rc1/src/midstream/__init__.py +5 -0
- midstream_platform-1.0.0rc1/src/midstream/config.py +577 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/__init__.py +34 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/identity.py +60 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/maintenance.py +207 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/request_intake.py +51 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/resource_observation.py +272 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/runtime_status.py +14 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/settings.py +29 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/table_events.py +34 -0
- midstream_platform-1.0.0rc1/src/midstream/handlers/workflow_apply.py +276 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/__init__.py +72 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/access_tokens.py +72 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/authorization.py +296 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/directory.py +144 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/passwords.py +169 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/permissions.py +519 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/principals.py +375 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/secrets.py +119 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/tokens.py +167 -0
- midstream_platform-1.0.0rc1/src/midstream/identity/validation.py +121 -0
- midstream_platform-1.0.0rc1/src/midstream/kafka/__init__.py +25 -0
- midstream_platform-1.0.0rc1/src/midstream/kafka/admin.py +403 -0
- midstream_platform-1.0.0rc1/src/midstream/kafka/metadata.py +429 -0
- midstream_platform-1.0.0rc1/src/midstream/kubernetes/__init__.py +5 -0
- midstream_platform-1.0.0rc1/src/midstream/kubernetes/client.py +5 -0
- midstream_platform-1.0.0rc1/src/midstream/kubernetes/kaspr_apps.py +341 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/__init__.py +5 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/catalog.py +406 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/commits.py +595 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/files.py +241 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/gcs.py +116 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/io.py +180 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/lifecycle.py +81 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/maintenance.py +604 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/offsets.py +147 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/parquet.py +103 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/pointer.py +228 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/rest.py +137 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/rows.py +82 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/__init__.py +1 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/conform.py +225 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/forkmap.py +111 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/index.py +304 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/infer.py +124 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/policy.py +867 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/schema/render.py +166 -0
- midstream_platform-1.0.0rc1/src/midstream/lakehouse/tables.py +280 -0
- midstream_platform-1.0.0rc1/src/midstream/maintenance_app/__init__.py +1 -0
- midstream_platform-1.0.0rc1/src/midstream/maintenance_app/compact_worker.py +234 -0
- midstream_platform-1.0.0rc1/src/midstream/maintenance_app/config.py +54 -0
- midstream_platform-1.0.0rc1/src/midstream/maintenance_app/handlers.py +42 -0
- midstream_platform-1.0.0rc1/src/midstream/maintenance_app/worker.py +981 -0
- midstream_platform-1.0.0rc1/src/midstream/models/__init__.py +163 -0
- midstream_platform-1.0.0rc1/src/midstream/models/autoscale.py +85 -0
- midstream_platform-1.0.0rc1/src/midstream/models/binding.py +98 -0
- midstream_platform-1.0.0rc1/src/midstream/models/envelopes.py +44 -0
- midstream_platform-1.0.0rc1/src/midstream/models/events.py +624 -0
- midstream_platform-1.0.0rc1/src/midstream/models/identity.py +893 -0
- midstream_platform-1.0.0rc1/src/midstream/models/live_tail.py +162 -0
- midstream_platform-1.0.0rc1/src/midstream/models/maintenance.py +139 -0
- midstream_platform-1.0.0rc1/src/midstream/models/materialization.py +429 -0
- midstream_platform-1.0.0rc1/src/midstream/models/offsets.py +43 -0
- midstream_platform-1.0.0rc1/src/midstream/models/settings.py +374 -0
- midstream_platform-1.0.0rc1/src/midstream/models/status.py +641 -0
- midstream_platform-1.0.0rc1/src/midstream/models/table.py +299 -0
- midstream_platform-1.0.0rc1/src/midstream/observation/__init__.py +41 -0
- midstream_platform-1.0.0rc1/src/midstream/observation/mapping.py +332 -0
- midstream_platform-1.0.0rc1/src/midstream/observation/policy.py +73 -0
- midstream_platform-1.0.0rc1/src/midstream/observation/progress.py +328 -0
- midstream_platform-1.0.0rc1/src/midstream/observation/reader.py +46 -0
- midstream_platform-1.0.0rc1/src/midstream/observation/restarts.py +101 -0
- midstream_platform-1.0.0rc1/src/midstream/protocol_logging.py +66 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime/__init__.py +51 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime/bindings.py +33 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime/naming.py +86 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime/templates.py +434 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/__init__.py +1 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/acks.py +329 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/committer.py +885 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/config.py +207 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/flow.py +100 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/__init__.py +11 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/runtime_settings.py +124 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/status.py +207 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/handlers/writer.py +366 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/live_settings.py +222 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/models/__init__.py +23 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/models/processing_state.py +402 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/outbox.py +74 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/performance.py +180 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/pipeline.py +995 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/schema/__init__.py +1 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/schema/drift.py +146 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/state.py +478 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/table_state.py +366 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/commit-timer-task.yaml.j2 +40 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/committer-agent.yaml.j2 +68 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/kaspr-app.yaml.j2 +11 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/pipeline-task.yaml.j2 +45 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/settings-agent.yaml.j2 +39 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/state-table.yaml.j2 +15 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/status-task.yaml.j2 +57 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/templates/writer-agent.yaml.j2 +58 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/__init__.py +19 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/batches.py +230 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/context.py +145 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/seal.py +261 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/seal_worker.py +45 -0
- midstream_platform-1.0.0rc1/src/midstream/runtime_app/writer/upload.py +58 -0
- midstream_platform-1.0.0rc1/src/midstream/scale.py +180 -0
- midstream_platform-1.0.0rc1/src/midstream/scheduler/__init__.py +81 -0
- midstream_platform-1.0.0rc1/src/midstream/schema/__init__.py +8 -0
- midstream_platform-1.0.0rc1/src/midstream/schema/discovery.py +423 -0
- midstream_platform-1.0.0rc1/src/midstream/services/__init__.py +51 -0
- midstream_platform-1.0.0rc1/src/midstream/services/autoscaling.py +294 -0
- midstream_platform-1.0.0rc1/src/midstream/services/consumer_lag.py +349 -0
- midstream_platform-1.0.0rc1/src/midstream/services/identity.py +1998 -0
- midstream_platform-1.0.0rc1/src/midstream/services/live_tail.py +139 -0
- midstream_platform-1.0.0rc1/src/midstream/services/observation.py +550 -0
- midstream_platform-1.0.0rc1/src/midstream/services/request_intake.py +608 -0
- midstream_platform-1.0.0rc1/src/midstream/services/retention.py +216 -0
- midstream_platform-1.0.0rc1/src/midstream/services/schema_projection.py +43 -0
- midstream_platform-1.0.0rc1/src/midstream/services/settings.py +456 -0
- midstream_platform-1.0.0rc1/src/midstream/services/sweep_cursor.py +39 -0
- midstream_platform-1.0.0rc1/src/midstream/services/table_claims.py +153 -0
- midstream_platform-1.0.0rc1/src/midstream/services/table_events.py +384 -0
- midstream_platform-1.0.0rc1/src/midstream/services/table_maintenance.py +155 -0
- midstream_platform-1.0.0rc1/src/midstream/services/topic_analysis.py +121 -0
- midstream_platform-1.0.0rc1/src/midstream/services/topic_catalog.py +174 -0
- midstream_platform-1.0.0rc1/src/midstream/services/topic_changes.py +210 -0
- midstream_platform-1.0.0rc1/src/midstream/services/workflow.py +3231 -0
- midstream_platform-1.0.0rc1/src/midstream/settings/__init__.py +49 -0
- midstream_platform-1.0.0rc1/src/midstream/settings/catalog.py +1332 -0
- midstream_platform-1.0.0rc1/src/midstream/settings/effective.py +221 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/__init__.py +139 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/autoscale.py +38 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/bindings.py +47 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/envelopes.py +72 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/identity.py +749 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/keys.py +79 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/materializations.py +230 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/objects.py +550 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/operations.py +72 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/resolve.py +42 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/schemas.py +180 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/settings.py +119 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/status.py +49 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/table.py +20 -0
- midstream_platform-1.0.0rc1/src/midstream/storage/tables.py +66 -0
- midstream_platform-1.0.0rc1/src/midstream/table_names.py +251 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/__init__.py +1 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/auth.py +149 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/buffer.py +445 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/config.py +191 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/consumer.py +238 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/control_plane.py +90 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/events.py +62 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/handlers.py +97 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/http.py +71 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/ingest.py +127 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/peek.py +218 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/routing.py +213 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/serve.py +233 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/table_tail.py +625 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/views.py +396 -0
- midstream_platform-1.0.0rc1/src/midstream/tail_app/worker.py +808 -0
- midstream_platform-1.0.0rc1/src/midstream/utils/__init__.py +1 -0
- midstream_platform-1.0.0rc1/src/midstream/utils/background_loop.py +127 -0
- midstream_platform-1.0.0rc1/src/midstream/version.py +33 -0
- midstream_platform-1.0.0rc1/src/midstream/web/__init__.py +52 -0
- midstream_platform-1.0.0rc1/src/midstream/web/auth.py +554 -0
- midstream_platform-1.0.0rc1/src/midstream/web/catalog.py +299 -0
- midstream_platform-1.0.0rc1/src/midstream/web/datasets.py +455 -0
- midstream_platform-1.0.0rc1/src/midstream/web/errors.py +136 -0
- midstream_platform-1.0.0rc1/src/midstream/web/http.py +29 -0
- midstream_platform-1.0.0rc1/src/midstream/web/identity.py +870 -0
- midstream_platform-1.0.0rc1/src/midstream/web/materializations.py +1287 -0
- midstream_platform-1.0.0rc1/src/midstream/web/responses.py +327 -0
- midstream_platform-1.0.0rc1/src/midstream/web/settings.py +424 -0
- midstream_platform-1.0.0rc1/src/midstream/web/system.py +75 -0
- midstream_platform-1.0.0rc1/src/midstream/web/tokens.py +251 -0
- midstream_platform-1.0.0rc1/src/midstream/web/topics.py +122 -0
- midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/PKG-INFO +38 -0
- midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/SOURCES.txt +243 -0
- midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/dependency_links.txt +1 -0
- midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/requires.txt +7 -0
- midstream_platform-1.0.0rc1/src/midstream_platform.egg-info/top_level.txt +1 -0
- midstream_platform-1.0.0rc1/tests/test_agent_manifests.py +1106 -0
- midstream_platform-1.0.0rc1/tests/test_autoscaling.py +446 -0
- midstream_platform-1.0.0rc1/tests/test_batched_destination_deletes.py +363 -0
- midstream_platform-1.0.0rc1/tests/test_catalog.py +423 -0
- midstream_platform-1.0.0rc1/tests/test_config.py +208 -0
- midstream_platform-1.0.0rc1/tests/test_datasets.py +703 -0
- midstream_platform-1.0.0rc1/tests/test_identity.py +1476 -0
- midstream_platform-1.0.0rc1/tests/test_kafka_admin.py +481 -0
- midstream_platform-1.0.0rc1/tests/test_kubernetes_client.py +324 -0
- midstream_platform-1.0.0rc1/tests/test_lakehouse_catalog.py +197 -0
- midstream_platform-1.0.0rc1/tests/test_lakehouse_commits.py +402 -0
- midstream_platform-1.0.0rc1/tests/test_lakehouse_conform.py +165 -0
- midstream_platform-1.0.0rc1/tests/test_lakehouse_maintenance.py +597 -0
- midstream_platform-1.0.0rc1/tests/test_lakehouse_schema_policy.py +247 -0
- midstream_platform-1.0.0rc1/tests/test_live_tail_scheduler.py +238 -0
- midstream_platform-1.0.0rc1/tests/test_maintenance.py +1997 -0
- midstream_platform-1.0.0rc1/tests/test_maintenance_app.py +665 -0
- midstream_platform-1.0.0rc1/tests/test_models.py +806 -0
- midstream_platform-1.0.0rc1/tests/test_observation.py +1661 -0
- midstream_platform-1.0.0rc1/tests/test_package_layout.py +40 -0
- midstream_platform-1.0.0rc1/tests/test_request_intake.py +511 -0
- midstream_platform-1.0.0rc1/tests/test_row_builder.py +128 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_acks.py +627 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_committer.py +385 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_config.py +61 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_flow.py +311 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_multi_topic.py +335 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_seal.py +218 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_settings.py +412 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_status.py +100 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_templates.py +516 -0
- midstream_platform-1.0.0rc1/tests/test_runtime_writer.py +171 -0
- midstream_platform-1.0.0rc1/tests/test_scheduler_routing.py +530 -0
- midstream_platform-1.0.0rc1/tests/test_schema_discovery.py +438 -0
- midstream_platform-1.0.0rc1/tests/test_settings.py +934 -0
- midstream_platform-1.0.0rc1/tests/test_storage.py +689 -0
- midstream_platform-1.0.0rc1/tests/test_table_events.py +466 -0
- midstream_platform-1.0.0rc1/tests/test_table_maintenance.py +196 -0
- midstream_platform-1.0.0rc1/tests/test_table_names.py +148 -0
- midstream_platform-1.0.0rc1/tests/test_tail_app.py +725 -0
- midstream_platform-1.0.0rc1/tests/test_tail_buffer.py +294 -0
- midstream_platform-1.0.0rc1/tests/test_tail_exactly_once.py +258 -0
- midstream_platform-1.0.0rc1/tests/test_tail_failures.py +276 -0
- midstream_platform-1.0.0rc1/tests/test_tail_http.py +88 -0
- midstream_platform-1.0.0rc1/tests/test_tail_parity.py +329 -0
- midstream_platform-1.0.0rc1/tests/test_tail_routing.py +265 -0
- midstream_platform-1.0.0rc1/tests/test_tail_security.py +230 -0
- midstream_platform-1.0.0rc1/tests/test_topic_catalog.py +576 -0
- midstream_platform-1.0.0rc1/tests/test_topic_changes.py +247 -0
- midstream_platform-1.0.0rc1/tests/test_versioning.py +164 -0
- midstream_platform-1.0.0rc1/tests/test_web.py +1681 -0
- midstream_platform-1.0.0rc1/tests/test_workflow.py +763 -0
- midstream_platform-1.0.0rc1/tests/test_workflow_topic_changes.py +527 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: midstream-platform
|
|
3
|
+
Version: 1.0.0rc1
|
|
4
|
+
Summary: Midstream: a streaming platform on Kaspr that materializes Kafka topics into Iceberg tables
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/vmirz/midstream
|
|
7
|
+
Project-URL: Repository, https://github.com/vmirz/midstream
|
|
8
|
+
Project-URL: Documentation, https://github.com/vmirz/midstream/tree/main/docs/handbook
|
|
9
|
+
Project-URL: Changelog, https://github.com/vmirz/midstream/releases
|
|
10
|
+
Keywords: kafka,iceberg,kaspr,streaming,lakehouse
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
Requires-Dist: jinja2>=3.1.0
|
|
21
|
+
Requires-Dist: google-cloud-storage>=2.18.0
|
|
22
|
+
Requires-Dist: kubernetes-asyncio>=33.1.0
|
|
23
|
+
Requires-Dist: pyarrow>=18.1.0
|
|
24
|
+
Requires-Dist: pyiceberg[pyarrow]<0.13,>=0.12
|
|
25
|
+
Requires-Dist: pyyaml>=6.0.0
|
|
26
|
+
Requires-Dist: tzdata>=2024.1
|
|
27
|
+
|
|
28
|
+
# midstream-platform
|
|
29
|
+
|
|
30
|
+
The Python package of [Midstream](https://github.com/vmirz/midstream), a streaming platform on
|
|
31
|
+
[Kaspr](https://kaspr.dev) that materializes Kafka topics into Apache Iceberg tables, exactly once,
|
|
32
|
+
with schemas that evolve by themselves, an Iceberg REST catalog, accounts and permissions, and a web
|
|
33
|
+
console.
|
|
34
|
+
|
|
35
|
+
One distribution holds every server-side part: the control plane, the per-materialization runtime,
|
|
36
|
+
the maintenance and tail apps, and the Iceberg library they share. It is not a library to import into
|
|
37
|
+
your own code: Kaspr applications install it in their pods, and the Helm chart in the repository
|
|
38
|
+
deploys them. See the repository's `docs/install.md` to run it, and `docs/handbook/` for how it works.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# midstream-platform
|
|
2
|
+
|
|
3
|
+
The Python package of [Midstream](https://github.com/vmirz/midstream), a streaming platform on
|
|
4
|
+
[Kaspr](https://kaspr.dev) that materializes Kafka topics into Apache Iceberg tables, exactly once,
|
|
5
|
+
with schemas that evolve by themselves, an Iceberg REST catalog, accounts and permissions, and a web
|
|
6
|
+
console.
|
|
7
|
+
|
|
8
|
+
One distribution holds every server-side part: the control plane, the per-materialization runtime,
|
|
9
|
+
the maintenance and tail apps, and the Iceberg library they share. It is not a library to import into
|
|
10
|
+
your own code: Kaspr applications install it in their pods, and the Helm chart in the repository
|
|
11
|
+
deploys them. See the repository's `docs/install.md` to run it, and `docs/handbook/` for how it works.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "midstream-platform"
|
|
7
|
+
version = "1.0.0rc1"
|
|
8
|
+
description = "Midstream: a streaming platform on Kaspr that materializes Kafka topics into Iceberg tables"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
requires-python = ">=3.12"
|
|
12
|
+
keywords = ["kafka", "iceberg", "kaspr", "streaming", "lakehouse"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 4 - Beta",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Topic :: System :: Distributed Computing",
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"jinja2>=3.1.0",
|
|
24
|
+
"google-cloud-storage>=2.18.0",
|
|
25
|
+
"kubernetes-asyncio>=33.1.0",
|
|
26
|
+
"pyarrow>=18.1.0",
|
|
27
|
+
# The minor is pinned: the committer uses documented-but-internal update APIs (lakehouse/catalog.py
|
|
28
|
+
# `commit_once`, lakehouse/commits.py). The tests exercise them, so a new minor is a deliberate bump.
|
|
29
|
+
"pyiceberg[pyarrow]>=0.12,<0.13",
|
|
30
|
+
"pyyaml>=6.0.0",
|
|
31
|
+
"tzdata>=2024.1",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[tool.setuptools]
|
|
35
|
+
package-dir = {"" = "src"}
|
|
36
|
+
|
|
37
|
+
[tool.setuptools.packages.find]
|
|
38
|
+
where = ["src"]
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.package-data]
|
|
41
|
+
"midstream.runtime_app" = ["templates/*.j2"]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/vmirz/midstream"
|
|
45
|
+
Repository = "https://github.com/vmirz/midstream"
|
|
46
|
+
Documentation = "https://github.com/vmirz/midstream/tree/main/docs/handbook"
|
|
47
|
+
Changelog = "https://github.com/vmirz/midstream/releases"
|