memcove 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memcove-0.3.4/.dockerignore +31 -0
- memcove-0.3.4/.env.example +79 -0
- memcove-0.3.4/.github/ISSUE_TEMPLATE/bug_report.yml +40 -0
- memcove-0.3.4/.github/ISSUE_TEMPLATE/config.yml +8 -0
- memcove-0.3.4/.github/ISSUE_TEMPLATE/feature_request.yml +24 -0
- memcove-0.3.4/.github/PULL_REQUEST_TEMPLATE.md +23 -0
- memcove-0.3.4/.github/workflows/ci.yml +91 -0
- memcove-0.3.4/.github/workflows/docs.yml +56 -0
- memcove-0.3.4/.github/workflows/release.yml +128 -0
- memcove-0.3.4/.gitignore +29 -0
- memcove-0.3.4/CHANGELOG.md +125 -0
- memcove-0.3.4/CODE_OF_CONDUCT.md +125 -0
- memcove-0.3.4/CONTRIBUTING.md +54 -0
- memcove-0.3.4/Dockerfile +57 -0
- memcove-0.3.4/LICENSE +202 -0
- memcove-0.3.4/PKG-INFO +150 -0
- memcove-0.3.4/README.md +111 -0
- memcove-0.3.4/SECURITY.md +36 -0
- memcove-0.3.4/TODOS.md +44 -0
- memcove-0.3.4/deploy/networkpolicy.example.yaml +30 -0
- memcove-0.3.4/deploy/values.example.yaml +49 -0
- memcove-0.3.4/docker-compose.yml +109 -0
- memcove-0.3.4/docs/changelog.md +3 -0
- memcove-0.3.4/docs/concepts/architecture.md +123 -0
- memcove-0.3.4/docs/concepts/isolation.md +73 -0
- memcove-0.3.4/docs/concepts/security.md +77 -0
- memcove-0.3.4/docs/configuration/auth.md +63 -0
- memcove-0.3.4/docs/configuration/local-dev.md +57 -0
- memcove-0.3.4/docs/configuration/settings.md +111 -0
- memcove-0.3.4/docs/deployment/checklist.md +57 -0
- memcove-0.3.4/docs/deployment/kubernetes.md +44 -0
- memcove-0.3.4/docs/getting-started/connecting.md +89 -0
- memcove-0.3.4/docs/getting-started/quickstart.md +74 -0
- memcove-0.3.4/docs/getting-started/walkthrough.md +109 -0
- memcove-0.3.4/docs/index.md +92 -0
- memcove-0.3.4/docs/reference/cli.md +37 -0
- memcove-0.3.4/docs/requirements.txt +4 -0
- memcove-0.3.4/docs/tools/deleting.md +22 -0
- memcove-0.3.4/docs/tools/exporting.md +39 -0
- memcove-0.3.4/docs/tools/index.md +43 -0
- memcove-0.3.4/docs/tools/querying.md +70 -0
- memcove-0.3.4/docs/tools/reading.md +82 -0
- memcove-0.3.4/docs/tools/resources.md +23 -0
- memcove-0.3.4/docs/tools/storing.md +89 -0
- memcove-0.3.4/docs/tools/streaming.md +78 -0
- memcove-0.3.4/infra/trino/etc/catalog/iceberg.properties +13 -0
- memcove-0.3.4/mkdocs.yml +105 -0
- memcove-0.3.4/pyproject.toml +70 -0
- memcove-0.3.4/scripts/agent_demo.py +254 -0
- memcove-0.3.4/scripts/flight_smoke.py +63 -0
- memcove-0.3.4/scripts/pipeline_demo.py +330 -0
- memcove-0.3.4/scripts/smoke.py +128 -0
- memcove-0.3.4/src/memcove/__init__.py +3 -0
- memcove-0.3.4/src/memcove/core/__init__.py +0 -0
- memcove-0.3.4/src/memcove/core/arrow_io.py +35 -0
- memcove-0.3.4/src/memcove/core/audit.py +27 -0
- memcove-0.3.4/src/memcove/core/catalog.py +183 -0
- memcove-0.3.4/src/memcove/core/config.py +121 -0
- memcove-0.3.4/src/memcove/core/errors.py +40 -0
- memcove-0.3.4/src/memcove/core/models.py +92 -0
- memcove-0.3.4/src/memcove/core/naming.py +20 -0
- memcove-0.3.4/src/memcove/core/registry.py +344 -0
- memcove-0.3.4/src/memcove/core/sql_guard.py +174 -0
- memcove-0.3.4/src/memcove/core/storage.py +100 -0
- memcove-0.3.4/src/memcove/core/tenancy.py +88 -0
- memcove-0.3.4/src/memcove/core/trino_client.py +97 -0
- memcove-0.3.4/src/memcove/data_plane/__init__.py +0 -0
- memcove-0.3.4/src/memcove/data_plane/flight_server.py +141 -0
- memcove-0.3.4/src/memcove/data_plane/tickets.py +108 -0
- memcove-0.3.4/src/memcove/reconciler.py +202 -0
- memcove-0.3.4/src/memcove/server.py +455 -0
- memcove-0.3.4/src/memcove/tools/__init__.py +0 -0
- memcove-0.3.4/src/memcove/tools/artifacts.py +72 -0
- memcove-0.3.4/src/memcove/tools/derive.py +92 -0
- memcove-0.3.4/src/memcove/tools/discovery.py +42 -0
- memcove-0.3.4/src/memcove/tools/ingest.py +120 -0
- memcove-0.3.4/src/memcove/tools/objects.py +175 -0
- memcove-0.3.4/src/memcove/tools/query.py +28 -0
- memcove-0.3.4/tests/__init__.py +0 -0
- memcove-0.3.4/tests/test_audit.py +28 -0
- memcove-0.3.4/tests/test_catalog_schema.py +84 -0
- memcove-0.3.4/tests/test_discovery.py +51 -0
- memcove-0.3.4/tests/test_flight.py +76 -0
- memcove-0.3.4/tests/test_health.py +54 -0
- memcove-0.3.4/tests/test_ingest_allowlist.py +37 -0
- memcove-0.3.4/tests/test_integration.py +146 -0
- memcove-0.3.4/tests/test_naming_tenancy.py +40 -0
- memcove-0.3.4/tests/test_objects_repair.py +60 -0
- memcove-0.3.4/tests/test_reconciler.py +128 -0
- memcove-0.3.4/tests/test_registry_guarded.py +99 -0
- memcove-0.3.4/tests/test_registry_pool.py +72 -0
- memcove-0.3.4/tests/test_sql_guard.py +149 -0
- memcove-0.3.4/tests/test_tenancy.py +60 -0
- memcove-0.3.4/tests/test_tickets.py +44 -0
- memcove-0.3.4/tests/test_trino_principal.py +25 -0
- memcove-0.3.4/uv.lock +2278 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Keep the build context small — only pyproject/lock, src, README, LICENSE are used.
|
|
2
|
+
.git
|
|
3
|
+
.github
|
|
4
|
+
.venv
|
|
5
|
+
.env
|
|
6
|
+
.env.example
|
|
7
|
+
.ruff_cache
|
|
8
|
+
.pytest_cache
|
|
9
|
+
__pycache__
|
|
10
|
+
*.py[oc]
|
|
11
|
+
build
|
|
12
|
+
dist
|
|
13
|
+
wheels
|
|
14
|
+
*.egg-info
|
|
15
|
+
|
|
16
|
+
# Not needed at runtime
|
|
17
|
+
tests
|
|
18
|
+
scripts
|
|
19
|
+
docs
|
|
20
|
+
site
|
|
21
|
+
infra
|
|
22
|
+
deploy
|
|
23
|
+
internal
|
|
24
|
+
*.md
|
|
25
|
+
!README.md
|
|
26
|
+
docker-compose.yml
|
|
27
|
+
Dockerfile
|
|
28
|
+
.dockerignore
|
|
29
|
+
mkdocs.yml
|
|
30
|
+
CHANGELOG.md
|
|
31
|
+
ROADMAP.md
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# --- Memcove configuration (copy to .env) ---
|
|
2
|
+
|
|
3
|
+
# MCP server
|
|
4
|
+
MEMCOVE_HOST=0.0.0.0
|
|
5
|
+
MEMCOVE_PORT=8090
|
|
6
|
+
|
|
7
|
+
# Iceberg REST catalog
|
|
8
|
+
MEMCOVE_ICEBERG_REST_URI=http://localhost:8181
|
|
9
|
+
MEMCOVE_ICEBERG_WAREHOUSE=s3://warehouse/
|
|
10
|
+
MEMCOVE_ICEBERG_CATALOG_NAME=memcove
|
|
11
|
+
|
|
12
|
+
# Object store (MinIO locally; real S3 in prod)
|
|
13
|
+
MEMCOVE_S3_ENDPOINT=http://localhost:9000
|
|
14
|
+
MEMCOVE_S3_REGION=us-east-1
|
|
15
|
+
MEMCOVE_S3_ACCESS_KEY=minio
|
|
16
|
+
MEMCOVE_S3_SECRET_KEY=minio12345
|
|
17
|
+
MEMCOVE_S3_PATH_STYLE=true
|
|
18
|
+
# Buckets / prefixes used by the data plane
|
|
19
|
+
MEMCOVE_WAREHOUSE_BUCKET=warehouse
|
|
20
|
+
MEMCOVE_STAGING_BUCKET=memcove-staging
|
|
21
|
+
MEMCOVE_ARTIFACTS_BUCKET=memcove-artifacts
|
|
22
|
+
|
|
23
|
+
# Trino (read / derive / export engine)
|
|
24
|
+
MEMCOVE_TRINO_HOST=localhost
|
|
25
|
+
MEMCOVE_TRINO_PORT=8080
|
|
26
|
+
MEMCOVE_TRINO_USER=memcove # service principal (connect identity when not impersonating)
|
|
27
|
+
MEMCOVE_TRINO_CATALOG=iceberg
|
|
28
|
+
MEMCOVE_TRINO_HTTP_SCHEME=http # set https for a TLS-fronted Trino
|
|
29
|
+
# Defense-in-depth: connect to Trino AS the caller's tenant so the operator's own
|
|
30
|
+
# Trino access control applies per tenant. Requires impersonation rights on the
|
|
31
|
+
# service principal + a configured grant backend. Off = single identity (dev).
|
|
32
|
+
MEMCOVE_TRINO_IMPERSONATION=false
|
|
33
|
+
# Session properties applied to every data connection (resource caps etc.) — generic
|
|
34
|
+
# passthrough; set whatever your Trino version supports. JSON object.
|
|
35
|
+
MEMCOVE_TRINO_SESSION_PROPERTIES={}
|
|
36
|
+
# e.g. {"query_max_run_time":"60s","query_max_scan_physical_bytes":"10GB"}
|
|
37
|
+
|
|
38
|
+
# Arrow Flight streaming data plane (M3)
|
|
39
|
+
MEMCOVE_FLIGHT_HOST=0.0.0.0
|
|
40
|
+
MEMCOVE_FLIGHT_PORT=8815
|
|
41
|
+
MEMCOVE_FLIGHT_ADVERTISE_URI=grpc://localhost:8815
|
|
42
|
+
# HMAC secret for signing Flight tickets/descriptors so a client cannot forge one for
|
|
43
|
+
# another tenant. MUST be overridden in any real deployment (default is insecure).
|
|
44
|
+
MEMCOVE_FLIGHT_TICKET_SECRET=dev-insecure-change-me
|
|
45
|
+
MEMCOVE_FLIGHT_TICKET_TTL_SECONDS=300 # signed tickets expire after this many seconds
|
|
46
|
+
|
|
47
|
+
# Postgres control-plane registry (host port 5433 avoids clashing with a local pg on 5432)
|
|
48
|
+
MEMCOVE_PG_DSN=postgresql://memcove:memcove@localhost:5433/memcove
|
|
49
|
+
# Connection pool sizing (min kept warm, grows to max under load; timeout in seconds
|
|
50
|
+
# to wait for a free connection before erroring)
|
|
51
|
+
MEMCOVE_PG_POOL_MIN_SIZE=1
|
|
52
|
+
MEMCOVE_PG_POOL_MAX_SIZE=10
|
|
53
|
+
MEMCOVE_PG_POOL_TIMEOUT=10.0
|
|
54
|
+
|
|
55
|
+
# Guardrails
|
|
56
|
+
MEMCOVE_PREVIEW_ROW_CAP=1000
|
|
57
|
+
MEMCOVE_INLINE_BYTES_CAP=8388608 # 8 MiB max for inline ingest
|
|
58
|
+
MEMCOVE_EXPORT_ROW_CAP=5000000 # safety cap for artifact export
|
|
59
|
+
MEMCOVE_PRESIGN_TTL_SECONDS=3600
|
|
60
|
+
|
|
61
|
+
# Tenancy. Default: trust a tenant header set by your auth proxy.
|
|
62
|
+
MEMCOVE_TENANT_HEADER=x-memcove-tenant
|
|
63
|
+
MEMCOVE_DEFAULT_TENANT=default
|
|
64
|
+
# Provisioning map (optional). Set a subject/group header and Memcove maps the verified
|
|
65
|
+
# identity -> internal tenant id, instead of trusting a raw tenant header. This is how
|
|
66
|
+
# you avoid using a raw OIDC sub as a namespace. Leave subject header empty to disable.
|
|
67
|
+
# Fail-closed: once the subject header is set, an unmapped identity is REJECTED (it never
|
|
68
|
+
# falls back to the raw tenant header).
|
|
69
|
+
MEMCOVE_TENANT_SUBJECT_HEADER= # e.g. x-auth-subject
|
|
70
|
+
MEMCOVE_TENANT_GROUP_HEADER= # e.g. x-auth-groups (comma-separated)
|
|
71
|
+
MEMCOVE_TENANT_MAP={} # JSON: {"oidc|abc":"acme","team-research":"research"}
|
|
72
|
+
|
|
73
|
+
# Shared read-only reference plane (the gateway). Schemas every tenant may SELECT
|
|
74
|
+
# from but none may write; not rewritten to the caller's namespace. JSON list.
|
|
75
|
+
MEMCOVE_SHARED_SCHEMAS=["ref_market"]
|
|
76
|
+
|
|
77
|
+
# Ingest allowlist for agent-supplied s3_parquet URIs. Empty = feature DISABLED
|
|
78
|
+
# (fail closed). JSON list of permitted s3:// prefixes.
|
|
79
|
+
MEMCOVE_ALLOWED_S3_INGEST_PREFIXES=[]
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Report a problem with Memcove
|
|
3
|
+
labels: ["bug"]
|
|
4
|
+
body:
|
|
5
|
+
- type: markdown
|
|
6
|
+
attributes:
|
|
7
|
+
value: |
|
|
8
|
+
Thanks for taking the time to file a bug! For **security vulnerabilities**,
|
|
9
|
+
do not use this form — see [SECURITY.md](../blob/main/SECURITY.md).
|
|
10
|
+
- type: textarea
|
|
11
|
+
id: what-happened
|
|
12
|
+
attributes:
|
|
13
|
+
label: What happened?
|
|
14
|
+
description: A clear description of the bug, including what you expected instead.
|
|
15
|
+
validations:
|
|
16
|
+
required: true
|
|
17
|
+
- type: textarea
|
|
18
|
+
id: repro
|
|
19
|
+
attributes:
|
|
20
|
+
label: Steps to reproduce
|
|
21
|
+
description: Minimal steps, the MCP tool / SQL / config involved, and any error output.
|
|
22
|
+
placeholder: |
|
|
23
|
+
1. …
|
|
24
|
+
2. …
|
|
25
|
+
validations:
|
|
26
|
+
required: true
|
|
27
|
+
- type: input
|
|
28
|
+
id: version
|
|
29
|
+
attributes:
|
|
30
|
+
label: Memcove version / commit
|
|
31
|
+
placeholder: "0.3.2 or a commit SHA"
|
|
32
|
+
validations:
|
|
33
|
+
required: true
|
|
34
|
+
- type: textarea
|
|
35
|
+
id: environment
|
|
36
|
+
attributes:
|
|
37
|
+
label: Environment
|
|
38
|
+
description: How you're running it (local compose, Kubernetes, …), Python version, and which registry DB / Trino setup.
|
|
39
|
+
validations:
|
|
40
|
+
required: false
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
blank_issues_enabled: false
|
|
2
|
+
contact_links:
|
|
3
|
+
- name: Security vulnerability
|
|
4
|
+
url: https://github.com/memcove/memcove/security/advisories/new
|
|
5
|
+
about: Please report security issues privately, not as public issues.
|
|
6
|
+
- name: Documentation
|
|
7
|
+
url: https://memcove.github.io/memcove/
|
|
8
|
+
about: Check the docs before filing — your question may be answered there.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: Suggest an idea or enhancement for Memcove
|
|
3
|
+
labels: ["enhancement"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: problem
|
|
7
|
+
attributes:
|
|
8
|
+
label: Problem / motivation
|
|
9
|
+
description: What are you trying to do, and what's getting in the way?
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: proposal
|
|
14
|
+
attributes:
|
|
15
|
+
label: Proposed solution
|
|
16
|
+
description: What would you like to see? Sketch the tool, config, or behavior.
|
|
17
|
+
validations:
|
|
18
|
+
required: true
|
|
19
|
+
- type: textarea
|
|
20
|
+
id: alternatives
|
|
21
|
+
attributes:
|
|
22
|
+
label: Alternatives considered
|
|
23
|
+
validations:
|
|
24
|
+
required: false
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
## Summary
|
|
2
|
+
|
|
3
|
+
<!-- What does this change and why? -->
|
|
4
|
+
|
|
5
|
+
## Changes
|
|
6
|
+
|
|
7
|
+
<!-- Bullet the notable changes. -->
|
|
8
|
+
|
|
9
|
+
-
|
|
10
|
+
|
|
11
|
+
## Testing
|
|
12
|
+
|
|
13
|
+
<!-- How did you verify this? -->
|
|
14
|
+
|
|
15
|
+
- [ ] `uv run ruff check .` passes
|
|
16
|
+
- [ ] `uv run pytest -m "not integration"` passes
|
|
17
|
+
- [ ] `uv run pytest -m integration` passes (if the change touches the data/registry path)
|
|
18
|
+
|
|
19
|
+
## Checklist
|
|
20
|
+
|
|
21
|
+
- [ ] Conventional Commit title (`feat:` / `fix:` / `chore:` / `docs:` …)
|
|
22
|
+
- [ ] `CHANGELOG.md` updated (and `pyproject.toml` version bumped) if user-facing
|
|
23
|
+
- [ ] Docs updated if behavior or configuration changed
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
# Lint + tests on every PR and on pushes to main. The docs site has its own
|
|
4
|
+
# workflow (docs.yml); this one gates code quality.
|
|
5
|
+
on:
|
|
6
|
+
pull_request:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
14
|
+
# A newer push to the same ref supersedes an in-flight run.
|
|
15
|
+
concurrency:
|
|
16
|
+
group: ci-${{ github.ref }}
|
|
17
|
+
cancel-in-progress: true
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
lint:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
- uses: astral-sh/setup-uv@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.12"
|
|
27
|
+
enable-cache: true
|
|
28
|
+
- name: Install (frozen — uses the committed uv.lock)
|
|
29
|
+
run: uv sync --extra dev --frozen
|
|
30
|
+
- name: Ruff lint
|
|
31
|
+
run: uv run ruff check .
|
|
32
|
+
# NOTE: `ruff format --check` is intentionally not gated yet — the codebase
|
|
33
|
+
# uses a compact hand-formatting style that predates ruff-format adoption.
|
|
34
|
+
# Add it here after a dedicated one-shot `ruff format` normalization commit.
|
|
35
|
+
|
|
36
|
+
test:
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v4
|
|
40
|
+
- uses: astral-sh/setup-uv@v5
|
|
41
|
+
with:
|
|
42
|
+
python-version: "3.12"
|
|
43
|
+
enable-cache: true
|
|
44
|
+
- name: Install (frozen)
|
|
45
|
+
run: uv sync --extra dev --frozen
|
|
46
|
+
- name: Unit tests
|
|
47
|
+
run: uv run pytest -m "not integration" -q
|
|
48
|
+
|
|
49
|
+
integration:
|
|
50
|
+
runs-on: ubuntu-latest
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v4
|
|
53
|
+
- uses: astral-sh/setup-uv@v5
|
|
54
|
+
with:
|
|
55
|
+
python-version: "3.12"
|
|
56
|
+
enable-cache: true
|
|
57
|
+
- name: Install (frozen)
|
|
58
|
+
run: uv sync --extra dev --frozen
|
|
59
|
+
- name: Bring up the local lakehouse
|
|
60
|
+
run: docker compose up -d --wait
|
|
61
|
+
- name: Integration tests
|
|
62
|
+
run: uv run pytest -m integration -q
|
|
63
|
+
- name: Tear down
|
|
64
|
+
if: always()
|
|
65
|
+
run: docker compose down -v
|
|
66
|
+
|
|
67
|
+
docker-build:
|
|
68
|
+
# Exercise the Dockerfile on every PR (build only, no push) so image breakage
|
|
69
|
+
# is caught here rather than at release time.
|
|
70
|
+
runs-on: ubuntu-latest
|
|
71
|
+
steps:
|
|
72
|
+
- uses: actions/checkout@v4
|
|
73
|
+
- uses: docker/setup-buildx-action@v3
|
|
74
|
+
- name: Build image (amd64, no push)
|
|
75
|
+
uses: docker/build-push-action@v6
|
|
76
|
+
with:
|
|
77
|
+
context: .
|
|
78
|
+
push: false
|
|
79
|
+
load: true
|
|
80
|
+
tags: memcove:ci
|
|
81
|
+
cache-from: type=gha
|
|
82
|
+
cache-to: type=gha,mode=max
|
|
83
|
+
- name: Smoke-test /health
|
|
84
|
+
run: |
|
|
85
|
+
docker run -d --name memcove-ci -p 8090:8090 memcove:ci
|
|
86
|
+
for i in $(seq 1 20); do
|
|
87
|
+
if curl -sf http://localhost:8090/health >/dev/null; then echo "healthy"; break; fi
|
|
88
|
+
sleep 1
|
|
89
|
+
done
|
|
90
|
+
curl -sf http://localhost:8090/health | grep -q ok
|
|
91
|
+
docker rm -f memcove-ci
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: Deploy docs
|
|
2
|
+
|
|
3
|
+
# Build the mkdocs-material site and publish it to GitHub Pages on every push to
|
|
4
|
+
# main that touches the docs. Requires the repo's Pages source to be set to
|
|
5
|
+
# "GitHub Actions" (Settings -> Pages -> Build and deployment -> Source).
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
paths:
|
|
10
|
+
- "docs/**"
|
|
11
|
+
- "mkdocs.yml"
|
|
12
|
+
- ".github/workflows/docs.yml"
|
|
13
|
+
workflow_dispatch:
|
|
14
|
+
|
|
15
|
+
# Least-privilege token: read the repo, publish to Pages via OIDC.
|
|
16
|
+
permissions:
|
|
17
|
+
contents: read
|
|
18
|
+
pages: write
|
|
19
|
+
id-token: write
|
|
20
|
+
|
|
21
|
+
# Never let two Pages deploys race; the newest push wins.
|
|
22
|
+
concurrency:
|
|
23
|
+
group: github-pages
|
|
24
|
+
cancel-in-progress: false
|
|
25
|
+
|
|
26
|
+
jobs:
|
|
27
|
+
build:
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@v4
|
|
31
|
+
- uses: actions/setup-python@v5
|
|
32
|
+
with:
|
|
33
|
+
python-version: "3.12"
|
|
34
|
+
cache: pip
|
|
35
|
+
- name: Install docs toolchain
|
|
36
|
+
run: pip install -r docs/requirements.txt
|
|
37
|
+
- name: Build site (strict — fails on broken links or missing nav)
|
|
38
|
+
run: mkdocs build --strict
|
|
39
|
+
- uses: actions/configure-pages@v5
|
|
40
|
+
with:
|
|
41
|
+
# Create/enable the Pages site (build type = GitHub Actions) if it isn't
|
|
42
|
+
# already, so the first deploy doesn't fail with "Get Pages site failed".
|
|
43
|
+
enablement: true
|
|
44
|
+
- uses: actions/upload-pages-artifact@v3
|
|
45
|
+
with:
|
|
46
|
+
path: ./site
|
|
47
|
+
|
|
48
|
+
deploy:
|
|
49
|
+
needs: build
|
|
50
|
+
runs-on: ubuntu-latest
|
|
51
|
+
environment:
|
|
52
|
+
name: github-pages
|
|
53
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
54
|
+
steps:
|
|
55
|
+
- id: deployment
|
|
56
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Cut a release on a semver tag (e.g. `git tag v0.3.4 && git push --tags`):
|
|
4
|
+
# - build a multi-arch image and push to Docker Hub + GHCR
|
|
5
|
+
# - publish the wheel to PyPI (trusted publishing / OIDC)
|
|
6
|
+
# - create a GitHub Release from the matching CHANGELOG section
|
|
7
|
+
#
|
|
8
|
+
# Required repo config:
|
|
9
|
+
# - secrets DOCKERHUB_USERNAME + DOCKERHUB_TOKEN (Docker Hub push)
|
|
10
|
+
# - a PyPI "trusted publisher" for this repo/workflow (no token needed)
|
|
11
|
+
# GHCR uses the built-in GITHUB_TOKEN.
|
|
12
|
+
on:
|
|
13
|
+
push:
|
|
14
|
+
tags: ["v*"]
|
|
15
|
+
workflow_dispatch:
|
|
16
|
+
inputs:
|
|
17
|
+
tag:
|
|
18
|
+
description: "Existing tag to (re)release, e.g. v0.3.4"
|
|
19
|
+
required: true
|
|
20
|
+
|
|
21
|
+
permissions:
|
|
22
|
+
contents: write # create the GitHub Release
|
|
23
|
+
packages: write # push to GHCR
|
|
24
|
+
id-token: write # PyPI trusted publishing
|
|
25
|
+
|
|
26
|
+
env:
|
|
27
|
+
# Docker Hub repo lives under the maintainer's personal namespace.
|
|
28
|
+
DOCKERHUB_IMAGE: andrzejgluszynski/memcove
|
|
29
|
+
GHCR_IMAGE: ghcr.io/memcove/memcove
|
|
30
|
+
|
|
31
|
+
jobs:
|
|
32
|
+
image:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v4
|
|
36
|
+
with:
|
|
37
|
+
ref: ${{ github.event.inputs.tag || github.ref }}
|
|
38
|
+
|
|
39
|
+
- uses: docker/setup-qemu-action@v3
|
|
40
|
+
- uses: docker/setup-buildx-action@v3
|
|
41
|
+
|
|
42
|
+
- name: Log in to Docker Hub
|
|
43
|
+
uses: docker/login-action@v3
|
|
44
|
+
with:
|
|
45
|
+
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
46
|
+
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
47
|
+
|
|
48
|
+
- name: Log in to GHCR
|
|
49
|
+
uses: docker/login-action@v3
|
|
50
|
+
with:
|
|
51
|
+
registry: ghcr.io
|
|
52
|
+
username: ${{ github.actor }}
|
|
53
|
+
password: ${{ secrets.GITHUB_TOKEN }}
|
|
54
|
+
|
|
55
|
+
- name: Derive version from tag
|
|
56
|
+
id: version
|
|
57
|
+
run: |
|
|
58
|
+
REF="${{ github.event.inputs.tag || github.ref_name }}"
|
|
59
|
+
V="${REF#v}"
|
|
60
|
+
echo "value=$V" >> "$GITHUB_OUTPUT"
|
|
61
|
+
echo "minor=${V%.*}" >> "$GITHUB_OUTPUT" # 0.3.4 -> 0.3
|
|
62
|
+
|
|
63
|
+
- name: Image metadata (tags + labels)
|
|
64
|
+
id: meta
|
|
65
|
+
uses: docker/metadata-action@v5
|
|
66
|
+
with:
|
|
67
|
+
images: |
|
|
68
|
+
${{ env.DOCKERHUB_IMAGE }}
|
|
69
|
+
${{ env.GHCR_IMAGE }}
|
|
70
|
+
# Derive tags from the explicit version (not the git ref) so this works
|
|
71
|
+
# on both a tag push and a manual workflow_dispatch re-run.
|
|
72
|
+
tags: |
|
|
73
|
+
type=raw,value=${{ steps.version.outputs.value }}
|
|
74
|
+
type=raw,value=${{ steps.version.outputs.minor }}
|
|
75
|
+
type=raw,value=latest
|
|
76
|
+
|
|
77
|
+
- name: Build and push (amd64 + arm64)
|
|
78
|
+
uses: docker/build-push-action@v6
|
|
79
|
+
with:
|
|
80
|
+
context: .
|
|
81
|
+
platforms: linux/amd64,linux/arm64
|
|
82
|
+
push: true
|
|
83
|
+
tags: ${{ steps.meta.outputs.tags }}
|
|
84
|
+
labels: ${{ steps.meta.outputs.labels }}
|
|
85
|
+
build-args: MEMCOVE_VERSION=${{ steps.version.outputs.value }}
|
|
86
|
+
cache-from: type=gha
|
|
87
|
+
cache-to: type=gha,mode=max
|
|
88
|
+
|
|
89
|
+
pypi:
|
|
90
|
+
runs-on: ubuntu-latest
|
|
91
|
+
environment: pypi
|
|
92
|
+
steps:
|
|
93
|
+
- uses: actions/checkout@v4
|
|
94
|
+
with:
|
|
95
|
+
ref: ${{ github.event.inputs.tag || github.ref }}
|
|
96
|
+
- uses: astral-sh/setup-uv@v5
|
|
97
|
+
with:
|
|
98
|
+
python-version: "3.12"
|
|
99
|
+
- name: Build sdist + wheel
|
|
100
|
+
run: uv build
|
|
101
|
+
- name: Publish to PyPI
|
|
102
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
103
|
+
|
|
104
|
+
github-release:
|
|
105
|
+
runs-on: ubuntu-latest
|
|
106
|
+
needs: [image, pypi]
|
|
107
|
+
steps:
|
|
108
|
+
- uses: actions/checkout@v4
|
|
109
|
+
with:
|
|
110
|
+
ref: ${{ github.event.inputs.tag || github.ref }}
|
|
111
|
+
- name: Extract this version's changelog section
|
|
112
|
+
id: notes
|
|
113
|
+
run: |
|
|
114
|
+
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
|
|
115
|
+
VERSION="${VERSION#v}"
|
|
116
|
+
# Print lines from "## [VERSION]" up to (not including) the next "## [".
|
|
117
|
+
awk -v v="$VERSION" '
|
|
118
|
+
$0 ~ "^## \\[" v "\\]" {p=1; next}
|
|
119
|
+
p && /^## \[/ {exit}
|
|
120
|
+
p {print}
|
|
121
|
+
' CHANGELOG.md > release_notes.md
|
|
122
|
+
if [ ! -s release_notes.md ]; then echo "See CHANGELOG.md" > release_notes.md; fi
|
|
123
|
+
- name: Create GitHub Release
|
|
124
|
+
uses: softprops/action-gh-release@v2
|
|
125
|
+
with:
|
|
126
|
+
tag_name: ${{ github.event.inputs.tag || github.ref_name }}
|
|
127
|
+
name: ${{ github.event.inputs.tag || github.ref_name }}
|
|
128
|
+
body_path: release_notes.md
|
memcove-0.3.4/.gitignore
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Python-generated files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv
|
|
11
|
+
|
|
12
|
+
# Local config
|
|
13
|
+
.env
|
|
14
|
+
|
|
15
|
+
# Test / pytest caches
|
|
16
|
+
.pytest_cache/
|
|
17
|
+
|
|
18
|
+
# Claude setup
|
|
19
|
+
.claude/
|
|
20
|
+
|
|
21
|
+
# Internal docs
|
|
22
|
+
internal/
|
|
23
|
+
|
|
24
|
+
# macOS
|
|
25
|
+
.DS_Store
|
|
26
|
+
ROADMAP.md
|
|
27
|
+
|
|
28
|
+
# mkdocs build output
|
|
29
|
+
site/
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to Memcove are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/), and the project aims to follow
|
|
5
|
+
semantic versioning once it reaches 1.0.
|
|
6
|
+
|
|
7
|
+
## [0.3.4] - 2026-07-07
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- **Container image** — a multi-stage `Dockerfile` (non-root, `python:3.12-slim`,
|
|
11
|
+
deps installed frozen from `uv.lock`) shipping all three entrypoints
|
|
12
|
+
(`memcove-server` default, `memcove-flight`, `memcove-reconcile`). Plus a
|
|
13
|
+
`.dockerignore`.
|
|
14
|
+
- **`/health` and `/ready` HTTP endpoints** on the MCP server — liveness always
|
|
15
|
+
returns 200; readiness checks the metadata registry (`SELECT 1`) and Trino
|
|
16
|
+
reachability and returns 503 with per-check detail when a dependency is down.
|
|
17
|
+
Wire these to Kubernetes probes. Adds `registry.ping()`.
|
|
18
|
+
- **Release automation** — `.github/workflows/release.yml`: on a `v*` tag, builds a
|
|
19
|
+
multi-arch (amd64+arm64) image and pushes to Docker Hub + GHCR, publishes the wheel
|
|
20
|
+
to PyPI (trusted publishing), and creates a GitHub Release from the CHANGELOG. CI now
|
|
21
|
+
also builds the image and smoke-tests `/health` on every PR.
|
|
22
|
+
|
|
23
|
+
## [0.3.3] - 2026-07-07
|
|
24
|
+
|
|
25
|
+
### Added
|
|
26
|
+
- **Open-source foundations** — `LICENSE` (Apache-2.0), `CONTRIBUTING.md`,
|
|
27
|
+
`CODE_OF_CONDUCT.md`, `SECURITY.md`, and GitHub issue/PR templates. `pyproject.toml`
|
|
28
|
+
now declares the license, classifiers, and project URLs.
|
|
29
|
+
- **CI test/lint gate** — `.github/workflows/ci.yml` runs `ruff check` and the unit
|
|
30
|
+
suite on every PR, plus an integration job against the docker-compose stack. (The
|
|
31
|
+
existing `docs.yml` still handles the docs site.)
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
- **Reproducible installs** — `uv.lock` is now committed (was gitignored).
|
|
35
|
+
- **Local-stack reliability** — `docker-compose.yml` gives Trino a real healthcheck so
|
|
36
|
+
`docker compose up -d --wait` blocks until it's ready, replacing the "sleep ~20s"
|
|
37
|
+
guidance in the README and quickstart.
|
|
38
|
+
|
|
39
|
+
## [0.3.2] - 2026-07-06
|
|
40
|
+
|
|
41
|
+
### Changed
|
|
42
|
+
- **orjson for JSON serialization** — the Flight ticket codec (`data_plane/tickets.py`),
|
|
43
|
+
the structured audit log (`core/audit.py`), and the JSON artifact export
|
|
44
|
+
(`tools/artifacts.py`) now serialize/parse with `orjson` instead of the stdlib `json`
|
|
45
|
+
module. orjson was already installed transitively (via `trino`) and is now a declared
|
|
46
|
+
dependency. Signing is unaffected — HMAC canonicalization uses `orjson.OPT_SORT_KEYS`
|
|
47
|
+
on both the sign and verify paths. One behavior change: JSON exports now render
|
|
48
|
+
`datetime`/`UUID` values as native ISO 8601 (e.g. `2024-01-01T00:00:00`) rather than
|
|
49
|
+
Python `str()`; `Decimal`/`bytes` still fall through to `default=str` unchanged.
|
|
50
|
+
|
|
51
|
+
## [0.3.1] - 2026-07-05
|
|
52
|
+
|
|
53
|
+
### Changed
|
|
54
|
+
- **Pooled Postgres connections** — the registry now backs its connection helpers with
|
|
55
|
+
a lazily-opened `psycopg_pool` pool instead of opening a fresh connection per op
|
|
56
|
+
(the reconciler and read-repair had raised per-op churn). Connections are health-
|
|
57
|
+
checked on borrow, so a Postgres restart or idle drop no longer hands a dead
|
|
58
|
+
connection to the next caller. Sizing/timeout via `MEMCOVE_PG_POOL_{MIN_SIZE,MAX_SIZE,TIMEOUT}`.
|
|
59
|
+
Write-atomicity is unaffected: pool timeouts subclass `OperationalError`, so a
|
|
60
|
+
saturated/unreachable registry is still swallowed as `metadata_pending`.
|
|
61
|
+
|
|
62
|
+
## [0.3.0] - 2026-07-05
|
|
63
|
+
|
|
64
|
+
Write atomicity (M5). A crash or concurrent reader can no longer see a half-written
|
|
65
|
+
object, and the registry now self-heals when it drifts from the Iceberg catalog.
|
|
66
|
+
|
|
67
|
+
### Added
|
|
68
|
+
- **Atomic full-table replace** — `mode=replace` uses each engine's native atomic
|
|
69
|
+
swap (PyIceberg `overwrite()` for ingest/Flight, Trino `CREATE OR REPLACE TABLE`
|
|
70
|
+
for derive) instead of drop-then-create. A concurrent reader sees the old rows or
|
|
71
|
+
the new rows, never a missing table, and a mid-write crash no longer loses data.
|
|
72
|
+
- **Schema-compatibility guard** — replace and append reject a changed shape
|
|
73
|
+
(added/removed/retyped column) with `SchemaMismatchError` rather than silently
|
|
74
|
+
evolving it, so downstream SQL never breaks. `forget()` then `create()` to reshape.
|
|
75
|
+
- **Guarded registry write** — the object row and its lineage commit in one
|
|
76
|
+
transaction; if the registry write fails after the data is committed, the tool
|
|
77
|
+
returns a `metadata_pending` object and logs the drift instead of failing the write.
|
|
78
|
+
- **Synchronous read-repair** — `describe_object` on a registry miss backfills a
|
|
79
|
+
`reconciled` row inline when the table exists, so an orphaned object becomes
|
|
80
|
+
visible on the next read (the guarantee holds before the M7 reconcile cron).
|
|
81
|
+
- **Reconciler** (`memcove-reconcile`) — sweeps registry vs catalog and repairs both
|
|
82
|
+
directions. Deletion is deliberately timid: fail-safe on an empty listing (never
|
|
83
|
+
wipes a tenant on a transient catalog hiccup), a deletion cap with an absolute
|
|
84
|
+
floor, a two-sweep grace period, and a live re-check just before deleting.
|
|
85
|
+
|
|
86
|
+
### Changed
|
|
87
|
+
- Dev/CI Trino pinned to `431` (the minimum for `CREATE OR REPLACE TABLE` on the
|
|
88
|
+
Iceberg connector); operators bringing their own Trino need `>= 431`.
|
|
89
|
+
|
|
90
|
+
## [0.2.0] - 2026-07-04
|
|
91
|
+
|
|
92
|
+
The agent SQL gateway security core and its configuration seams. Every tenant
|
|
93
|
+
gets a private namespace plus a shared read-only reference plane, enforced by the
|
|
94
|
+
SQL guard and (optionally) the operator's own Trino access control.
|
|
95
|
+
|
|
96
|
+
### Added
|
|
97
|
+
- **Shared reference plane** — `MEMCOVE_SHARED_SCHEMAS` lets every tenant read
|
|
98
|
+
configured read-only schemas (e.g. `ref_market`) while writing only their own.
|
|
99
|
+
The SQL guard resolves shared schemas to themselves and rejects everything else.
|
|
100
|
+
- **Configurable Trino principal** — `MEMCOVE_TRINO_IMPERSONATION` connects to
|
|
101
|
+
Trino as the calling tenant so the operator's grant backend applies per tenant
|
|
102
|
+
(defense-in-depth beneath the guard). Off by default (single service identity).
|
|
103
|
+
- **Provisioning-map tenant resolution** — map a verified identity (subject/group)
|
|
104
|
+
to an internal tenant id via `MEMCOVE_TENANT_SUBJECT_HEADER` + `MEMCOVE_TENANT_MAP`
|
|
105
|
+
instead of trusting a raw OIDC `sub`. Fail-closed: unmapped identities are rejected.
|
|
106
|
+
- **Signed Arrow Flight tickets** — HMAC-signed, expiring tickets/descriptors so a
|
|
107
|
+
gRPC client cannot forge one for another tenant (`MEMCOVE_FLIGHT_TICKET_SECRET`).
|
|
108
|
+
- **`discover_reference_data` tool** — a curated view of the shared schemas, since
|
|
109
|
+
agents are denied `information_schema`.
|
|
110
|
+
- **Audit log** — structured JSON on the `memcove.audit` logger for every accepted
|
|
111
|
+
read/derive/export, plus configurable Trino resource caps
|
|
112
|
+
(`MEMCOVE_TRINO_SESSION_PROPERTIES`).
|
|
113
|
+
- **S3 ingest allowlist** — agent-supplied `s3_parquet` URIs must match
|
|
114
|
+
`MEMCOVE_ALLOWED_S3_INGEST_PREFIXES` (disabled/fail-closed by default).
|
|
115
|
+
- **Configuration guide + example deploy manifests** — `docs/CONFIGURATION.md`,
|
|
116
|
+
`deploy/networkpolicy.example.yaml`, `deploy/values.example.yaml`.
|
|
117
|
+
|
|
118
|
+
### Security
|
|
119
|
+
- SQL guard now denies metadata/enumeration schemas (`information_schema`,
|
|
120
|
+
`system`, `SHOW`/`DESCRIBE`/`EXPLAIN`), case-folds identifiers so a quoted
|
|
121
|
+
mixed-case name can't smuggle a foreign schema, and fails closed on any
|
|
122
|
+
FROM-item it can't classify (e.g. polymorphic `TABLE(...)` functions).
|
|
123
|
+
- `upload_handle` ingest is bound to the calling tenant's `uploads/{tenant}/` prefix.
|
|
124
|
+
- S3 allowlist matches on a path boundary (`s3://bucket` no longer permits
|
|
125
|
+
`s3://bucket-evil`).
|