rdst 0.1.185__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. rdst-0.1.185/.buildkite/build_package.sh +83 -0
  2. rdst-0.1.185/.buildkite/deploy_dev.sh +97 -0
  3. rdst-0.1.185/.buildkite/deploy_release.sh +253 -0
  4. rdst-0.1.185/.buildkite/pipeline.yml +122 -0
  5. rdst-0.1.185/.buildkite/run_integration_tests.sh +120 -0
  6. rdst-0.1.185/.buildkite/run_unit_tests.sh +27 -0
  7. rdst-0.1.185/.gitignore +33 -0
  8. rdst-0.1.185/CLAUDE.md +217 -0
  9. rdst-0.1.185/MIGRATION.md +179 -0
  10. rdst-0.1.185/PKG-INFO +155 -0
  11. rdst-0.1.185/README.md +116 -0
  12. rdst-0.1.185/_version.py +36 -0
  13. rdst-0.1.185/cleanup_mac_instance.sh +179 -0
  14. rdst-0.1.185/cleanup_old_mac_hosts.sh +123 -0
  15. rdst-0.1.185/copy_rdst_production.sh +20 -0
  16. rdst-0.1.185/deploy_mac_host_cleanup_lambda.sh +173 -0
  17. rdst-0.1.185/lib/__init__.py +4 -0
  18. rdst-0.1.185/lib/cache_manager/__init__.py +9 -0
  19. rdst-0.1.185/lib/cache_manager/cache_manager.py +835 -0
  20. rdst-0.1.185/lib/cli/__init__.py +29 -0
  21. rdst-0.1.185/lib/cli/analyze_command.py +2495 -0
  22. rdst-0.1.185/lib/cli/cache_command.py +306 -0
  23. rdst-0.1.185/lib/cli/configuration_wizard.py +1210 -0
  24. rdst-0.1.185/lib/cli/howdoi_command.py +497 -0
  25. rdst-0.1.185/lib/cli/init_command.py +547 -0
  26. rdst-0.1.185/lib/cli/interactive_mode.py +530 -0
  27. rdst-0.1.185/lib/cli/output_formatter.py +996 -0
  28. rdst-0.1.185/lib/cli/parameter_prompt.py +490 -0
  29. rdst-0.1.185/lib/cli/query_command.py +1271 -0
  30. rdst-0.1.185/lib/cli/rdst_cli.py +1180 -0
  31. rdst-0.1.185/lib/cli/rdstdbg_command.py +313 -0
  32. rdst-0.1.185/lib/cli/readyset_setup.py +363 -0
  33. rdst-0.1.185/lib/cli/report_command.py +492 -0
  34. rdst-0.1.185/lib/cli/run_workflow.py +323 -0
  35. rdst-0.1.185/lib/cli/schema_command.py +929 -0
  36. rdst-0.1.185/lib/cli/top.py +885 -0
  37. rdst-0.1.185/lib/data_manager/__init__.py +2 -0
  38. rdst-0.1.185/lib/data_manager/data_manager.py +1078 -0
  39. rdst-0.1.185/lib/data_manager_service/__init__.py +27 -0
  40. rdst-0.1.185/lib/data_manager_service/data_manager_service.py +791 -0
  41. rdst-0.1.185/lib/data_manager_service/data_manager_service_command_sets.py +670 -0
  42. rdst-0.1.185/lib/data_structures/__init__.py +7 -0
  43. rdst-0.1.185/lib/data_structures/query_defaults.py +133 -0
  44. rdst-0.1.185/lib/data_structures/semantic_layer.py +673 -0
  45. rdst-0.1.185/lib/db_connection.py +125 -0
  46. rdst-0.1.185/lib/debug/__init__.py +19 -0
  47. rdst-0.1.185/lib/debug/formatters.py +217 -0
  48. rdst-0.1.185/lib/debug/llm_inspector.py +270 -0
  49. rdst-0.1.185/lib/debug/session_inspector.py +208 -0
  50. rdst-0.1.185/lib/debug/snapshot_browser.py +247 -0
  51. rdst-0.1.185/lib/debug/state_viewer.py +227 -0
  52. rdst-0.1.185/lib/engines/ask3/__init__.py +88 -0
  53. rdst-0.1.185/lib/engines/ask3/agent.py +334 -0
  54. rdst-0.1.185/lib/engines/ask3/agent_context.py +256 -0
  55. rdst-0.1.185/lib/engines/ask3/agent_tools.py +504 -0
  56. rdst-0.1.185/lib/engines/ask3/clarification.py +235 -0
  57. rdst-0.1.185/lib/engines/ask3/context.py +268 -0
  58. rdst-0.1.185/lib/engines/ask3/engine.py +407 -0
  59. rdst-0.1.185/lib/engines/ask3/escalation.py +199 -0
  60. rdst-0.1.185/lib/engines/ask3/phases/__init__.py +26 -0
  61. rdst-0.1.185/lib/engines/ask3/phases/clarify.py +205 -0
  62. rdst-0.1.185/lib/engines/ask3/phases/execute.py +264 -0
  63. rdst-0.1.185/lib/engines/ask3/phases/expand.py +220 -0
  64. rdst-0.1.185/lib/engines/ask3/phases/filter.py +586 -0
  65. rdst-0.1.185/lib/engines/ask3/phases/generate.py +155 -0
  66. rdst-0.1.185/lib/engines/ask3/phases/present.py +86 -0
  67. rdst-0.1.185/lib/engines/ask3/phases/schema.py +336 -0
  68. rdst-0.1.185/lib/engines/ask3/phases/validate.py +164 -0
  69. rdst-0.1.185/lib/engines/ask3/presenter.py +393 -0
  70. rdst-0.1.185/lib/engines/ask3/types.py +205 -0
  71. rdst-0.1.185/lib/functions/__init__.py +81 -0
  72. rdst-0.1.185/lib/functions/ambiguity_detection.py +383 -0
  73. rdst-0.1.185/lib/functions/db_config_check.py +45 -0
  74. rdst-0.1.185/lib/functions/error_parser.py +218 -0
  75. rdst-0.1.185/lib/functions/explain_analysis.py +930 -0
  76. rdst-0.1.185/lib/functions/llm_analysis.py +954 -0
  77. rdst-0.1.185/lib/functions/parallel_merge.py +114 -0
  78. rdst-0.1.185/lib/functions/performance_comparison.py +579 -0
  79. rdst-0.1.185/lib/functions/postgres_metadata.py +97 -0
  80. rdst-0.1.185/lib/functions/query_metrics.py +578 -0
  81. rdst-0.1.185/lib/functions/query_parameterization.py +281 -0
  82. rdst-0.1.185/lib/functions/query_safety.py +204 -0
  83. rdst-0.1.185/lib/functions/readyset_cacheability.py +325 -0
  84. rdst-0.1.185/lib/functions/readyset_container.py +466 -0
  85. rdst-0.1.185/lib/functions/readyset_explain_cache.py +721 -0
  86. rdst-0.1.185/lib/functions/readyset_setup.py +65 -0
  87. rdst-0.1.185/lib/functions/result_display.py +445 -0
  88. rdst-0.1.185/lib/functions/result_export.py +189 -0
  89. rdst-0.1.185/lib/functions/rewrite_testing.py +734 -0
  90. rdst-0.1.185/lib/functions/schema_collector.py +534 -0
  91. rdst-0.1.185/lib/functions/smart_defaults.py +341 -0
  92. rdst-0.1.185/lib/functions/sql_generation.py +800 -0
  93. rdst-0.1.185/lib/functions/sql_validation.py +670 -0
  94. rdst-0.1.185/lib/functions/target_setup.py +820 -0
  95. rdst-0.1.185/lib/functions/test_data_generator.py +678 -0
  96. rdst-0.1.185/lib/functions/validation.py +105 -0
  97. rdst-0.1.185/lib/functions/workflow_integration.py +364 -0
  98. rdst-0.1.185/lib/history/__init__.py +5 -0
  99. rdst-0.1.185/lib/history/ask_history.py +155 -0
  100. rdst-0.1.185/lib/llm_manager/__init__.py +6 -0
  101. rdst-0.1.185/lib/llm_manager/base.py +117 -0
  102. rdst-0.1.185/lib/llm_manager/claude_provider.py +155 -0
  103. rdst-0.1.185/lib/llm_manager/gemini_provider.py +129 -0
  104. rdst-0.1.185/lib/llm_manager/llm_manager.py +237 -0
  105. rdst-0.1.185/lib/llm_manager/lmstudio_provider.py +142 -0
  106. rdst-0.1.185/lib/llm_manager/openai_provider.py +83 -0
  107. rdst-0.1.185/lib/prompts/__init__.py +24 -0
  108. rdst-0.1.185/lib/prompts/agent_prompts.py +234 -0
  109. rdst-0.1.185/lib/prompts/analyze_prompts.py +315 -0
  110. rdst-0.1.185/lib/prompts/ask_prompts.py +244 -0
  111. rdst-0.1.185/lib/prompts/ask_prompts_v2.py +409 -0
  112. rdst-0.1.185/lib/query_registry/__init__.py +42 -0
  113. rdst-0.1.185/lib/query_registry/analysis_results.py +427 -0
  114. rdst-0.1.185/lib/query_registry/conversation_registry.py +300 -0
  115. rdst-0.1.185/lib/query_registry/query_registry.py +665 -0
  116. rdst-0.1.185/lib/semantic_layer/__init__.py +9 -0
  117. rdst-0.1.185/lib/semantic_layer/ai_annotator.py +425 -0
  118. rdst-0.1.185/lib/semantic_layer/annotate_wizard.py +671 -0
  119. rdst-0.1.185/lib/semantic_layer/introspector.py +693 -0
  120. rdst-0.1.185/lib/semantic_layer/learner.py +373 -0
  121. rdst-0.1.185/lib/semantic_layer/manager.py +419 -0
  122. rdst-0.1.185/lib/telemetry/__init__.py +28 -0
  123. rdst-0.1.185/lib/telemetry/telemetry_manager.py +831 -0
  124. rdst-0.1.185/lib/top_display.py +425 -0
  125. rdst-0.1.185/lib/top_monitor.py +334 -0
  126. rdst-0.1.185/lib/top_realtime.py +408 -0
  127. rdst-0.1.185/lib/workflow_manager/README.md +151 -0
  128. rdst-0.1.185/lib/workflow_manager/workflow_manager.py +476 -0
  129. rdst-0.1.185/lib/workflows/analyze_workflow.json +254 -0
  130. rdst-0.1.185/lib/workflows/analyze_workflow_simple.json +154 -0
  131. rdst-0.1.185/lib/workflows/fetch_user_data_workflow.json +30 -0
  132. rdst-0.1.185/lib/workflows/install_readyset_for_target.json +160 -0
  133. rdst-0.1.185/lib/workflows/resources.py +16 -0
  134. rdst-0.1.185/lib/workflows/run_analyze_workflow.py +301 -0
  135. rdst-0.1.185/mcp_server.py +1461 -0
  136. rdst-0.1.185/pyproject.toml +74 -0
  137. rdst-0.1.185/rdst.py +983 -0
  138. rdst-0.1.185/requirements.txt +10 -0
  139. rdst-0.1.185/run_local_tests.sh +111 -0
  140. rdst-0.1.185/tests/MANUAL_TESTING.md +332 -0
  141. rdst-0.1.185/tests/__init__.py +1 -0
  142. rdst-0.1.185/tests/ask_experimental/MANUAL_TEST_CASES.md +427 -0
  143. rdst-0.1.185/tests/ask_experimental/README.md +60 -0
  144. rdst-0.1.185/tests/ask_experimental/ask_validation/test_limit_injection.py +203 -0
  145. rdst-0.1.185/tests/ask_experimental/conftest.py +15 -0
  146. rdst-0.1.185/tests/ask_experimental/semantic_layer_integration/test_context_enrichment.py +315 -0
  147. rdst-0.1.185/tests/ask_experimental/semantic_layer_integration/test_schema_performance.py +358 -0
  148. rdst-0.1.185/tests/ask_experimental/semantic_layer_integration/test_terminology_resolution.py +303 -0
  149. rdst-0.1.185/tests/ask_experimental/test_ask3_engine/__init__.py +1 -0
  150. rdst-0.1.185/tests/ask_experimental/test_ask3_engine/test_context.py +235 -0
  151. rdst-0.1.185/tests/ask_experimental/test_ask3_engine/test_engine.py +297 -0
  152. rdst-0.1.185/tests/ask_experimental/test_ask3_engine/test_presenter.py +148 -0
  153. rdst-0.1.185/tests/benchmarks/BULK_INDEX_TESTING.md +184 -0
  154. rdst-0.1.185/tests/benchmarks/TPCH_BENCHMARK_GUIDE.md +222 -0
  155. rdst-0.1.185/tests/benchmarks/tpch_queries.sql +740 -0
  156. rdst-0.1.185/tests/conftest.py +172 -0
  157. rdst-0.1.185/tests/integration/README.md +294 -0
  158. rdst-0.1.185/tests/integration/lib/helpers.sh +227 -0
  159. rdst-0.1.185/tests/integration/lib/setup.sh +588 -0
  160. rdst-0.1.185/tests/integration/lib/top_mysql_workload.py +83 -0
  161. rdst-0.1.185/tests/integration/lib/top_postgres_workload.py +83 -0
  162. rdst-0.1.185/tests/integration/run_single_test.sh +92 -0
  163. rdst-0.1.185/tests/integration/run_tests.sh +138 -0
  164. rdst-0.1.185/tests/integration/run_tests_local.sh +192 -0
  165. rdst-0.1.185/tests/integration/tests/test_analyze.sh +149 -0
  166. rdst-0.1.185/tests/integration/tests/test_cache.sh +49 -0
  167. rdst-0.1.185/tests/integration/tests/test_config.sh +229 -0
  168. rdst-0.1.185/tests/integration/tests/test_docs_sync.py +187 -0
  169. rdst-0.1.185/tests/integration/tests/test_errors.sh +42 -0
  170. rdst-0.1.185/tests/integration/tests/test_mcp_sync.py +193 -0
  171. rdst-0.1.185/tests/integration/tests/test_query_command.sh +272 -0
  172. rdst-0.1.185/tests/integration/tests/test_top_and_registry.sh +414 -0
  173. rdst-0.1.185/tests/requirements.txt +16 -0
  174. rdst-0.1.185/tests/unit/__init__.py +1 -0
  175. rdst-0.1.185/tests/unit/test_analysis_functions.py +500 -0
  176. rdst-0.1.185/tests/unit/test_cache_manager.py.skip +381 -0
  177. rdst-0.1.185/tests/unit/test_cacheability_integration.py +267 -0
  178. rdst-0.1.185/tests/unit/test_cli.py +428 -0
  179. rdst-0.1.185/tests/unit/test_cli_commands.py +387 -0
  180. rdst-0.1.185/tests/unit/test_connection_string_parser.py +245 -0
  181. rdst-0.1.185/tests/unit/test_data_manager.py +57 -0
  182. rdst-0.1.185/tests/unit/test_data_manager.py.skip +612 -0
  183. rdst-0.1.185/tests/unit/test_extension_detection.py +659 -0
  184. rdst-0.1.185/tests/unit/test_functions.py +402 -0
  185. rdst-0.1.185/tests/unit/test_llm_manager.py +231 -0
  186. rdst-0.1.185/tests/unit/test_providers.py.skip +322 -0
  187. rdst-0.1.185/tests/unit/test_query_parameterization.py +352 -0
  188. rdst-0.1.185/tests/unit/test_query_registry.py +642 -0
  189. rdst-0.1.185/tests/unit/test_query_safety.py +393 -0
  190. rdst-0.1.185/tests/unit/test_validation.py +378 -0
  191. rdst-0.1.185/tests/unit/test_workflow_manager.py +586 -0
@@ -0,0 +1,83 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ # Build Python package (wheel and sdist) with automatic versioning
5
+ cd rdst
6
+
7
+ # Clean old build artifacts
8
+ echo "Cleaning old build artifacts..."
9
+ rm -rf dist/ build/ *.egg-info
10
+
11
+ echo "Installing build dependencies..."
12
+ # Use PyPI directly to avoid CodeArtifact credential issues for build tools
13
+ python -m pip install --upgrade --index-url https://pypi.org/simple build twine
14
+
15
+ # Read major and minor version from _version.py
16
+ # Change to rdst directory if we're in the repo root
17
+ if [[ -f "rdst/_version.py" ]]; then
18
+ VERSION_FILE="rdst/_version.py"
19
+ elif [[ -f "_version.py" ]]; then
20
+ VERSION_FILE="_version.py"
21
+ else
22
+ echo "Error: Cannot find _version.py"
23
+ exit 1
24
+ fi
25
+
26
+ MAJOR=$(grep "^MAJOR = " "$VERSION_FILE" | cut -d '=' -f 2 | tr -d ' ')
27
+ MINOR=$(grep "^MINOR = " "$VERSION_FILE" | cut -d '=' -f 2 | tr -d ' ')
28
+
29
+ if [[ -z "$MAJOR" ]] || [[ -z "$MINOR" ]]; then
30
+ echo "Error: Could not read MAJOR/MINOR from $VERSION_FILE"
31
+ exit 1
32
+ fi
33
+
34
+ echo "Base version from _version.py: $MAJOR.$MINOR"
35
+
36
+ # Generate full version using semantic versioning (major.minor.patch[.build])
37
+ if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
38
+ # Buildkite release: major.minor.patch where patch = build number
39
+ # Format: 0.1.123 (e.g., version 0.1 with build 123)
40
+ export RDST_VERSION="${MAJOR}.${MINOR}.${BUILDKITE_BUILD_NUMBER}"
41
+ echo "Release build version: $RDST_VERSION"
42
+ else
43
+ # Local/dev build: major.minor.0.dev<timestamp> (PEP 440 compliant)
44
+ # Format: 0.1.0.dev20251204081653
45
+ TIMESTAMP=$(date +%Y%m%d%H%M%S)
46
+ export RDST_VERSION="${MAJOR}.${MINOR}.0.dev${TIMESTAMP}"
47
+ echo "Local dev build version: $RDST_VERSION"
48
+ fi
49
+
50
+ # Get git hash for metadata (optional but useful for tracing)
51
+ GIT_HASH=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
52
+ echo "Git commit: $GIT_HASH"
53
+
54
+ # Write version to _version_build.py for hatchling to read (not tracked in git)
55
+ echo "Writing version $RDST_VERSION to _version_build.py..."
56
+ cat > _version_build.py <<EOF
57
+ """
58
+ RDST Version Information
59
+
60
+ This file is AUTO-GENERATED by the build system.
61
+ Do not edit manually - it will be overwritten on next build.
62
+
63
+ Current version: $RDST_VERSION
64
+ Git commit: $GIT_HASH
65
+ """
66
+
67
+ # Version constants
68
+ MAJOR = ${MAJOR}
69
+ MINOR = ${MINOR}
70
+
71
+ # Full version (set by build system)
72
+ __version__ = "${RDST_VERSION}"
73
+ __version_info__ = tuple(map(int, "${RDST_VERSION}".split('.')[:2]))
74
+ EOF
75
+
76
+ # Force build tool to use PyPI for isolated environment (fixes 401 error)
77
+ export PIP_INDEX_URL=https://pypi.org/simple
78
+
79
+ echo "Building Python package version $RDST_VERSION..."
80
+ python -m build
81
+
82
+ echo "Build artifacts:"
83
+ ls -lh dist/
@@ -0,0 +1,97 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ # Ensure twine is installed (may run on different agent than build step)
5
+ python -m pip install --upgrade --quiet twine
6
+
7
+ # Deploy Python packages to AWS CodeArtifact (dev repository)
8
+ # This provides better version management and standard pip/uvx installation
9
+
10
+ # CodeArtifact configuration - adjust these as needed
11
+ CODEARTIFACT_DOMAIN="${CODEARTIFACT_DOMAIN:-readyset}"
12
+ CODEARTIFACT_REPOSITORY="${CODEARTIFACT_REPOSITORY:-rdst-dev}"
13
+ CODEARTIFACT_REGION="${CODEARTIFACT_REGION:-us-east-2}"
14
+ CODEARTIFACT_ACCOUNT="${CODEARTIFACT_ACCOUNT:-888984949675}"
15
+
16
+ echo "Publishing to AWS CodeArtifact..."
17
+ echo " Domain: $CODEARTIFACT_DOMAIN"
18
+ echo " Repository: $CODEARTIFACT_REPOSITORY"
19
+ echo " Region: $CODEARTIFACT_REGION"
20
+ echo " Account: $CODEARTIFACT_ACCOUNT"
21
+
22
+ # Verify AWS account
23
+ CURRENT_ACCOUNT=$(aws sts get-caller-identity --query Account --output text 2>/dev/null || echo "unknown")
24
+ if [[ "$CURRENT_ACCOUNT" == "$CODEARTIFACT_ACCOUNT" ]]; then
25
+ echo "✓ Using account $CODEARTIFACT_ACCOUNT"
26
+ elif [[ "$CURRENT_ACCOUNT" != "unknown" ]]; then
27
+ echo "ℹ️ Cross-account deployment from $CURRENT_ACCOUNT to account $CODEARTIFACT_ACCOUNT"
28
+ else
29
+ echo "⚠️ Could not determine current AWS account, proceeding anyway..."
30
+ fi
31
+
32
+ # Skip artifact download if running locally (artifacts only exist in Buildkite)
33
+ if command -v buildkite-agent &> /dev/null; then
34
+ echo "Downloading build artifacts..."
35
+ buildkite-agent artifact download "rdst/dist/*.whl" .
36
+ buildkite-agent artifact download "rdst/dist/*.tar.gz" .
37
+ DIST_PATH="rdst/dist"
38
+ else
39
+ echo "Running locally - using existing dist/ folder"
40
+ # Detect if we're in repo root or rdst/ directory
41
+ if [[ -d "rdst/dist" ]]; then
42
+ DIST_PATH="rdst/dist"
43
+ elif [[ -d "dist" ]]; then
44
+ DIST_PATH="dist"
45
+ else
46
+ echo "Error: Cannot find dist/ directory"
47
+ exit 1
48
+ fi
49
+ fi
50
+
51
+ # Get CodeArtifact authentication token (valid for 12 hours)
52
+ echo "Authenticating with CodeArtifact..."
53
+ CODEARTIFACT_TOKEN=$(aws codeartifact get-authorization-token \
54
+ --domain "$CODEARTIFACT_DOMAIN" \
55
+ --domain-owner "$CODEARTIFACT_ACCOUNT" \
56
+ --region "$CODEARTIFACT_REGION" \
57
+ --query authorizationToken \
58
+ --output text)
59
+
60
+ # Get CodeArtifact repository endpoint
61
+ CODEARTIFACT_ENDPOINT=$(aws codeartifact get-repository-endpoint \
62
+ --domain "$CODEARTIFACT_DOMAIN" \
63
+ --domain-owner "$CODEARTIFACT_ACCOUNT" \
64
+ --repository "$CODEARTIFACT_REPOSITORY" \
65
+ --region "$CODEARTIFACT_REGION" \
66
+ --format pypi \
67
+ --query repositoryEndpoint \
68
+ --output text)
69
+
70
+ echo "✓ Authenticated with CodeArtifact"
71
+ echo " Endpoint: $CODEARTIFACT_ENDPOINT"
72
+
73
+ # Publish to CodeArtifact using twine
74
+ echo "Publishing packages to CodeArtifact..."
75
+ python -m twine upload \
76
+ --repository-url "$CODEARTIFACT_ENDPOINT" \
77
+ --username aws \
78
+ --password "$CODEARTIFACT_TOKEN" \
79
+ "$DIST_PATH"/*.whl \
80
+ "$DIST_PATH"/*.tar.gz
81
+
82
+ echo ""
83
+ echo "✓ Dev deployment complete!"
84
+ echo ""
85
+ echo "Users can install with:"
86
+ echo " # Configure pip to use CodeArtifact (one-time setup):"
87
+ echo " aws codeartifact login --tool pip \\"
88
+ echo " --domain $CODEARTIFACT_DOMAIN \\"
89
+ echo " --domain-owner $CODEARTIFACT_ACCOUNT \\"
90
+ echo " --repository $CODEARTIFACT_REPOSITORY \\"
91
+ echo " --region $CODEARTIFACT_REGION"
92
+ echo ""
93
+ echo " # Then install/upgrade rdst:"
94
+ echo " pip install rdst"
95
+ echo " pipx install rdst"
96
+ echo " uvx rdst"
97
+ echo ""
@@ -0,0 +1,253 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ # Ensure twine is installed (may run on different agent than build step)
5
+ python -m pip install --upgrade --quiet twine
6
+
7
+ # Deploy Python packages to AWS CodeArtifact AND public PyPI
8
+ # This hybrid approach provides:
9
+ # - CodeArtifact: Private repository access for internal teams
10
+ # - PyPI: Public access for anyone (pipx install rdst, uvx rdst)
11
+
12
+ # CodeArtifact configuration - adjust these as needed
13
+ CODEARTIFACT_DOMAIN="${CODEARTIFACT_DOMAIN:-readyset}"
14
+ CODEARTIFACT_REPOSITORY="${CODEARTIFACT_REPOSITORY:-rdst}"
15
+ CODEARTIFACT_REGION="${CODEARTIFACT_REGION:-us-east-2}"
16
+ CODEARTIFACT_ACCOUNT="${CODEARTIFACT_ACCOUNT:-888984949675}"
17
+
18
+ # PyPI configuration
19
+ PUBLISH_TO_PYPI="${PUBLISH_TO_PYPI:-true}" # Set to false to skip PyPI publishing
20
+
21
+ echo "Publishing to AWS CodeArtifact (RELEASE)..."
22
+ echo " Domain: $CODEARTIFACT_DOMAIN"
23
+ echo " Repository: $CODEARTIFACT_REPOSITORY (PRODUCTION)"
24
+ echo " Region: $CODEARTIFACT_REGION"
25
+ echo " Account: $CODEARTIFACT_ACCOUNT"
26
+
27
+ # Verify AWS account
28
+ CURRENT_ACCOUNT=$(aws sts get-caller-identity --query Account --output text 2>/dev/null || echo "unknown")
29
+ if [[ "$CURRENT_ACCOUNT" == "$CODEARTIFACT_ACCOUNT" ]]; then
30
+ echo "✓ Using account $CODEARTIFACT_ACCOUNT"
31
+ elif [[ "$CURRENT_ACCOUNT" != "unknown" ]]; then
32
+ echo "ℹ️ Cross-account deployment from $CURRENT_ACCOUNT to account $CODEARTIFACT_ACCOUNT"
33
+ else
34
+ echo "⚠️ Could not determine current AWS account, proceeding anyway..."
35
+ fi
36
+
37
+ # Skip artifact download if running locally (artifacts only exist in Buildkite)
38
+ if command -v buildkite-agent &> /dev/null; then
39
+ echo "Downloading build artifacts..."
40
+ buildkite-agent artifact download "rdst/dist/*.whl" .
41
+ buildkite-agent artifact download "rdst/dist/*.tar.gz" .
42
+ DIST_PATH="rdst/dist"
43
+ else
44
+ echo "Running locally - using existing dist/ folder"
45
+ # Detect if we're in repo root or rdst/ directory
46
+ if [[ -d "rdst/dist" ]]; then
47
+ DIST_PATH="rdst/dist"
48
+ elif [[ -d "dist" ]]; then
49
+ DIST_PATH="dist"
50
+ else
51
+ echo "Error: Cannot find dist/ directory"
52
+ echo "Please run .buildkite/build_package.sh first to create build artifacts"
53
+ exit 1
54
+ fi
55
+ fi
56
+
57
+ # Get package version from wheel filename
58
+ WHEEL_FILE=$(ls "$DIST_PATH"/*.whl | head -1)
59
+ WHEEL_BASENAME=$(basename "$WHEEL_FILE")
60
+ # Extract version using semantic versioning pattern (e.g., rdst-0.1.123-py3-none-any.whl)
61
+ VERSION=$(echo "$WHEEL_BASENAME" | grep -oP '(?<=rdst-)[0-9]+\.[0-9]+\.[0-9]+' || echo "unknown")
62
+
63
+ echo ""
64
+ echo "📦 Publishing RDST version: $VERSION"
65
+ echo ""
66
+ echo "ℹ️ This version will be published to BOTH:"
67
+ echo " • CodeArtifact (rdst repository)"
68
+ echo " • PyPI (public)"
69
+ echo " Using the same build artifacts to ensure version consistency."
70
+ echo ""
71
+
72
+ # Get CodeArtifact authentication token (valid for 12 hours)
73
+ echo "Authenticating with CodeArtifact..."
74
+ CODEARTIFACT_TOKEN=$(aws codeartifact get-authorization-token \
75
+ --domain "$CODEARTIFACT_DOMAIN" \
76
+ --domain-owner "$CODEARTIFACT_ACCOUNT" \
77
+ --region "$CODEARTIFACT_REGION" \
78
+ --query authorizationToken \
79
+ --output text)
80
+
81
+ # Get CodeArtifact repository endpoint
82
+ CODEARTIFACT_ENDPOINT=$(aws codeartifact get-repository-endpoint \
83
+ --domain "$CODEARTIFACT_DOMAIN" \
84
+ --domain-owner "$CODEARTIFACT_ACCOUNT" \
85
+ --repository "$CODEARTIFACT_REPOSITORY" \
86
+ --region "$CODEARTIFACT_REGION" \
87
+ --format pypi \
88
+ --query repositoryEndpoint \
89
+ --output text)
90
+
91
+ echo "✓ Authenticated with CodeArtifact"
92
+ echo " Endpoint: $CODEARTIFACT_ENDPOINT"
93
+
94
+ # Publish to CodeArtifact using twine
95
+ echo "Publishing packages to CodeArtifact (PRODUCTION)..."
96
+ python -m twine upload \
97
+ --repository-url "$CODEARTIFACT_ENDPOINT" \
98
+ --username aws \
99
+ --password "$CODEARTIFACT_TOKEN" \
100
+ "$DIST_PATH"/*.whl \
101
+ "$DIST_PATH"/*.tar.gz
102
+
103
+ echo ""
104
+ echo "✓ Published to CodeArtifact successfully!"
105
+ echo ""
106
+
107
+ # ============================================================================
108
+ # Publish to public PyPI
109
+ # ============================================================================
110
+
111
+ if [[ "$PUBLISH_TO_PYPI" != "true" ]]; then
112
+ echo ""
113
+ echo "⚠️ Skipping PyPI publication (PUBLISH_TO_PYPI=$PUBLISH_TO_PYPI)"
114
+ echo ""
115
+ exit 0
116
+ fi
117
+
118
+ echo ""
119
+ echo "========================================="
120
+ echo "Publishing to PyPI (public)"
121
+ echo "========================================="
122
+ echo ""
123
+
124
+ # Get PyPI token from environment or AWS Secrets Manager
125
+ # NOTE: Secret buildkite/rdst/pypi-token is stored as JSON: {"PYPI_TOKEN": "pypi-..."}
126
+ if [[ -n "${PYPI_TOKEN:-}" ]]; then
127
+ echo "✓ Using PYPI_TOKEN from environment"
128
+ elif command -v aws &> /dev/null; then
129
+ echo "Fetching PyPI token from AWS Secrets Manager..."
130
+ echo " Secret: buildkite/rdst/pypi-token (account 305232526136, region us-east-2)"
131
+
132
+ # Fetch the secret JSON
133
+ SECRET_JSON=$(aws secretsmanager get-secret-value \
134
+ --secret-id buildkite/rdst/pypi-token \
135
+ --region us-east-2 \
136
+ --query SecretString \
137
+ --output text 2>/dev/null)
138
+
139
+ if [[ -n "$SECRET_JSON" ]]; then
140
+ # Extract PYPI_TOKEN from JSON key-value pair
141
+ PYPI_TOKEN=$(echo "$SECRET_JSON" | python3 -c "import sys, json; print(json.load(sys.stdin).get('PYPI_TOKEN', ''))" 2>/dev/null)
142
+
143
+ if [[ -n "$PYPI_TOKEN" ]]; then
144
+ echo "✓ Retrieved PYPI_TOKEN from AWS Secrets Manager"
145
+ export PYPI_TOKEN
146
+ else
147
+ echo "❌ ERROR: Secret found but PYPI_TOKEN key is missing or empty in JSON"
148
+ echo "Secret should be stored as: {\"PYPI_TOKEN\": \"pypi-...\"}"
149
+ exit 1
150
+ fi
151
+ else
152
+ echo ""
153
+ echo "❌ ERROR: Could not retrieve PYPI_TOKEN"
154
+ echo ""
155
+ echo "PyPI token not found in environment or AWS Secrets Manager."
156
+ echo ""
157
+ echo "To fix this, either:"
158
+ echo " 1. Set PYPI_TOKEN environment variable in Buildkite"
159
+ echo " 2. Store token in AWS Secrets Manager (account 305232526136):"
160
+ echo " aws secretsmanager create-secret \\"
161
+ echo " --name buildkite/rdst/pypi-token \\"
162
+ echo " --secret-string '{\"PYPI_TOKEN\":\"pypi-...\"}' \\"
163
+ echo " --region us-east-2"
164
+ echo ""
165
+ echo "See PYPI_SETUP.md for detailed instructions."
166
+ echo ""
167
+ exit 1
168
+ fi
169
+ else
170
+ echo ""
171
+ echo "❌ ERROR: PYPI_TOKEN not set and AWS CLI not available"
172
+ echo ""
173
+ echo "Set PYPI_TOKEN environment variable or install AWS CLI."
174
+ echo "See PYPI_SETUP.md for setup instructions."
175
+ echo ""
176
+ exit 1
177
+ fi
178
+
179
+ # Verify token format
180
+ if [[ ! "$PYPI_TOKEN" =~ ^pypi- ]]; then
181
+ echo ""
182
+ echo "⚠️ WARNING: PYPI_TOKEN doesn't start with 'pypi-'"
183
+ echo "This might not be a valid PyPI token."
184
+ echo ""
185
+ fi
186
+
187
+ # Publish to PyPI using twine
188
+ echo "Publishing rdst version $VERSION to PyPI..."
189
+ echo ""
190
+
191
+ python -m twine upload \
192
+ --repository pypi \
193
+ --username __token__ \
194
+ --password "$PYPI_TOKEN" \
195
+ --non-interactive \
196
+ --skip-existing \
197
+ --verbose \
198
+ "$DIST_PATH"/*.whl \
199
+ "$DIST_PATH"/*.tar.gz
200
+
201
+ PYPI_EXIT_CODE=$?
202
+
203
+ if [[ $PYPI_EXIT_CODE -eq 0 ]]; then
204
+ echo ""
205
+ echo "========================================="
206
+ echo "✓ Published to PyPI successfully!"
207
+ echo "========================================="
208
+ echo ""
209
+ echo "📦 Package: rdst version $VERSION"
210
+ echo ""
211
+ echo "✅ Version Consistency Check:"
212
+ echo " • CodeArtifact (rdst): $VERSION"
213
+ echo " • PyPI (public): $VERSION"
214
+ echo " → Same version published to both locations ✓"
215
+ echo ""
216
+ echo "🌍 Public Access (PyPI):"
217
+ echo " PyPI page: https://pypi.org/project/rdst/$VERSION/"
218
+ echo " Install: pipx install rdst"
219
+ echo " Run: uvx rdst"
220
+ echo " Upgrade: pipx upgrade rdst"
221
+ echo ""
222
+ echo "🔒 Internal Access (CodeArtifact):"
223
+ echo " aws codeartifact login --tool pip \\"
224
+ echo " --domain $CODEARTIFACT_DOMAIN \\"
225
+ echo " --domain-owner $CODEARTIFACT_ACCOUNT \\"
226
+ echo " --repository $CODEARTIFACT_REPOSITORY \\"
227
+ echo " --region $CODEARTIFACT_REGION"
228
+ echo " pip install rdst==$VERSION"
229
+ echo ""
230
+ else
231
+ echo ""
232
+ echo "========================================="
233
+ echo "⚠️ PyPI publication failed!"
234
+ echo "========================================="
235
+ echo ""
236
+ echo "Exit code: $PYPI_EXIT_CODE"
237
+ echo ""
238
+ echo "Common issues:"
239
+ echo " - Version $VERSION already exists on PyPI (cannot overwrite)"
240
+ echo " - Invalid PYPI_TOKEN"
241
+ echo " - Network connectivity issues"
242
+ echo " - Package metadata issues"
243
+ echo ""
244
+ echo "The package was successfully published to CodeArtifact."
245
+ echo "Users with AWS credentials can still access it."
246
+ echo ""
247
+ echo "To retry PyPI publication:"
248
+ echo " 1. Fix the issue (see error above)"
249
+ echo " 2. Increment version in _version.py if version conflict"
250
+ echo " 3. Re-run this script or trigger a new build"
251
+ echo ""
252
+ exit 1
253
+ fi
@@ -0,0 +1,122 @@
1
+ # RDST Build and Deployment Pipeline
2
+ # Tests run on pre-merge only (CLs). After merge to main:
3
+ # - Dev builds → published to AWS CodeArtifact (private)
4
+ # - Production releases → published to CodeArtifact AND public PyPI (manual approval required)
5
+
6
+ steps:
7
+ # If this is a gerrit change list build, notify Gerrit that the build started
8
+ - label: ":gerrit: Start CL"
9
+ branches: "!refs/heads/main"
10
+ command: .buildkite/set_gerrit_running.sh
11
+ agents:
12
+ queue: t3a-small
13
+
14
+ - wait
15
+ # Run unit tests on pre-merge only
16
+ - label: ":test_tube: Run RDST Unit Tests"
17
+ key: "rdst-unit-tests"
18
+ branches: "!refs/heads/main"
19
+ command: "chmod +x rdst/.buildkite/run_unit_tests.sh && rdst/.buildkite/run_unit_tests.sh"
20
+ artifact_paths:
21
+ - "rdst/test-results/*.xml"
22
+ agents:
23
+ queue: t3a-small
24
+ timeout_in_minutes: 15
25
+ retry:
26
+ automatic:
27
+ - exit_status: -1
28
+ limit: 2
29
+
30
+ # Run integration tests on pre-merge only - PostgreSQL
31
+ - label: ":postgres: RDST Integration Tests (PostgreSQL)"
32
+ key: "rdst-integration-postgresql"
33
+ branches: "!refs/heads/main"
34
+ depends_on: "rdst-unit-tests"
35
+ command: "chmod +x rdst/.buildkite/run_integration_tests.sh && rdst/.buildkite/run_integration_tests.sh postgresql"
36
+ env:
37
+ DUPLO_ENV: dev
38
+ DUPLO_TENANT: dev01
39
+ agents:
40
+ queue: t3a-small
41
+ timeout_in_minutes: 30
42
+ retry:
43
+ automatic:
44
+ - exit_status: -1
45
+ limit: 2
46
+
47
+ # Run integration tests on pre-merge only - MySQL
48
+ - label: ":mysql: RDST Integration Tests (MySQL)"
49
+ key: "rdst-integration-mysql"
50
+ branches: "!refs/heads/main"
51
+ depends_on: "rdst-unit-tests"
52
+ command: "chmod +x rdst/.buildkite/run_integration_tests.sh && rdst/.buildkite/run_integration_tests.sh mysql"
53
+ env:
54
+ DUPLO_ENV: dev
55
+ DUPLO_TENANT: dev01
56
+ agents:
57
+ queue: t3a-small
58
+ timeout_in_minutes: 30
59
+ retry:
60
+ automatic:
61
+ - exit_status: -1
62
+ limit: 2
63
+
64
+ # Wait for all non-main-only steps to complete before updating Gerrit CL status
65
+ - wait: ~
66
+ branches: '!refs/heads/main'
67
+ continue_on_failure: true
68
+
69
+ # Update gerrit with CL build status (pass/fail)
70
+ - label: ":gerrit: Update CL status"
71
+ key: set-gerrit-status
72
+ branches: '!refs/heads/main'
73
+ depends_on:
74
+ - "rdst-unit-tests"
75
+ - "rdst-integration-postgresql"
76
+ - "rdst-integration-mysql"
77
+ command: .buildkite/set_gerrit_status.sh
78
+ agents:
79
+ queue: t3a-small
80
+ retry:
81
+ manual:
82
+ permit_on_passed: true
83
+
84
+ # All main-only steps must appear below this line
85
+ # -------------------------------------------------------------------------------------------------
86
+
87
+ # After merge to main: Build and publish Python package to dev CodeArtifact
88
+ - label: ":package: Build RDST Python Package"
89
+ key: "build-package"
90
+ branches: "refs/heads/main"
91
+ command: "chmod +x rdst/.buildkite/build_package.sh && rdst/.buildkite/build_package.sh"
92
+ artifact_paths:
93
+ - "rdst/dist/*.whl"
94
+ - "rdst/dist/*.tar.gz"
95
+ agents:
96
+ queue: t3a-small
97
+
98
+ # Publish to dev CodeArtifact repository
99
+ - label: ":rocket: Publish to Dev CodeArtifact"
100
+ key: "publish-dev"
101
+ branches: "refs/heads/main"
102
+ depends_on:
103
+ - "build-package"
104
+ command: "chmod +x rdst/.buildkite/deploy_dev.sh && rdst/.buildkite/deploy_dev.sh"
105
+ agents:
106
+ queue: t3a-small
107
+
108
+ # Manual approval required to promote to production (CodeArtifact + PyPI)
109
+ - block: ":rocket: Publish to Production (PyPI)"
110
+ key: "release-approval"
111
+ branches: "refs/heads/main"
112
+ prompt: "Ready to publish RDST to public PyPI and production CodeArtifact? This will make the package publicly available."
113
+ depends_on: "publish-dev"
114
+
115
+ # Publish to production: CodeArtifact (rdst) AND public PyPI
116
+ - label: ":globe_with_meridians: Publish to PyPI + CodeArtifact"
117
+ key: "publish-release"
118
+ branches: "refs/heads/main"
119
+ depends_on: "release-approval"
120
+ command: "chmod +x rdst/.buildkite/deploy_release.sh && rdst/.buildkite/deploy_release.sh"
121
+ agents:
122
+ queue: t3a-small
@@ -0,0 +1,120 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -euo pipefail
4
+
5
+ # Get database type from argument
6
+ DB_TYPE="${1:-}"
7
+
8
+ if [[ -z "$DB_TYPE" ]]; then
9
+ echo "ERROR: Database type argument required (postgresql or mysql)" >&2
10
+ echo "Usage: $0 [postgresql|mysql]" >&2
11
+ exit 1
12
+ fi
13
+
14
+ if [[ "$DB_TYPE" != "postgresql" && "$DB_TYPE" != "mysql" ]]; then
15
+ echo "ERROR: Invalid database type: $DB_TYPE" >&2
16
+ echo "Must be 'postgresql' or 'mysql'" >&2
17
+ exit 1
18
+ fi
19
+
20
+ echo "================================================================="
21
+ echo "RDST CLI Integration Tests - $DB_TYPE"
22
+ echo "================================================================="
23
+ echo ""
24
+
25
+ # Set up environment
26
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
27
+ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
28
+ TEST_SCRIPT="${REPO_ROOT}/rdst/tests/integration/run_tests.sh"
29
+
30
+ # Verify test script exists
31
+ if [[ ! -f "$TEST_SCRIPT" ]]; then
32
+ echo "ERROR: Test script not found at: $TEST_SCRIPT" >&2
33
+ exit 1
34
+ fi
35
+
36
+ # Ensure script is executable
37
+ chmod +x "$TEST_SCRIPT"
38
+
39
+ # Determine API base URL based on environment
40
+ if [[ -z "${DUPLO_ENV:-}" ]] || [[ -z "${DUPLO_TENANT:-}" ]]; then
41
+ echo "ERROR: DUPLO_ENV and DUPLO_TENANT must be set" >&2
42
+ exit 1
43
+ fi
44
+
45
+ if [[ "${DUPLO_ENV}" == "dev" ]]; then
46
+ if [[ "${DUPLO_TENANT}" == "dev01" ]]; then
47
+ API_PREFIX="api-dev01.apps"
48
+ elif [[ "${DUPLO_TENANT}" == "dev02" ]]; then
49
+ API_PREFIX="api-dev02.apps"
50
+ else
51
+ API_PREFIX="api-${DUPLO_TENANT}.apps"
52
+ fi
53
+ elif [[ "${DUPLO_ENV}" == "stage" ]]; then
54
+ API_PREFIX="api-stage"
55
+ elif [[ "${DUPLO_ENV}" == "prod" ]]; then
56
+ API_PREFIX="api"
57
+ else
58
+ echo "ERROR: Unsupported environment: ${DUPLO_ENV}" >&2
59
+ exit 1
60
+ fi
61
+
62
+ export API_BASE_URL="https://${API_PREFIX}.readyset.cloud"
63
+
64
+ # Obtain Supabase authentication token for admin API
65
+ echo "Obtaining authentication token..."
66
+
67
+ # Get Supabase project secrets
68
+ SUPABASE_PROJECT_SECRETS_NAME="supabase/${DUPLO_TENANT}"
69
+ SUPABASE_PROJECT_SECRETS=$(aws secretsmanager get-secret-value --secret-id ${SUPABASE_PROJECT_SECRETS_NAME} --cli-connect-timeout 1 | jq -r .SecretString)
70
+
71
+ SUPABASE_PROJECT_URL=$(echo ${SUPABASE_PROJECT_SECRETS} | jq -r '.SUPABASE_PROJECT_URL')
72
+ SUPABASE_PROJECT_API_KEY=$(echo ${SUPABASE_PROJECT_SECRETS} | jq -r '.SUPABASE_PROJECT_API_KEY')
73
+
74
+ # Get Supabase user credentials for authentication
75
+ SUPABASE_BUILDKITE_SECRETS_NAME="supabase-buildkite-creds/${DUPLO_TENANT}"
76
+ SUPABASE_BUILDKITE_SECRETS=$(aws secretsmanager get-secret-value --secret-id ${SUPABASE_BUILDKITE_SECRETS_NAME} --cli-connect-timeout 1 | jq -r .SecretString)
77
+
78
+ SUPABASE_USERNAME=$(echo ${SUPABASE_BUILDKITE_SECRETS} | jq -r '.username')
79
+ SUPABASE_PASSWORD=$(echo ${SUPABASE_BUILDKITE_SECRETS} | jq -r '.password')
80
+
81
+ # Obtain an access token from Supabase
82
+ TMP_CREDS=$(curl -s --location --request POST "${SUPABASE_PROJECT_URL}/auth/v1/token?grant_type=password" \
83
+ --header "Content-Type: application/json" \
84
+ --header "apikey: ${SUPABASE_PROJECT_API_KEY}" \
85
+ --data-raw '{"email": "'${SUPABASE_USERNAME}'", "password": "'${SUPABASE_PASSWORD}'"}')
86
+
87
+ export ADMIN_API_TOKEN=$(echo ${TMP_CREDS} | jq -r '.access_token')
88
+
89
+ if [[ -z "$ADMIN_API_TOKEN" || "$ADMIN_API_TOKEN" == "null" ]]; then
90
+ echo "ERROR: Failed to obtain authentication token" >&2
91
+ echo "Response: $TMP_CREDS" >&2
92
+ exit 1
93
+ fi
94
+
95
+ echo "✓ Authentication token obtained"
96
+
97
+ # Install Python dependencies
98
+ echo ""
99
+ echo "Installing Python dependencies..."
100
+ cd "$REPO_ROOT/rdst"
101
+ if [[ -f "requirements.txt" ]]; then
102
+ pip3 install --quiet --user -r requirements.txt 2>&1 | grep -v "WARNING: The script" || true
103
+ echo "✓ Python dependencies installed"
104
+ fi
105
+
106
+ USER_SITE=$(python3 -c "import site; print(site.getusersitepackages())")
107
+ if [[ -n "$USER_SITE" ]]; then
108
+ export PYTHONPATH="${USER_SITE}${PYTHONPATH:+:${PYTHONPATH}}"
109
+ fi
110
+
111
+ echo ""
112
+ echo "Configuration:"
113
+ echo " Environment: ${DUPLO_ENV}"
114
+ echo " Tenant: ${DUPLO_TENANT}"
115
+ echo " Database Type: $DB_TYPE"
116
+ echo " API Base URL: $API_BASE_URL"
117
+ echo ""
118
+
119
+ # Run the tests for the specified database type
120
+ exec "$TEST_SCRIPT" "$DB_TYPE"