specweave 0.18.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +229 -1817
- package/README.md +68 -0
- package/bin/specweave.js +62 -6
- package/dist/plugins/specweave/lib/hooks/sync-living-docs.d.ts.map +1 -1
- package/dist/plugins/specweave/lib/hooks/sync-living-docs.js +3 -0
- package/dist/plugins/specweave/lib/hooks/sync-living-docs.js.map +1 -1
- package/dist/plugins/specweave/lib/hooks/update-ac-status.d.ts +21 -0
- package/dist/plugins/specweave/lib/hooks/update-ac-status.d.ts.map +1 -0
- package/dist/plugins/specweave/lib/hooks/update-ac-status.js +162 -0
- package/dist/plugins/specweave/lib/hooks/update-ac-status.js.map +1 -0
- package/dist/plugins/specweave-ado/lib/ado-spec-content-sync.d.ts.map +1 -1
- package/dist/plugins/specweave-ado/lib/ado-spec-content-sync.js +65 -6
- package/dist/plugins/specweave-ado/lib/ado-spec-content-sync.js.map +1 -1
- package/dist/plugins/specweave-github/lib/completion-calculator.d.ts +112 -0
- package/dist/plugins/specweave-github/lib/completion-calculator.d.ts.map +1 -0
- package/dist/plugins/specweave-github/lib/completion-calculator.js +301 -0
- package/dist/plugins/specweave-github/lib/completion-calculator.js.map +1 -0
- package/dist/plugins/specweave-github/lib/duplicate-detector.d.ts +3 -3
- package/dist/plugins/specweave-github/lib/duplicate-detector.js +3 -3
- package/dist/plugins/specweave-github/lib/epic-content-builder.d.ts +7 -0
- package/dist/plugins/specweave-github/lib/epic-content-builder.d.ts.map +1 -1
- package/dist/plugins/specweave-github/lib/epic-content-builder.js +42 -0
- package/dist/plugins/specweave-github/lib/epic-content-builder.js.map +1 -1
- package/dist/plugins/specweave-github/lib/github-client-v2.d.ts +14 -0
- package/dist/plugins/specweave-github/lib/github-client-v2.d.ts.map +1 -1
- package/dist/plugins/specweave-github/lib/github-client-v2.js +51 -0
- package/dist/plugins/specweave-github/lib/github-client-v2.js.map +1 -1
- package/dist/plugins/specweave-github/lib/github-epic-sync.js +1 -1
- package/dist/plugins/specweave-github/lib/github-epic-sync.js.map +1 -1
- package/dist/plugins/specweave-github/lib/github-feature-sync.d.ts +87 -0
- package/dist/plugins/specweave-github/lib/github-feature-sync.d.ts.map +1 -0
- package/dist/plugins/specweave-github/lib/github-feature-sync.js +412 -0
- package/dist/plugins/specweave-github/lib/github-feature-sync.js.map +1 -0
- package/dist/plugins/specweave-github/lib/github-spec-content-sync.d.ts.map +1 -1
- package/dist/plugins/specweave-github/lib/github-spec-content-sync.js +64 -13
- package/dist/plugins/specweave-github/lib/github-spec-content-sync.js.map +1 -1
- package/dist/plugins/specweave-github/lib/progress-comment-builder.d.ts +78 -0
- package/dist/plugins/specweave-github/lib/progress-comment-builder.d.ts.map +1 -0
- package/dist/plugins/specweave-github/lib/progress-comment-builder.js +237 -0
- package/dist/plugins/specweave-github/lib/progress-comment-builder.js.map +1 -0
- package/dist/plugins/specweave-github/lib/user-story-content-builder.d.ts +97 -0
- package/dist/plugins/specweave-github/lib/user-story-content-builder.d.ts.map +1 -0
- package/dist/plugins/specweave-github/lib/user-story-content-builder.js +301 -0
- package/dist/plugins/specweave-github/lib/user-story-content-builder.js.map +1 -0
- package/dist/plugins/specweave-github/lib/user-story-issue-builder.d.ts +83 -0
- package/dist/plugins/specweave-github/lib/user-story-issue-builder.d.ts.map +1 -0
- package/dist/plugins/specweave-github/lib/user-story-issue-builder.js +386 -0
- package/dist/plugins/specweave-github/lib/user-story-issue-builder.js.map +1 -0
- package/dist/plugins/specweave-jira/lib/enhanced-jira-sync.d.ts +8 -6
- package/dist/plugins/specweave-jira/lib/enhanced-jira-sync.d.ts.map +1 -1
- package/dist/plugins/specweave-jira/lib/enhanced-jira-sync.js +78 -117
- package/dist/plugins/specweave-jira/lib/enhanced-jira-sync.js.map +1 -1
- package/dist/plugins/specweave-kafka/lib/cli/kcat-wrapper.d.ts +57 -0
- package/dist/plugins/specweave-kafka/lib/cli/kcat-wrapper.d.ts.map +1 -0
- package/dist/plugins/specweave-kafka/lib/cli/kcat-wrapper.js +248 -0
- package/dist/plugins/specweave-kafka/lib/cli/kcat-wrapper.js.map +1 -0
- package/dist/plugins/specweave-kafka/lib/cli/types.d.ts +82 -0
- package/dist/plugins/specweave-kafka/lib/cli/types.d.ts.map +1 -0
- package/dist/plugins/specweave-kafka/lib/cli/types.js +13 -0
- package/dist/plugins/specweave-kafka/lib/cli/types.js.map +1 -0
- package/dist/plugins/specweave-kafka/lib/mcp/detector.d.ts +49 -0
- package/dist/plugins/specweave-kafka/lib/mcp/detector.d.ts.map +1 -0
- package/dist/plugins/specweave-kafka/lib/mcp/detector.js +316 -0
- package/dist/plugins/specweave-kafka/lib/mcp/detector.js.map +1 -0
- package/dist/plugins/specweave-kafka/lib/mcp/types.d.ts +70 -0
- package/dist/plugins/specweave-kafka/lib/mcp/types.d.ts.map +1 -0
- package/dist/plugins/specweave-kafka/lib/mcp/types.js +23 -0
- package/dist/plugins/specweave-kafka/lib/mcp/types.js.map +1 -0
- package/dist/plugins/specweave-kafka/lib/utils/partitioning.d.ts +85 -0
- package/dist/plugins/specweave-kafka/lib/utils/partitioning.d.ts.map +1 -0
- package/dist/plugins/specweave-kafka/lib/utils/partitioning.js +281 -0
- package/dist/plugins/specweave-kafka/lib/utils/partitioning.js.map +1 -0
- package/dist/plugins/specweave-kafka/lib/utils/sizing.d.ts +75 -0
- package/dist/plugins/specweave-kafka/lib/utils/sizing.d.ts.map +1 -0
- package/dist/plugins/specweave-kafka/lib/utils/sizing.js +238 -0
- package/dist/plugins/specweave-kafka/lib/utils/sizing.js.map +1 -0
- package/dist/src/cli/commands/import-docs.js +4 -4
- package/dist/src/cli/commands/import-docs.js.map +1 -1
- package/dist/src/cli/commands/init-multiproject.d.ts.map +1 -1
- package/dist/src/cli/commands/init-multiproject.js +17 -18
- package/dist/src/cli/commands/init-multiproject.js.map +1 -1
- package/dist/src/cli/commands/migrate-to-multiproject.d.ts.map +1 -1
- package/dist/src/cli/commands/migrate-to-multiproject.js +8 -4
- package/dist/src/cli/commands/migrate-to-multiproject.js.map +1 -1
- package/dist/src/cli/commands/switch-project.d.ts.map +1 -1
- package/dist/src/cli/commands/switch-project.js +9 -26
- package/dist/src/cli/commands/switch-project.js.map +1 -1
- package/dist/src/cli/commands/sync-spec-content.js +3 -0
- package/dist/src/cli/commands/sync-spec-content.js.map +1 -1
- package/dist/src/core/deduplication/command-deduplicator.d.ts +166 -0
- package/dist/src/core/deduplication/command-deduplicator.d.ts.map +1 -0
- package/dist/src/core/deduplication/command-deduplicator.js +254 -0
- package/dist/src/core/deduplication/command-deduplicator.js.map +1 -0
- package/dist/src/core/increment/active-increment-manager.d.ts +42 -15
- package/dist/src/core/increment/active-increment-manager.d.ts.map +1 -1
- package/dist/src/core/increment/active-increment-manager.js +113 -46
- package/dist/src/core/increment/active-increment-manager.js.map +1 -1
- package/dist/src/core/increment/conflict-resolver.d.ts +40 -0
- package/dist/src/core/increment/conflict-resolver.d.ts.map +1 -0
- package/dist/src/core/increment/conflict-resolver.js +219 -0
- package/dist/src/core/increment/conflict-resolver.js.map +1 -0
- package/dist/src/core/increment/discipline-checker.d.ts.map +1 -1
- package/dist/src/core/increment/discipline-checker.js +7 -1
- package/dist/src/core/increment/discipline-checker.js.map +1 -1
- package/dist/src/core/increment/duplicate-detector.d.ts +52 -0
- package/dist/src/core/increment/duplicate-detector.d.ts.map +1 -0
- package/dist/src/core/increment/duplicate-detector.js +276 -0
- package/dist/src/core/increment/duplicate-detector.js.map +1 -0
- package/dist/src/core/increment/increment-archiver.d.ts +90 -0
- package/dist/src/core/increment/increment-archiver.d.ts.map +1 -0
- package/dist/src/core/increment/increment-archiver.js +368 -0
- package/dist/src/core/increment/increment-archiver.js.map +1 -0
- package/dist/src/core/increment/increment-reopener.d.ts +165 -0
- package/dist/src/core/increment/increment-reopener.d.ts.map +1 -0
- package/dist/src/core/increment/increment-reopener.js +390 -0
- package/dist/src/core/increment/increment-reopener.js.map +1 -0
- package/dist/src/core/increment/metadata-manager.d.ts +26 -1
- package/dist/src/core/increment/metadata-manager.d.ts.map +1 -1
- package/dist/src/core/increment/metadata-manager.js +143 -5
- package/dist/src/core/increment/metadata-manager.js.map +1 -1
- package/dist/src/core/increment/recent-work-scanner.d.ts +121 -0
- package/dist/src/core/increment/recent-work-scanner.d.ts.map +1 -0
- package/dist/src/core/increment/recent-work-scanner.js +303 -0
- package/dist/src/core/increment/recent-work-scanner.js.map +1 -0
- package/dist/src/core/increment/types.d.ts +1 -0
- package/dist/src/core/increment/types.d.ts.map +1 -1
- package/dist/src/core/increment-utils.d.ts +112 -0
- package/dist/src/core/increment-utils.d.ts.map +1 -0
- package/dist/src/core/increment-utils.js +210 -0
- package/dist/src/core/increment-utils.js.map +1 -0
- package/dist/src/core/living-docs/ac-project-specific-generator.d.ts +65 -0
- package/dist/src/core/living-docs/ac-project-specific-generator.d.ts.map +1 -0
- package/dist/src/core/living-docs/ac-project-specific-generator.js +175 -0
- package/dist/src/core/living-docs/ac-project-specific-generator.js.map +1 -0
- package/dist/src/core/living-docs/feature-archiver.d.ts +130 -0
- package/dist/src/core/living-docs/feature-archiver.d.ts.map +1 -0
- package/dist/src/core/living-docs/feature-archiver.js +549 -0
- package/dist/src/core/living-docs/feature-archiver.js.map +1 -0
- package/dist/src/core/living-docs/feature-id-manager.d.ts +81 -0
- package/dist/src/core/living-docs/feature-id-manager.d.ts.map +1 -0
- package/dist/src/core/living-docs/feature-id-manager.js +339 -0
- package/dist/src/core/living-docs/feature-id-manager.js.map +1 -0
- package/dist/src/core/living-docs/hierarchy-mapper.d.ts +144 -83
- package/dist/src/core/living-docs/hierarchy-mapper.d.ts.map +1 -1
- package/dist/src/core/living-docs/hierarchy-mapper.js +488 -270
- package/dist/src/core/living-docs/hierarchy-mapper.js.map +1 -1
- package/dist/src/core/living-docs/index.d.ts +6 -0
- package/dist/src/core/living-docs/index.d.ts.map +1 -1
- package/dist/src/core/living-docs/index.js +6 -0
- package/dist/src/core/living-docs/index.js.map +1 -1
- package/dist/src/core/living-docs/project-detector.d.ts +6 -0
- package/dist/src/core/living-docs/project-detector.d.ts.map +1 -1
- package/dist/src/core/living-docs/project-detector.js +35 -1
- package/dist/src/core/living-docs/project-detector.js.map +1 -1
- package/dist/src/core/living-docs/spec-distributor.d.ts +100 -26
- package/dist/src/core/living-docs/spec-distributor.d.ts.map +1 -1
- package/dist/src/core/living-docs/spec-distributor.js +1275 -258
- package/dist/src/core/living-docs/spec-distributor.js.map +1 -1
- package/dist/src/core/living-docs/task-project-specific-generator.d.ts +109 -0
- package/dist/src/core/living-docs/task-project-specific-generator.d.ts.map +1 -0
- package/dist/src/core/living-docs/task-project-specific-generator.js +221 -0
- package/dist/src/core/living-docs/task-project-specific-generator.js.map +1 -0
- package/dist/src/core/living-docs/types.d.ts +143 -0
- package/dist/src/core/living-docs/types.d.ts.map +1 -1
- package/dist/src/core/project-manager.d.ts +2 -17
- package/dist/src/core/project-manager.d.ts.map +1 -1
- package/dist/src/core/project-manager.js +68 -48
- package/dist/src/core/project-manager.js.map +1 -1
- package/dist/src/core/spec-content-sync.d.ts +1 -1
- package/dist/src/core/spec-content-sync.d.ts.map +1 -1
- package/dist/src/core/sync/enhanced-content-builder.d.ts.map +1 -1
- package/dist/src/core/sync/enhanced-content-builder.js +2 -1
- package/dist/src/core/sync/enhanced-content-builder.js.map +1 -1
- package/dist/src/core/sync/performance-optimizer.d.ts +153 -0
- package/dist/src/core/sync/performance-optimizer.d.ts.map +1 -0
- package/dist/src/core/sync/performance-optimizer.js +220 -0
- package/dist/src/core/sync/performance-optimizer.js.map +1 -0
- package/dist/src/core/sync/retry-handler.d.ts +98 -0
- package/dist/src/core/sync/retry-handler.d.ts.map +1 -0
- package/dist/src/core/sync/retry-handler.js +196 -0
- package/dist/src/core/sync/retry-handler.js.map +1 -0
- package/dist/src/core/types/config.d.ts +94 -0
- package/dist/src/core/types/config.d.ts.map +1 -1
- package/dist/src/core/types/config.js +16 -0
- package/dist/src/core/types/config.js.map +1 -1
- package/dist/src/core/types/increment-metadata.d.ts +6 -0
- package/dist/src/core/types/increment-metadata.d.ts.map +1 -1
- package/dist/src/core/types/increment-metadata.js +10 -1
- package/dist/src/core/types/increment-metadata.js.map +1 -1
- package/dist/src/integrations/jira/jira-incremental-mapper.d.ts.map +1 -1
- package/dist/src/integrations/jira/jira-incremental-mapper.js +4 -8
- package/dist/src/integrations/jira/jira-incremental-mapper.js.map +1 -1
- package/dist/src/integrations/jira/jira-mapper.d.ts.map +1 -1
- package/dist/src/integrations/jira/jira-mapper.js +4 -8
- package/dist/src/integrations/jira/jira-mapper.js.map +1 -1
- package/package.json +1 -1
- package/plugins/specweave/COMMANDS.md +13 -4
- package/plugins/specweave/commands/specweave-abandon.md +22 -20
- package/plugins/specweave/commands/specweave-archive-features.md +121 -0
- package/plugins/specweave/commands/specweave-archive-increments.md +82 -0
- package/plugins/specweave/commands/specweave-archive.md +363 -0
- package/plugins/specweave/commands/specweave-backlog.md +211 -0
- package/plugins/specweave/commands/specweave-fix-duplicates.md +517 -0
- package/plugins/specweave/commands/specweave-increment.md +4 -3
- package/plugins/specweave/commands/specweave-progress.md +176 -27
- package/plugins/specweave/commands/specweave-reopen.md +391 -0
- package/plugins/specweave/commands/specweave-restore-feature.md +90 -0
- package/plugins/specweave/commands/specweave-restore.md +309 -0
- package/plugins/specweave/commands/specweave-resume.md +51 -23
- package/plugins/specweave/commands/specweave-status.md +41 -7
- package/plugins/specweave/commands/specweave-sync-specs.md +425 -0
- package/plugins/specweave/hooks/hooks.json +4 -0
- package/plugins/specweave/hooks/lib/sync-spec-content.sh +2 -2
- package/plugins/specweave/hooks/post-task-completion.sh +39 -0
- package/plugins/specweave/hooks/pre-command-deduplication.sh +83 -0
- package/plugins/specweave/hooks/user-prompt-submit.sh +1 -1
- package/plugins/specweave/lib/hooks/sync-living-docs.js +2 -0
- package/plugins/specweave/lib/hooks/sync-living-docs.ts +4 -0
- package/plugins/specweave/lib/hooks/update-ac-status.js +102 -0
- package/plugins/specweave/lib/hooks/update-ac-status.ts +192 -0
- package/plugins/specweave/skills/archive-increments/SKILL.md +198 -0
- package/plugins/specweave/skills/increment-planner/scripts/feature-utils.js +14 -0
- package/plugins/specweave/skills/smart-reopen-detector/SKILL.md +244 -0
- package/plugins/specweave-ado/lib/ado-spec-content-sync.js +49 -5
- package/plugins/specweave-ado/lib/ado-spec-content-sync.ts +72 -6
- package/plugins/specweave-confluent/.claude-plugin/plugin.json +23 -0
- package/plugins/specweave-confluent/README.md +375 -0
- package/plugins/specweave-confluent/agents/confluent-architect/AGENT.md +306 -0
- package/plugins/specweave-confluent/skills/confluent-kafka-connect/SKILL.md +453 -0
- package/plugins/specweave-confluent/skills/confluent-ksqldb/SKILL.md +470 -0
- package/plugins/specweave-confluent/skills/confluent-schema-registry/SKILL.md +316 -0
- package/plugins/specweave-github/agents/github-task-splitter/AGENT.md +2 -2
- package/plugins/specweave-github/agents/user-story-updater/AGENT.md +148 -0
- package/plugins/specweave-github/commands/specweave-github-cleanup-duplicates.md +1 -1
- package/plugins/specweave-github/commands/specweave-github-update-user-story.md +156 -0
- package/plugins/specweave-github/hooks/post-task-completion.sh +10 -9
- package/plugins/specweave-github/lib/completion-calculator.js +262 -0
- package/plugins/specweave-github/lib/completion-calculator.ts +434 -0
- package/plugins/specweave-github/lib/duplicate-detector.js +3 -3
- package/plugins/specweave-github/lib/duplicate-detector.ts +4 -4
- package/plugins/specweave-github/lib/epic-content-builder.js +38 -0
- package/plugins/specweave-github/lib/epic-content-builder.ts +59 -0
- package/plugins/specweave-github/lib/github-client-v2.js +49 -0
- package/plugins/specweave-github/lib/github-client-v2.ts +59 -0
- package/plugins/specweave-github/lib/github-epic-sync.ts +1 -1
- package/plugins/specweave-github/lib/github-feature-sync.js +381 -0
- package/plugins/specweave-github/lib/github-feature-sync.ts +568 -0
- package/plugins/specweave-github/lib/github-spec-content-sync.js +40 -10
- package/plugins/specweave-github/lib/github-spec-content-sync.ts +82 -14
- package/plugins/specweave-github/lib/progress-comment-builder.js +229 -0
- package/plugins/specweave-github/lib/progress-comment-builder.ts +324 -0
- package/plugins/specweave-github/lib/user-story-content-builder.js +299 -0
- package/plugins/specweave-github/lib/user-story-content-builder.ts +413 -0
- package/plugins/specweave-github/lib/user-story-issue-builder.js +344 -0
- package/plugins/specweave-github/lib/user-story-issue-builder.ts +543 -0
- package/plugins/specweave-github/skills/github-issue-standard/SKILL.md +189 -0
- package/plugins/specweave-jira/lib/enhanced-jira-sync.js +134 -0
- package/plugins/specweave-jira/lib/{enhanced-jira-sync.ts.disabled → enhanced-jira-sync.ts} +26 -52
- package/plugins/specweave-kafka/.claude-plugin/plugin.json +26 -0
- package/plugins/specweave-kafka/IMPLEMENTATION-COMPLETE.md +483 -0
- package/plugins/specweave-kafka/README.md +242 -0
- package/plugins/specweave-kafka/agents/kafka-architect/AGENT.md +235 -0
- package/plugins/specweave-kafka/agents/kafka-devops/AGENT.md +209 -0
- package/plugins/specweave-kafka/agents/kafka-observability/AGENT.md +266 -0
- package/plugins/specweave-kafka/commands/deploy.md +99 -0
- package/plugins/specweave-kafka/commands/dev-env.md +176 -0
- package/plugins/specweave-kafka/commands/mcp-configure.md +101 -0
- package/plugins/specweave-kafka/commands/monitor-setup.md +96 -0
- package/plugins/specweave-kafka/docker/kafka-local/docker-compose.yml +187 -0
- package/plugins/specweave-kafka/docker/redpanda/docker-compose.yml +199 -0
- package/plugins/specweave-kafka/docker/templates/consumer-nodejs.js +225 -0
- package/plugins/specweave-kafka/docker/templates/consumer-python.py +220 -0
- package/plugins/specweave-kafka/docker/templates/producer-nodejs.js +168 -0
- package/plugins/specweave-kafka/docker/templates/producer-python.py +167 -0
- package/plugins/specweave-kafka/lib/adapters/apache-kafka-adapter.js +438 -0
- package/plugins/specweave-kafka/lib/adapters/apache-kafka-adapter.ts +541 -0
- package/plugins/specweave-kafka/lib/adapters/platform-adapter.js +47 -0
- package/plugins/specweave-kafka/lib/adapters/platform-adapter.ts +343 -0
- package/plugins/specweave-kafka/lib/cli/kcat-wrapper.js +258 -0
- package/plugins/specweave-kafka/lib/cli/kcat-wrapper.ts +298 -0
- package/plugins/specweave-kafka/lib/cli/types.js +10 -0
- package/plugins/specweave-kafka/lib/cli/types.ts +92 -0
- package/plugins/specweave-kafka/lib/connectors/connector-catalog.js +305 -0
- package/plugins/specweave-kafka/lib/connectors/connector-catalog.ts +528 -0
- package/plugins/specweave-kafka/lib/documentation/diagram-generator.js +114 -0
- package/plugins/specweave-kafka/lib/documentation/diagram-generator.ts +195 -0
- package/plugins/specweave-kafka/lib/documentation/exporter.js +210 -0
- package/plugins/specweave-kafka/lib/documentation/exporter.ts +338 -0
- package/plugins/specweave-kafka/lib/documentation/schema-catalog-generator.js +60 -0
- package/plugins/specweave-kafka/lib/documentation/schema-catalog-generator.ts +130 -0
- package/plugins/specweave-kafka/lib/documentation/topology-generator.js +143 -0
- package/plugins/specweave-kafka/lib/documentation/topology-generator.ts +290 -0
- package/plugins/specweave-kafka/lib/mcp/detector.js +298 -0
- package/plugins/specweave-kafka/lib/mcp/detector.ts +352 -0
- package/plugins/specweave-kafka/lib/mcp/types.js +21 -0
- package/plugins/specweave-kafka/lib/mcp/types.ts +77 -0
- package/plugins/specweave-kafka/lib/multi-cluster/cluster-config-manager.js +193 -0
- package/plugins/specweave-kafka/lib/multi-cluster/cluster-config-manager.ts +362 -0
- package/plugins/specweave-kafka/lib/multi-cluster/cluster-switcher.js +188 -0
- package/plugins/specweave-kafka/lib/multi-cluster/cluster-switcher.ts +359 -0
- package/plugins/specweave-kafka/lib/multi-cluster/health-aggregator.js +195 -0
- package/plugins/specweave-kafka/lib/multi-cluster/health-aggregator.ts +380 -0
- package/plugins/specweave-kafka/lib/observability/opentelemetry-kafka.js +209 -0
- package/plugins/specweave-kafka/lib/observability/opentelemetry-kafka.ts +358 -0
- package/plugins/specweave-kafka/lib/patterns/advanced-ksqldb-patterns.js +354 -0
- package/plugins/specweave-kafka/lib/patterns/advanced-ksqldb-patterns.ts +563 -0
- package/plugins/specweave-kafka/lib/patterns/circuit-breaker-resilience.js +259 -0
- package/plugins/specweave-kafka/lib/patterns/circuit-breaker-resilience.ts +516 -0
- package/plugins/specweave-kafka/lib/patterns/dead-letter-queue.js +233 -0
- package/plugins/specweave-kafka/lib/patterns/dead-letter-queue.ts +423 -0
- package/plugins/specweave-kafka/lib/patterns/exactly-once-semantics.js +266 -0
- package/plugins/specweave-kafka/lib/patterns/exactly-once-semantics.ts +445 -0
- package/plugins/specweave-kafka/lib/patterns/flink-kafka-integration.js +312 -0
- package/plugins/specweave-kafka/lib/patterns/flink-kafka-integration.ts +561 -0
- package/plugins/specweave-kafka/lib/patterns/multi-dc-replication.js +289 -0
- package/plugins/specweave-kafka/lib/patterns/multi-dc-replication.ts +607 -0
- package/plugins/specweave-kafka/lib/patterns/rate-limiting-backpressure.js +264 -0
- package/plugins/specweave-kafka/lib/patterns/rate-limiting-backpressure.ts +498 -0
- package/plugins/specweave-kafka/lib/patterns/stream-processing-optimization.js +263 -0
- package/plugins/specweave-kafka/lib/patterns/stream-processing-optimization.ts +549 -0
- package/plugins/specweave-kafka/lib/patterns/tiered-storage-compaction.js +205 -0
- package/plugins/specweave-kafka/lib/patterns/tiered-storage-compaction.ts +399 -0
- package/plugins/specweave-kafka/lib/performance/performance-optimizer.js +249 -0
- package/plugins/specweave-kafka/lib/performance/performance-optimizer.ts +427 -0
- package/plugins/specweave-kafka/lib/security/kafka-security.js +252 -0
- package/plugins/specweave-kafka/lib/security/kafka-security.ts +494 -0
- package/plugins/specweave-kafka/lib/utils/capacity-planner.js +203 -0
- package/plugins/specweave-kafka/lib/utils/capacity-planner.ts +469 -0
- package/plugins/specweave-kafka/lib/utils/config-validator.js +419 -0
- package/plugins/specweave-kafka/lib/utils/config-validator.ts +564 -0
- package/plugins/specweave-kafka/lib/utils/partitioning.js +329 -0
- package/plugins/specweave-kafka/lib/utils/partitioning.ts +473 -0
- package/plugins/specweave-kafka/lib/utils/sizing.js +221 -0
- package/plugins/specweave-kafka/lib/utils/sizing.ts +374 -0
- package/plugins/specweave-kafka/monitoring/grafana/dashboards/kafka-broker-metrics.json +628 -0
- package/plugins/specweave-kafka/monitoring/grafana/dashboards/kafka-cluster-overview.json +564 -0
- package/plugins/specweave-kafka/monitoring/grafana/dashboards/kafka-consumer-lag.json +509 -0
- package/plugins/specweave-kafka/monitoring/grafana/dashboards/kafka-jvm-metrics.json +674 -0
- package/plugins/specweave-kafka/monitoring/grafana/dashboards/kafka-topic-metrics.json +578 -0
- package/plugins/specweave-kafka/monitoring/grafana/provisioning/dashboards/kafka.yml +17 -0
- package/plugins/specweave-kafka/monitoring/grafana/provisioning/datasources/prometheus.yml +17 -0
- package/plugins/specweave-kafka/monitoring/prometheus/kafka-alerts.yml +415 -0
- package/plugins/specweave-kafka/monitoring/prometheus/kafka-jmx-exporter.yml +256 -0
- package/plugins/specweave-kafka/package.json +41 -0
- package/plugins/specweave-kafka/skills/kafka-architecture/SKILL.md +647 -0
- package/plugins/specweave-kafka/skills/kafka-cli-tools/SKILL.md +433 -0
- package/plugins/specweave-kafka/skills/kafka-iac-deployment/SKILL.md +449 -0
- package/plugins/specweave-kafka/skills/kafka-kubernetes/SKILL.md +667 -0
- package/plugins/specweave-kafka/skills/kafka-mcp-integration/SKILL.md +273 -0
- package/plugins/specweave-kafka/skills/kafka-observability/SKILL.md +576 -0
- package/plugins/specweave-kafka/templates/config/broker-production.properties +254 -0
- package/plugins/specweave-kafka/templates/config/consumer-low-latency.properties +112 -0
- package/plugins/specweave-kafka/templates/config/producer-high-throughput.properties +120 -0
- package/plugins/specweave-kafka/templates/migration/mirrormaker2-config.properties +234 -0
- package/plugins/specweave-kafka/templates/monitoring/grafana/multi-cluster-dashboard.json +686 -0
- package/plugins/specweave-kafka/terraform/apache-kafka/main.tf +347 -0
- package/plugins/specweave-kafka/terraform/apache-kafka/outputs.tf +107 -0
- package/plugins/specweave-kafka/terraform/apache-kafka/templates/kafka-broker-init.sh.tpl +216 -0
- package/plugins/specweave-kafka/terraform/apache-kafka/variables.tf +156 -0
- package/plugins/specweave-kafka/terraform/aws-msk/main.tf +362 -0
- package/plugins/specweave-kafka/terraform/aws-msk/outputs.tf +93 -0
- package/plugins/specweave-kafka/terraform/aws-msk/templates/server.properties.tpl +32 -0
- package/plugins/specweave-kafka/terraform/aws-msk/variables.tf +235 -0
- package/plugins/specweave-kafka/terraform/azure-event-hubs/main.tf +281 -0
- package/plugins/specweave-kafka/terraform/azure-event-hubs/outputs.tf +118 -0
- package/plugins/specweave-kafka/terraform/azure-event-hubs/variables.tf +148 -0
- package/plugins/specweave-kafka/tsconfig.json +21 -0
- package/plugins/specweave-kafka-streams/.claude-plugin/plugin.json +23 -0
- package/plugins/specweave-kafka-streams/README.md +310 -0
- package/plugins/specweave-kafka-streams/skills/kafka-streams-topology/SKILL.md +539 -0
- package/plugins/specweave-n8n/.claude-plugin/plugin.json +22 -0
- package/plugins/specweave-n8n/README.md +354 -0
- package/plugins/specweave-n8n/skills/n8n-kafka-workflows/SKILL.md +504 -0
- package/plugins/specweave-release/commands/specweave-release-platform.md +1 -1
- package/plugins/specweave-release/hooks/post-task-completion.sh +2 -2
- package/src/templates/AGENTS.md.template +601 -7
- package/src/templates/CLAUDE.md.template +188 -88
- package/plugins/specweave-ado/commands/specweave-ado-sync-spec.md +0 -255
- package/plugins/specweave-github/commands/specweave-github-sync-epic.md +0 -248
- package/plugins/specweave-github/commands/specweave-github-sync-from.md +0 -147
- package/plugins/specweave-github/commands/specweave-github-sync-spec.md +0 -208
- package/plugins/specweave-github/commands/specweave-github-sync-tasks.md +0 -530
- package/plugins/specweave-jira/commands/specweave-jira-sync-epic.md +0 -267
- package/plugins/specweave-jira/commands/specweave-jira-sync-spec.md +0 -240
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"partitioning.d.ts","sourceRoot":"","sources":["../../../../../plugins/specweave-kafka/lib/utils/partitioning.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,SAAS,GAAG,QAAQ,GAAG,iBAAiB,CAAC;IACvD,WAAW,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,0BAA0B;IACzC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,EAAE,aAAa,GAAG,eAAe,GAAG,oBAAoB,GAAG,aAAa,CAAC;IACpF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,wBAAwB;IAEvC,SAAS,EAAE,MAAM,CAAC;IAClB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,yBAAyB,EAAE,MAAM,CAAC;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IAGxB,aAAa,EAAE,KAAK,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,oBAAoB,EAAE,MAAM,CAAC;KAC9B,CAAC,CAAC;IAGH,gBAAgB,EAAE,OAAO,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,qBAAa,4BAA4B;IACvC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAM;IACpD,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAK;IAEnD;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,wBAAwB,GAAG,0BAA0B;IAwElE;;OAEG;IACH,OAAO,CAAC,UAAU;IAqClB;;OAEG;IACH,OAAO,CAAC,aAAa;IAkBrB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAiD5B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAuDxB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IA+FxB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,UAAU;IAIlB;;OAEG;IACH,qBAAqB,CACnB,YAAY,EAAE,MAAM,EAAE,EACtB,cAAc,EAAE,MAAM,GACrB;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE;IAsB7D;;OAEG;IACH,OAAO,CAAC,WAAW;CAgCpB"}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Partitioning Strategy Analyzer
|
|
3
|
+
*
|
|
4
|
+
* Analyzes partition key distribution and provides recommendations
|
|
5
|
+
* for optimal Kafka topic partitioning strategies.
|
|
6
|
+
*/
|
|
7
|
+
export class PartitioningStrategyAnalyzer {
|
|
8
|
+
constructor() {
|
|
9
|
+
this.MAX_PARTITION_THROUGHPUT_MBPS = 20;
|
|
10
|
+
this.MAX_PARTITION_MSG_PER_SEC = 50000;
|
|
11
|
+
this.IDEAL_PARTITIONS_PER_CONSUMER = 2;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Analyze partition key candidates and recommend strategy
|
|
15
|
+
*/
|
|
16
|
+
analyze(req) {
|
|
17
|
+
const peakMultiplier = req.peakMultiplier ?? 3;
|
|
18
|
+
const targetLatency = req.targetLatencyMs ?? 100;
|
|
19
|
+
const peakMessagesPerSec = req.expectedMessagesPerSecond * peakMultiplier;
|
|
20
|
+
// Analyze each potential key
|
|
21
|
+
const keyAnalyses = req.potentialKeys.map(key => this.analyzeKey(key));
|
|
22
|
+
// Find best key based on distribution
|
|
23
|
+
const bestKey = this.selectBestKey(keyAnalyses, req.orderingRequired);
|
|
24
|
+
// Calculate partitions needed based on throughput
|
|
25
|
+
let partitionsForThroughput = 1;
|
|
26
|
+
if (req.targetThroughputMBps) {
|
|
27
|
+
partitionsForThroughput = Math.ceil(req.targetThroughputMBps / this.MAX_PARTITION_THROUGHPUT_MBPS);
|
|
28
|
+
}
|
|
29
|
+
// Calculate partitions needed based on message rate
|
|
30
|
+
const partitionsForMsgRate = Math.ceil(peakMessagesPerSec / this.MAX_PARTITION_MSG_PER_SEC);
|
|
31
|
+
// Calculate partitions for consumer parallelism
|
|
32
|
+
const partitionsForParallelism = Math.max(4, partitionsForMsgRate * this.IDEAL_PARTITIONS_PER_CONSUMER);
|
|
33
|
+
// Take maximum (most constraining)
|
|
34
|
+
let recommendedPartitions = Math.max(partitionsForThroughput, partitionsForMsgRate, partitionsForParallelism);
|
|
35
|
+
// Round up to nearest power of 2 or multiple of 3 (for rack awareness)
|
|
36
|
+
if (recommendedPartitions <= 16) {
|
|
37
|
+
recommendedPartitions = this.nextPowerOfTwo(recommendedPartitions);
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
recommendedPartitions = Math.ceil(recommendedPartitions / 12) * 12; // Multiple of 12 (3 racks × 4)
|
|
41
|
+
}
|
|
42
|
+
// Ensure minimum partitions
|
|
43
|
+
recommendedPartitions = Math.max(recommendedPartitions, 3);
|
|
44
|
+
// Determine key strategy
|
|
45
|
+
const { keyStrategy, keyField, reasoning } = this.determineKeyStrategy(bestKey, req.orderingRequired, recommendedPartitions);
|
|
46
|
+
// Generate warnings
|
|
47
|
+
const warnings = this.generateWarnings({
|
|
48
|
+
bestKey,
|
|
49
|
+
recommendedPartitions,
|
|
50
|
+
currentPartitionCount: req.currentPartitionCount,
|
|
51
|
+
req
|
|
52
|
+
});
|
|
53
|
+
// Generate usage examples
|
|
54
|
+
const examples = this.generateExamples(keyStrategy, keyField, req.topicName);
|
|
55
|
+
return {
|
|
56
|
+
recommendedPartitions,
|
|
57
|
+
keyStrategy,
|
|
58
|
+
keyField,
|
|
59
|
+
reasoning,
|
|
60
|
+
warnings,
|
|
61
|
+
examples
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Analyze a single partition key candidate
|
|
66
|
+
*/
|
|
67
|
+
analyzeKey(key) {
|
|
68
|
+
const cardinality = key.estimatedUniqueCount;
|
|
69
|
+
const sampleSize = key.sampleValues.length;
|
|
70
|
+
// Estimate distribution from sample
|
|
71
|
+
const uniqueInSample = new Set(key.sampleValues).size;
|
|
72
|
+
const uniquenessRatio = uniqueInSample / sampleSize;
|
|
73
|
+
// Classify distribution
|
|
74
|
+
let distribution;
|
|
75
|
+
if (uniquenessRatio > 0.9) {
|
|
76
|
+
distribution = 'uniform'; // Most values are unique
|
|
77
|
+
}
|
|
78
|
+
else if (uniquenessRatio > 0.5) {
|
|
79
|
+
distribution = 'skewed'; // Some duplicate values
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
distribution = 'severely-skewed'; // Many duplicate values
|
|
83
|
+
}
|
|
84
|
+
// Assess hotspot risk
|
|
85
|
+
let hotspotRisk;
|
|
86
|
+
if (cardinality > 10000 && distribution === 'uniform') {
|
|
87
|
+
hotspotRisk = 'low';
|
|
88
|
+
}
|
|
89
|
+
else if (cardinality > 1000 && distribution !== 'severely-skewed') {
|
|
90
|
+
hotspotRisk = 'medium';
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
hotspotRisk = 'high';
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
key: key.fieldName,
|
|
97
|
+
sampleValues: key.sampleValues.slice(0, 5), // Top 5 examples
|
|
98
|
+
estimatedCardinality: cardinality,
|
|
99
|
+
distribution,
|
|
100
|
+
hotspotRisk
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Select best partition key from candidates
|
|
105
|
+
*/
|
|
106
|
+
selectBestKey(analyses, orderingRequired) {
|
|
107
|
+
// Sort by hotspot risk (low is best) and cardinality (high is best)
|
|
108
|
+
const sorted = [...analyses].sort((a, b) => {
|
|
109
|
+
const riskScore = { low: 0, medium: 1, high: 2 };
|
|
110
|
+
const riskDiff = riskScore[a.hotspotRisk] - riskScore[b.hotspotRisk];
|
|
111
|
+
if (riskDiff !== 0)
|
|
112
|
+
return riskDiff; // Lower risk wins
|
|
113
|
+
// If risks equal, higher cardinality wins
|
|
114
|
+
return b.estimatedCardinality - a.estimatedCardinality;
|
|
115
|
+
});
|
|
116
|
+
return sorted[0];
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Determine optimal partitioning key strategy
|
|
120
|
+
*/
|
|
121
|
+
determineKeyStrategy(bestKey, orderingRequired, partitionCount) {
|
|
122
|
+
// No ordering required and no good key → round-robin
|
|
123
|
+
if (!orderingRequired && (!bestKey || bestKey.hotspotRisk === 'high')) {
|
|
124
|
+
return {
|
|
125
|
+
keyStrategy: 'round-robin',
|
|
126
|
+
keyField: null,
|
|
127
|
+
reasoning: 'No ordering requirement and no suitable key field. Round-robin provides best load distribution.'
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
// Ordering required but bad key → need compound key or custom partitioner
|
|
131
|
+
if (orderingRequired && bestKey && bestKey.hotspotRisk === 'high') {
|
|
132
|
+
return {
|
|
133
|
+
keyStrategy: 'compound-hash',
|
|
134
|
+
keyField: bestKey.key,
|
|
135
|
+
reasoning: `Ordering required but ${bestKey.key} has high hotspot risk. Use compound key (e.g., ${bestKey.key} + timestamp) to distribute load.`
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
// Good key with ordering → simple hash
|
|
139
|
+
if (orderingRequired && bestKey && bestKey.hotspotRisk !== 'high') {
|
|
140
|
+
return {
|
|
141
|
+
keyStrategy: 'simple-hash',
|
|
142
|
+
keyField: bestKey.key,
|
|
143
|
+
reasoning: `${bestKey.key} has ${bestKey.distribution} distribution with ${bestKey.hotspotRisk} hotspot risk. Use simple hash partitioning.`
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
// No ordering, but good key available → can use for locality
|
|
147
|
+
if (!orderingRequired && bestKey && bestKey.hotspotRisk === 'low') {
|
|
148
|
+
return {
|
|
149
|
+
keyStrategy: 'simple-hash',
|
|
150
|
+
keyField: bestKey.key,
|
|
151
|
+
reasoning: `${bestKey.key} has excellent distribution. Use hash partitioning for data locality even though ordering not required.`
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
// Fallback: custom partitioner for complex cases
|
|
155
|
+
return {
|
|
156
|
+
keyStrategy: 'custom-partitioner',
|
|
157
|
+
keyField: bestKey?.key ?? null,
|
|
158
|
+
reasoning: 'Complex partitioning requirements. Implement custom partitioner for optimal distribution.'
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Generate warnings for partitioning strategy
|
|
163
|
+
*/
|
|
164
|
+
generateWarnings(params) {
|
|
165
|
+
const warnings = [];
|
|
166
|
+
// Hotspot warnings
|
|
167
|
+
if (params.bestKey && params.bestKey.hotspotRisk === 'high') {
|
|
168
|
+
warnings.push(`⚠️ Best key (${params.bestKey.key}) has HIGH hotspot risk with cardinality ${params.bestKey.estimatedCardinality}. ` +
|
|
169
|
+
`Consider compound key or custom partitioner.`);
|
|
170
|
+
}
|
|
171
|
+
if (params.bestKey && params.bestKey.hotspotRisk === 'medium') {
|
|
172
|
+
warnings.push(`⚠️ Key (${params.bestKey.key}) has MEDIUM hotspot risk. Monitor partition distribution in production.`);
|
|
173
|
+
}
|
|
174
|
+
// Partition count warnings
|
|
175
|
+
if (params.currentPartitionCount && params.recommendedPartitions > params.currentPartitionCount) {
|
|
176
|
+
warnings.push(`⚠️ Current partition count (${params.currentPartitionCount}) is below recommended (${params.recommendedPartitions}). ` +
|
|
177
|
+
`Increasing partitions requires re-creating topic or using partition expansion (Kafka 2.4+).`);
|
|
178
|
+
}
|
|
179
|
+
if (params.recommendedPartitions > 100) {
|
|
180
|
+
warnings.push(`⚠️ Recommended ${params.recommendedPartitions} partitions is high. Ensure cluster can handle this (max ~4000 per broker).`);
|
|
181
|
+
}
|
|
182
|
+
// Ordering warnings
|
|
183
|
+
if (params.req.orderingRequired && !params.bestKey) {
|
|
184
|
+
warnings.push(`🚨 CRITICAL: Ordering required but no suitable partition key found. ALL messages will be unordered!`);
|
|
185
|
+
}
|
|
186
|
+
// Throughput warnings
|
|
187
|
+
if (params.req.targetThroughputMBps && params.req.targetThroughputMBps > params.recommendedPartitions * this.MAX_PARTITION_THROUGHPUT_MBPS) {
|
|
188
|
+
const neededPartitions = Math.ceil(params.req.targetThroughputMBps / this.MAX_PARTITION_THROUGHPUT_MBPS);
|
|
189
|
+
warnings.push(`⚠️ Target throughput (${params.req.targetThroughputMBps} MB/s) requires ${neededPartitions} partitions, ` +
|
|
190
|
+
`but recommended only ${params.recommendedPartitions}. Increase partition count.`);
|
|
191
|
+
}
|
|
192
|
+
return warnings;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Generate usage examples for the recommended strategy
|
|
196
|
+
*/
|
|
197
|
+
generateExamples(keyStrategy, keyField, topicName) {
|
|
198
|
+
const examples = [];
|
|
199
|
+
switch (keyStrategy) {
|
|
200
|
+
case 'simple-hash':
|
|
201
|
+
examples.push(`// Producer (JavaScript/Node.js)`, `await producer.send({`, ` topic: '${topicName}',`, ` messages: [{`, ` key: record.${keyField}, // Hash this key`, ` value: JSON.stringify(record)`, ` }]`, `});`, ``, `// Producer (Java)`, `ProducerRecord<String, String> record = new ProducerRecord<>(`, ` "${topicName}",`, ` record.get${this.capitalize(keyField || 'id')}(), // Partition key`, ` jsonValue`, `);`, `producer.send(record);`);
|
|
202
|
+
break;
|
|
203
|
+
case 'compound-hash':
|
|
204
|
+
examples.push(`// Producer (JavaScript/Node.js) - Compound Key`, `const compoundKey = \`\${record.${keyField}}-\${Date.now() % 100}\`; // Add temporal component`, `await producer.send({`, ` topic: '${topicName}',`, ` messages: [{`, ` key: compoundKey,`, ` value: JSON.stringify(record)`, ` }]`, `});`, ``, `// Alternative: Geographic + Entity ID`, `const compoundKey = \`\${record.region}-\${record.${keyField}}\`;`);
|
|
205
|
+
break;
|
|
206
|
+
case 'custom-partitioner':
|
|
207
|
+
examples.push(`// Custom Partitioner (Java)`, `public class CustomPartitioner implements Partitioner {`, ` @Override`, ` public int partition(String topic, Object key, byte[] keyBytes,`, ` Object value, byte[] valueBytes, Cluster cluster) {`, ` int partitionCount = cluster.partitionCountForTopic(topic);`, ` `, ` // Custom logic here`, ` String keyStr = (String) key;`, ` if (keyStr.startsWith("priority-")) {`, ` return 0; // High-priority partition`, ` }`, ` `, ` // Default hashing for others`, ` return Math.abs(keyStr.hashCode()) % (partitionCount - 1) + 1;`, ` }`, `}`, ``, `// Configure producer`, `props.put("partitioner.class", "com.example.CustomPartitioner");`);
|
|
208
|
+
break;
|
|
209
|
+
case 'round-robin':
|
|
210
|
+
examples.push(`// Producer (JavaScript/Node.js) - No Key`, `await producer.send({`, ` topic: '${topicName}',`, ` messages: [{`, ` value: JSON.stringify(record) // No key = round-robin`, ` }]`, `});`, ``, `// Producer (Java)`, `ProducerRecord<String, String> record = new ProducerRecord<>(`, ` "${topicName}",`, ` null, // No key`, ` jsonValue`, `);`, `producer.send(record);`);
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
return examples;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Helper: Next power of 2
|
|
217
|
+
*/
|
|
218
|
+
nextPowerOfTwo(n) {
|
|
219
|
+
return Math.pow(2, Math.ceil(Math.log2(n)));
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Helper: Capitalize first letter
|
|
223
|
+
*/
|
|
224
|
+
capitalize(str) {
|
|
225
|
+
return str.charAt(0).toUpperCase() + str.slice(1);
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Calculate partition distribution for a given key
|
|
229
|
+
*/
|
|
230
|
+
calculateDistribution(sampleValues, partitionCount) {
|
|
231
|
+
// Simulate Kafka's DefaultPartitioner
|
|
232
|
+
const partitionCounts = new Map();
|
|
233
|
+
for (const value of sampleValues) {
|
|
234
|
+
const hash = this.murmur2Hash(value);
|
|
235
|
+
const partition = Math.abs(hash) % partitionCount;
|
|
236
|
+
partitionCounts.set(partition, (partitionCounts.get(partition) || 0) + 1);
|
|
237
|
+
}
|
|
238
|
+
// Convert to array and calculate percentages
|
|
239
|
+
const distribution = Array.from(partitionCounts.entries())
|
|
240
|
+
.map(([partition, count]) => ({
|
|
241
|
+
partition,
|
|
242
|
+
count,
|
|
243
|
+
percentage: (count / sampleValues.length) * 100
|
|
244
|
+
}))
|
|
245
|
+
.sort((a, b) => b.count - a.count);
|
|
246
|
+
return distribution;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Simple MurmurHash2 implementation (matches Kafka's DefaultPartitioner)
|
|
250
|
+
*/
|
|
251
|
+
murmur2Hash(data) {
|
|
252
|
+
const bytes = Buffer.from(data, 'utf-8');
|
|
253
|
+
const length = bytes.length;
|
|
254
|
+
const seed = 0x9747b28c;
|
|
255
|
+
const m = 0x5bd1e995;
|
|
256
|
+
const r = 24;
|
|
257
|
+
let h = seed ^ length;
|
|
258
|
+
for (let i = 0; i + 4 <= length; i += 4) {
|
|
259
|
+
let k = bytes.readInt32LE(i);
|
|
260
|
+
k = Math.imul(k, m);
|
|
261
|
+
k ^= k >>> r;
|
|
262
|
+
k = Math.imul(k, m);
|
|
263
|
+
h = Math.imul(h, m);
|
|
264
|
+
h ^= k;
|
|
265
|
+
}
|
|
266
|
+
const remaining = length % 4;
|
|
267
|
+
if (remaining >= 3)
|
|
268
|
+
h ^= bytes[length - 3] << 16;
|
|
269
|
+
if (remaining >= 2)
|
|
270
|
+
h ^= bytes[length - 2] << 8;
|
|
271
|
+
if (remaining >= 1) {
|
|
272
|
+
h ^= bytes[length - 1];
|
|
273
|
+
h = Math.imul(h, m);
|
|
274
|
+
}
|
|
275
|
+
h ^= h >>> 13;
|
|
276
|
+
h = Math.imul(h, m);
|
|
277
|
+
h ^= h >>> 15;
|
|
278
|
+
return h;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
//# sourceMappingURL=partitioning.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"partitioning.js","sourceRoot":"","sources":["../../../../../plugins/specweave-kafka/lib/utils/partitioning.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAyCH,MAAM,OAAO,4BAA4B;IAAzC;QACmB,kCAA6B,GAAG,EAAE,CAAC;QACnC,8BAAyB,GAAG,KAAK,CAAC;QAClC,kCAA6B,GAAG,CAAC,CAAC;IAuarD,CAAC;IAraC;;OAEG;IACH,OAAO,CAAC,GAA6B;QACnC,MAAM,cAAc,GAAG,GAAG,CAAC,cAAc,IAAI,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,GAAG,CAAC,eAAe,IAAI,GAAG,CAAC;QACjD,MAAM,kBAAkB,GAAG,GAAG,CAAC,yBAAyB,GAAG,cAAc,CAAC;QAE1E,6BAA6B;QAC7B,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAEvE,sCAAsC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAEtE,kDAAkD;QAClD,IAAI,uBAAuB,GAAG,CAAC,CAAC;QAChC,IAAI,GAAG,CAAC,oBAAoB,EAAE,CAAC;YAC7B,uBAAuB,GAAG,IAAI,CAAC,IAAI,CACjC,GAAG,CAAC,oBAAoB,GAAG,IAAI,CAAC,6BAA6B,CAC9D,CAAC;QACJ,CAAC;QAED,oDAAoD;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CACpC,kBAAkB,GAAG,IAAI,CAAC,yBAAyB,CACpD,CAAC;QAEF,gDAAgD;QAChD,MAAM,wBAAwB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAExG,mCAAmC;QACnC,IAAI,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAClC,uBAAuB,EACvB,oBAAoB,EACpB,wBAAwB,CACzB,CAAC;QAEF,uEAAuE;QACvE,IAAI,qBAAqB,IAAI,EAAE,EAAE,CAAC;YAChC,qBAAqB,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,+BAA+B;QACrG,CAAC;QAED,4BAA4B;QAC5B,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;QAE3D,yBAAyB;QACzB,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,oBAAoB,CACpE,OAAO,EACP,GAAG,CAAC,gBAAgB,EACpB,qBAAqB,CACtB,CAAC;QAEF,oBAAoB;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACrC,OAAO;YACP,qBAAqB;YACrB,qBAAqB,EAAE,GAAG,CAAC,qBAAqB;YAChD,GAAG;SACJ,CAAC,CAAC;QAEH,0BAA0B;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QAE7E,OAAO;YACL,qBAAqB;YACrB,WAAW;YACX,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,QAAQ;SACT,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAgF;QACjG,MAAM,WAAW,GAAG,GAAG,CAAC,oBAAoB,CAAC;QAC7C,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC;QAE3C,oCAAoC;QACpC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;QACtD,MAAM,eAAe,GAAG,cAAc,GAAG,UAAU,CAAC;QAEpD,wBAAwB;QACxB,IAAI,YAAsD,CAAC;QAC3D,IAAI,eAAe,GAAG,GAAG,EAAE,CAAC;YAC1B,YAAY,GAAG,SAAS,CAAC,CAAC,yBAAyB;QACrD,CAAC;aAAM,IAAI,eAAe,GAAG,GAAG,EAAE,CAAC;YACjC,YAAY,GAAG,QAAQ,CAAC,CAAC,wBAAwB;QACnD,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,iBAAiB,CAAC,CAAC,wBAAwB;QAC5D,CAAC;QAED,sBAAsB;QACtB,IAAI,WAAsC,CAAC;QAC3C,IAAI,WAAW,GAAG,KAAK,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACtD,WAAW,GAAG,KAAK,CAAC;QACtB,CAAC;aAAM,IAAI,WAAW,GAAG,IAAI,IAAI,YAAY,KAAK,iBAAiB,EAAE,CAAC;YACpE,WAAW,GAAG,QAAQ,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;QAED,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,SAAS;YAClB,YAAY,EAAE,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,iBAAiB;YAC7D,oBAAoB,EAAE,WAAW;YACjC,YAAY;YACZ,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,QAAgC,EAChC,gBAAyB;QAEzB,oEAAoE;QACpE,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACzC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YAErE,IAAI,QAAQ,KAAK,CAAC;gBAAE,OAAO,QAAQ,CAAC,CAAC,kBAAkB;YAEvD,0CAA0C;YAC1C,OAAO,CAAC,CAAC,oBAAoB,GAAG,CAAC,CAAC,oBAAoB,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAC1B,OAAoC,EACpC,gBAAyB,EACzB,cAAsB;QAEtB,qDAAqD;QACrD,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,EAAE,CAAC;YACtE,OAAO;gBACL,WAAW,EAAE,aAAa;gBAC1B,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,iGAAiG;aAC7G,CAAC;QACJ,CAAC;QAED,0EAA0E;QAC1E,IAAI,gBAAgB,IAAI,OAAO,IAAI,OAAO,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YAClE,OAAO;gBACL,WAAW,EAAE,eAAe;gBAC5B,QAAQ,EAAE,OAAO,CAAC,GAAG;gBACrB,SAAS,EAAE,yBAAyB,OAAO,CAAC,GAAG,mDAAmD,OAAO,CAAC,GAAG,mCAAmC;aACjJ,CAAC;QACJ,CAAC;QAED,uCAAuC;QACvC,IAAI,gBAAgB,IAAI,OAAO,IAAI,OAAO,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YAClE,OAAO;gBACL,WAAW,EAAE,aAAa;gBAC1B,QAAQ,EAAE,OAAO,CAAC,GAAG;gBACrB,SAAS,EAAE,GAAG,OAAO,CAAC,GAAG,QAAQ,OAAO,CAAC,YAAY,sBAAsB,OAAO,CAAC,WAAW,8CAA8C;aAC7I,CAAC;QACJ,CAAC;QAED,6DAA6D;QAC7D,IAAI,CAAC,gBAAgB,IAAI,OAAO,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAClE,OAAO;gBACL,WAAW,EAAE,aAAa;gBAC1B,QAAQ,EAAE,OAAO,CAAC,GAAG;gBACrB,SAAS,EAAE,GAAG,OAAO,CAAC,GAAG,yGAAyG;aACnI,CAAC;QACJ,CAAC;QAED,iDAAiD;QACjD,OAAO;YACL,WAAW,EAAE,oBAAoB;YACjC,QAAQ,EAAE,OAAO,EAAE,GAAG,IAAI,IAAI;YAC9B,SAAS,EAAE,2FAA2F;SACvG,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,MAKxB;QACC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,mBAAmB;QACnB,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YAC5D,QAAQ,CAAC,IAAI,CACX,iBAAiB,MAAM,CAAC,OAAO,CAAC,GAAG,4CAA4C,MAAM,CAAC,OAAO,CAAC,oBAAoB,IAAI;gBACtH,8CAA8C,CAC/C,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC9D,QAAQ,CAAC,IAAI,CACX,YAAY,MAAM,CAAC,OAAO,CAAC,GAAG,0EAA0E,CACzG,CAAC;QACJ,CAAC;QAED,2BAA2B;QAC3B,IAAI,MAAM,CAAC,qBAAqB,IAAI,MAAM,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;YAChG,QAAQ,CAAC,IAAI,CACX,gCAAgC,MAAM,CAAC,qBAAqB,2BAA2B,MAAM,CAAC,qBAAqB,KAAK;gBACxH,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,qBAAqB,GAAG,GAAG,EAAE,CAAC;YACvC,QAAQ,CAAC,IAAI,CACX,mBAAmB,MAAM,CAAC,qBAAqB,6EAA6E,CAC7H,CAAC;QACJ,CAAC;QAED,oBAAoB;QACpB,IAAI,MAAM,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACnD,QAAQ,CAAC,IAAI,CACX,qGAAqG,CACtG,CAAC;QACJ,CAAC;QAED,sBAAsB;QACtB,IAAI,MAAM,CAAC,GAAG,CAAC,oBAAoB,IAAI,MAAM,CAAC,GAAG,CAAC,oBAAoB,GAAG,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC,6BAA6B,EAAE,CAAC;YAC3I,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oBAAoB,GAAG,IAAI,CAAC,6BAA6B,CAAC,CAAC;YACzG,QAAQ,CAAC,IAAI,CACX,0BAA0B,MAAM,CAAC,GAAG,CAAC,oBAAoB,mBAAmB,gBAAgB,eAAe;gBAC3G,wBAAwB,MAAM,CAAC,qBAAqB,6BAA6B,CAClF,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,gBAAgB,CACtB,WAAmB,EACnB,QAAuB,EACvB,SAAiB;QAEjB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,QAAQ,WAAW,EAAE,CAAC;YACpB,KAAK,aAAa;gBAChB,QAAQ,CAAC,IAAI,CACX,kCAAkC,EAClC,uBAAuB,EACvB,aAAa,SAAS,IAAI,EAC1B,gBAAgB,EAChB,mBAAmB,QAAQ,oBAAoB,EAC/C,mCAAmC,EACnC,MAAM,EACN,KAAK,EACL,EAAE,EACF,oBAAoB,EACpB,+DAA+D,EAC/D,MAAM,SAAS,IAAI,EACnB,eAAe,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,IAAI,CAAC,sBAAsB,EACtE,aAAa,EACb,IAAI,EACJ,wBAAwB,CACzB,CAAC;gBACF,MAAM;YAER,KAAK,eAAe;gBAClB,QAAQ,CAAC,IAAI,CACX,iDAAiD,EACjD,mCAAmC,QAAQ,qDAAqD,EAChG,uBAAuB,EACvB,aAAa,SAAS,IAAI,EAC1B,gBAAgB,EAChB,uBAAuB,EACvB,mCAAmC,EACnC,MAAM,EACN,KAAK,EACL,EAAE,EACF,wCAAwC,EACxC,qDAAqD,QAAQ,MAAM,CACpE,CAAC;gBACF,MAAM;YAER,KAAK,oBAAoB;gBACvB,QAAQ,CAAC,IAAI,CACX,8BAA8B,EAC9B,yDAAyD,EACzD,aAAa,EACb,mEAAmE,EACnE,4EAA4E,EAC5E,iEAAiE,EACjE,MAAM,EACN,0BAA0B,EAC1B,mCAAmC,EACnC,2CAA2C,EAC3C,4CAA4C,EAC5C,OAAO,EACP,MAAM,EACN,mCAAmC,EACnC,oEAAoE,EACpE,KAAK,EACL,GAAG,EACH,EAAE,EACF,uBAAuB,EACvB,kEAAkE,CACnE,CAAC;gBACF,MAAM;YAER,KAAK,aAAa;gBAChB,QAAQ,CAAC,IAAI,CACX,2CAA2C,EAC3C,uBAAuB,EACvB,aAAa,SAAS,IAAI,EAC1B,gBAAgB,EAChB,2DAA2D,EAC3D,MAAM,EACN,KAAK,EACL,EAAE,EACF,oBAAoB,EACpB,+DAA+D,EAC/D,MAAM,SAAS,IAAI,EACnB,mBAAmB,EACnB,aAAa,EACb,IAAI,EACJ,wBAAwB,CACzB,CAAC;gBACF,MAAM;QACV,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,CAAS;QAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW;QAC5B,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,qBAAqB,CACnB,YAAsB,EACtB,cAAsB;QAEtB,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QAElD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC;YAClD,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;aACvD,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5B,SAAS;YACT,KAAK;YACL,UAAU,EAAE,CAAC,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,GAAG;SAChD,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAErC,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,IAAY;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5B,MAAM,IAAI,GAAG,UAAU,CAAC;QAExB,MAAM,CAAC,GAAG,UAAU,CAAC;QACrB,MAAM,CAAC,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACb,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpB,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,IAAI,SAAS,IAAI,CAAC;YAAE,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACjD,IAAI,SAAS,IAAI,CAAC;YAAE,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACnB,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACvB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtB,CAAC;QAED,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAEd,OAAO,CAAC,CAAC;IACX,CAAC;CACF"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cluster Sizing Calculator
|
|
3
|
+
*
|
|
4
|
+
* Calculates optimal Kafka cluster size based on workload requirements.
|
|
5
|
+
* Considers throughput, retention, replication, and growth factors.
|
|
6
|
+
*/
|
|
7
|
+
export interface SizingRequirements {
|
|
8
|
+
writeThroughputMBps: number;
|
|
9
|
+
readThroughputMBps: number;
|
|
10
|
+
retentionDays: number;
|
|
11
|
+
avgMessageSizeKB: number;
|
|
12
|
+
messagesPerDay: number;
|
|
13
|
+
replicationFactor: number;
|
|
14
|
+
minInsyncReplicas?: number;
|
|
15
|
+
topicCount: number;
|
|
16
|
+
avgPartitionsPerTopic: number;
|
|
17
|
+
growthFactor?: number;
|
|
18
|
+
targetCPUUtilization?: number;
|
|
19
|
+
targetDiskUtilization?: number;
|
|
20
|
+
}
|
|
21
|
+
export interface ClusterSizingRecommendation {
|
|
22
|
+
brokerCount: number;
|
|
23
|
+
totalPartitions: number;
|
|
24
|
+
partitionsPerBroker: number;
|
|
25
|
+
cpuCores: number;
|
|
26
|
+
ramGB: number;
|
|
27
|
+
diskGB: number;
|
|
28
|
+
networkGbps: number;
|
|
29
|
+
estimatedWriteThroughputMBps: number;
|
|
30
|
+
estimatedReadThroughputMBps: number;
|
|
31
|
+
estimatedLatencyP99Ms: number;
|
|
32
|
+
dailyDataVolumeMB: number;
|
|
33
|
+
totalStorageRequiredGB: number;
|
|
34
|
+
storageWithReplicationGB: number;
|
|
35
|
+
warnings: string[];
|
|
36
|
+
recommendations: string[];
|
|
37
|
+
}
|
|
38
|
+
export declare class ClusterSizingCalculator {
|
|
39
|
+
private readonly MAX_PARTITIONS_PER_BROKER;
|
|
40
|
+
private readonly SINGLE_PARTITION_WRITE_MBPS;
|
|
41
|
+
private readonly SINGLE_PARTITION_READ_MBPS;
|
|
42
|
+
private readonly NETWORK_OVERHEAD;
|
|
43
|
+
private readonly BASE_RAM_GB;
|
|
44
|
+
private readonly RAM_PER_PARTITION_MB;
|
|
45
|
+
private readonly RAM_PER_REPLICATION_MB;
|
|
46
|
+
/**
|
|
47
|
+
* Calculate recommended cluster size
|
|
48
|
+
*/
|
|
49
|
+
calculate(req: SizingRequirements): ClusterSizingRecommendation;
|
|
50
|
+
/**
|
|
51
|
+
* Calculate network bandwidth requirements
|
|
52
|
+
*/
|
|
53
|
+
private calculateNetworkBandwidth;
|
|
54
|
+
/**
|
|
55
|
+
* Estimate p99 latency based on partition count and disk size
|
|
56
|
+
*/
|
|
57
|
+
private estimateLatency;
|
|
58
|
+
/**
|
|
59
|
+
* Generate warnings for potential issues
|
|
60
|
+
*/
|
|
61
|
+
private generateWarnings;
|
|
62
|
+
/**
|
|
63
|
+
* Generate optimization recommendations
|
|
64
|
+
*/
|
|
65
|
+
private generateRecommendations;
|
|
66
|
+
/**
|
|
67
|
+
* Calculate disk IOPS requirements
|
|
68
|
+
*/
|
|
69
|
+
calculateDiskIOPS(writeThroughputMBps: number, replicationFactor: number): {
|
|
70
|
+
readIOPS: number;
|
|
71
|
+
writeIOPS: number;
|
|
72
|
+
recommendation: string;
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=sizing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sizing.d.ts","sourceRoot":"","sources":["../../../../../plugins/specweave-kafka/lib/utils/sizing.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,kBAAkB;IAEjC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAG3B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IAGvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,UAAU,EAAE,MAAM,CAAC;IACnB,qBAAqB,EAAE,MAAM,CAAC;IAG9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,2BAA2B;IAE1C,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAG5B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IAGpB,4BAA4B,EAAE,MAAM,CAAC;IACrC,2BAA2B,EAAE,MAAM,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;IAG9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,wBAAwB,EAAE,MAAM,CAAC;IAGjC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,uBAAuB;IAElC,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IAClD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAM;IACjD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAO;IAExC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAK;IACjC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAK;IAC1C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAK;IAE5C;;OAEG;IACH,SAAS,CAAC,GAAG,EAAE,kBAAkB,GAAG,2BAA2B;IAqG/D;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAiBjC;;OAEG;IACH,OAAO,CAAC,eAAe;IAoBvB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAyDxB;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA+D/B;;OAEG;IACH,iBAAiB,CAAC,mBAAmB,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG;QACzE,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;KACxB;CAwBF"}
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cluster Sizing Calculator
|
|
3
|
+
*
|
|
4
|
+
* Calculates optimal Kafka cluster size based on workload requirements.
|
|
5
|
+
* Considers throughput, retention, replication, and growth factors.
|
|
6
|
+
*/
|
|
7
|
+
export class ClusterSizingCalculator {
|
|
8
|
+
constructor() {
|
|
9
|
+
// Constants (empirical limits from production deployments)
|
|
10
|
+
this.MAX_PARTITIONS_PER_BROKER = 4000;
|
|
11
|
+
this.SINGLE_PARTITION_WRITE_MBPS = 20; // Conservative estimate
|
|
12
|
+
this.SINGLE_PARTITION_READ_MBPS = 40;
|
|
13
|
+
this.NETWORK_OVERHEAD = 1.3; // 30% overhead for protocol, replication
|
|
14
|
+
this.BASE_RAM_GB = 8; // Minimum OS + Kafka overhead
|
|
15
|
+
this.RAM_PER_PARTITION_MB = 1; // Page cache per partition
|
|
16
|
+
this.RAM_PER_REPLICATION_MB = 2; // Replica fetcher overhead
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Calculate recommended cluster size
|
|
20
|
+
*/
|
|
21
|
+
calculate(req) {
|
|
22
|
+
const growthFactor = req.growthFactor ?? 2.0;
|
|
23
|
+
const targetCPU = req.targetCPUUtilization ?? 0.6;
|
|
24
|
+
const targetDisk = req.targetDiskUtilization ?? 0.7;
|
|
25
|
+
// Apply growth factor to throughput
|
|
26
|
+
const writeThru = req.writeThroughputMBps * growthFactor;
|
|
27
|
+
const readThru = req.readThroughputMBps * growthFactor;
|
|
28
|
+
// Calculate total partitions needed
|
|
29
|
+
const totalPartitions = req.topicCount * req.avgPartitionsPerTopic;
|
|
30
|
+
// Calculate brokers needed based on throughput
|
|
31
|
+
const brokersForWrite = Math.ceil((writeThru * this.NETWORK_OVERHEAD) / (this.SINGLE_PARTITION_WRITE_MBPS * this.MAX_PARTITIONS_PER_BROKER / 10));
|
|
32
|
+
const brokersForRead = Math.ceil((readThru * this.NETWORK_OVERHEAD) / (this.SINGLE_PARTITION_READ_MBPS * this.MAX_PARTITIONS_PER_BROKER / 10));
|
|
33
|
+
// Calculate brokers needed based on partition count
|
|
34
|
+
const brokersForPartitions = Math.ceil(totalPartitions / this.MAX_PARTITIONS_PER_BROKER);
|
|
35
|
+
// Take maximum (most constraining factor)
|
|
36
|
+
let brokerCount = Math.max(brokersForWrite, brokersForRead, brokersForPartitions);
|
|
37
|
+
// Ensure minimum for replication factor
|
|
38
|
+
brokerCount = Math.max(brokerCount, req.replicationFactor);
|
|
39
|
+
// Ensure broker count allows for rack awareness (multiples of 3)
|
|
40
|
+
if (brokerCount > 3 && brokerCount % 3 !== 0) {
|
|
41
|
+
brokerCount = Math.ceil(brokerCount / 3) * 3;
|
|
42
|
+
}
|
|
43
|
+
// Calculate partitions per broker
|
|
44
|
+
const partitionsPerBroker = Math.ceil(totalPartitions / brokerCount);
|
|
45
|
+
// Calculate daily data volume
|
|
46
|
+
const dailyDataVolumeMB = (req.messagesPerDay * req.avgMessageSizeKB) / 1024;
|
|
47
|
+
// Calculate total storage needed
|
|
48
|
+
const totalStorageGB = (dailyDataVolumeMB * req.retentionDays) / 1024;
|
|
49
|
+
const storageWithReplicationGB = totalStorageGB * req.replicationFactor;
|
|
50
|
+
// Calculate disk per broker (with headroom)
|
|
51
|
+
const diskPerBrokerGB = Math.ceil((storageWithReplicationGB / brokerCount) / targetDisk);
|
|
52
|
+
// Calculate RAM per broker
|
|
53
|
+
const ramForPartitionsGB = (partitionsPerBroker * this.RAM_PER_PARTITION_MB) / 1024;
|
|
54
|
+
const ramForReplicationGB = (partitionsPerBroker * req.replicationFactor * this.RAM_PER_REPLICATION_MB) / 1024;
|
|
55
|
+
const ramGB = Math.ceil(this.BASE_RAM_GB + ramForPartitionsGB + ramForReplicationGB);
|
|
56
|
+
// Calculate CPU cores (rule: 1 core per 500 partitions + 8 base cores)
|
|
57
|
+
const cpuCores = Math.ceil(partitionsPerBroker / 500) + 8;
|
|
58
|
+
// Calculate network bandwidth needed
|
|
59
|
+
const networkGbps = this.calculateNetworkBandwidth(writeThru, readThru, brokerCount);
|
|
60
|
+
// Estimate performance
|
|
61
|
+
const estimatedWriteThroughputMBps = brokerCount * this.SINGLE_PARTITION_WRITE_MBPS * (partitionsPerBroker / 10);
|
|
62
|
+
const estimatedReadThroughputMBps = brokerCount * this.SINGLE_PARTITION_READ_MBPS * (partitionsPerBroker / 10);
|
|
63
|
+
const estimatedLatencyP99Ms = this.estimateLatency(partitionsPerBroker, diskPerBrokerGB);
|
|
64
|
+
// Generate warnings
|
|
65
|
+
const warnings = this.generateWarnings({
|
|
66
|
+
partitionsPerBroker,
|
|
67
|
+
brokerCount,
|
|
68
|
+
req,
|
|
69
|
+
diskPerBrokerGB
|
|
70
|
+
});
|
|
71
|
+
// Generate recommendations
|
|
72
|
+
const recommendations = this.generateRecommendations({
|
|
73
|
+
partitionsPerBroker,
|
|
74
|
+
brokerCount,
|
|
75
|
+
req,
|
|
76
|
+
ramGB,
|
|
77
|
+
diskPerBrokerGB
|
|
78
|
+
});
|
|
79
|
+
return {
|
|
80
|
+
brokerCount,
|
|
81
|
+
totalPartitions,
|
|
82
|
+
partitionsPerBroker,
|
|
83
|
+
cpuCores,
|
|
84
|
+
ramGB,
|
|
85
|
+
diskGB: diskPerBrokerGB,
|
|
86
|
+
networkGbps,
|
|
87
|
+
estimatedWriteThroughputMBps,
|
|
88
|
+
estimatedReadThroughputMBps,
|
|
89
|
+
estimatedLatencyP99Ms,
|
|
90
|
+
dailyDataVolumeMB,
|
|
91
|
+
totalStorageRequiredGB: totalStorageGB,
|
|
92
|
+
storageWithReplicationGB,
|
|
93
|
+
warnings,
|
|
94
|
+
recommendations
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Calculate network bandwidth requirements
|
|
99
|
+
*/
|
|
100
|
+
calculateNetworkBandwidth(writeMBps, readMBps, brokerCount) {
|
|
101
|
+
// Each broker handles replication traffic
|
|
102
|
+
const replicationTrafficMBps = writeMBps * 2; // Leader + follower writes
|
|
103
|
+
const totalTrafficMBps = writeMBps + readMBps + replicationTrafficMBps;
|
|
104
|
+
const perBrokerMBps = totalTrafficMBps / brokerCount;
|
|
105
|
+
// Convert to Gbps and round up
|
|
106
|
+
const gbps = Math.ceil((perBrokerMBps * 8) / 1000);
|
|
107
|
+
// Recommend common NIC sizes: 1, 10, 25, 40, 100 Gbps
|
|
108
|
+
if (gbps <= 1)
|
|
109
|
+
return 1;
|
|
110
|
+
if (gbps <= 10)
|
|
111
|
+
return 10;
|
|
112
|
+
if (gbps <= 25)
|
|
113
|
+
return 25;
|
|
114
|
+
if (gbps <= 40)
|
|
115
|
+
return 40;
|
|
116
|
+
return 100;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Estimate p99 latency based on partition count and disk size
|
|
120
|
+
*/
|
|
121
|
+
estimateLatency(partitionsPerBroker, diskGB) {
|
|
122
|
+
let latencyMs = 5; // Base latency (network + CPU)
|
|
123
|
+
// Add latency for partition count (more partitions = more overhead)
|
|
124
|
+
if (partitionsPerBroker > 2000) {
|
|
125
|
+
latencyMs += 5;
|
|
126
|
+
}
|
|
127
|
+
else if (partitionsPerBroker > 1000) {
|
|
128
|
+
latencyMs += 2;
|
|
129
|
+
}
|
|
130
|
+
// Add latency for disk size (larger disks = potential seek time)
|
|
131
|
+
if (diskGB > 10000) {
|
|
132
|
+
latencyMs += 10; // HDD territory
|
|
133
|
+
}
|
|
134
|
+
else if (diskGB > 5000) {
|
|
135
|
+
latencyMs += 5; // Large SSD
|
|
136
|
+
}
|
|
137
|
+
return latencyMs;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Generate warnings for potential issues
|
|
141
|
+
*/
|
|
142
|
+
generateWarnings(params) {
|
|
143
|
+
const warnings = [];
|
|
144
|
+
// Partition count warnings
|
|
145
|
+
if (params.partitionsPerBroker > 4000) {
|
|
146
|
+
warnings.push(`⚠️ ${params.partitionsPerBroker} partitions per broker exceeds recommended limit (4000). ` +
|
|
147
|
+
`Consider increasing broker count to ${Math.ceil(params.req.topicCount * params.req.avgPartitionsPerTopic / 4000)}.`);
|
|
148
|
+
}
|
|
149
|
+
if (params.partitionsPerBroker < 100) {
|
|
150
|
+
warnings.push(`⚠️ ${params.partitionsPerBroker} partitions per broker is very low. ` +
|
|
151
|
+
`You may be over-provisioned. Consider reducing to ${Math.max(3, Math.ceil(params.brokerCount / 2))} brokers.`);
|
|
152
|
+
}
|
|
153
|
+
// Broker count warnings
|
|
154
|
+
if (params.brokerCount < params.req.replicationFactor) {
|
|
155
|
+
warnings.push(`🚨 CRITICAL: ${params.brokerCount} brokers < replication factor (${params.req.replicationFactor}). ` +
|
|
156
|
+
`Minimum required: ${params.req.replicationFactor} brokers.`);
|
|
157
|
+
}
|
|
158
|
+
if (params.brokerCount === params.req.replicationFactor) {
|
|
159
|
+
warnings.push(`⚠️ Broker count equals replication factor. No fault tolerance! ` +
|
|
160
|
+
`Increase to at least ${params.req.replicationFactor + 1} brokers.`);
|
|
161
|
+
}
|
|
162
|
+
// Disk size warnings
|
|
163
|
+
if (params.diskPerBrokerGB > 10000) {
|
|
164
|
+
warnings.push(`⚠️ ${params.diskPerBrokerGB} GB per broker is very large. ` +
|
|
165
|
+
`Consider using HDD for cost savings or increasing broker count to reduce disk per broker.`);
|
|
166
|
+
}
|
|
167
|
+
// Replication warnings
|
|
168
|
+
if (params.req.minInsyncReplicas && params.req.minInsyncReplicas >= params.req.replicationFactor) {
|
|
169
|
+
warnings.push(`🚨 CRITICAL: min.insync.replicas (${params.req.minInsyncReplicas}) >= replication.factor (${params.req.replicationFactor}). ` +
|
|
170
|
+
`This will cause writes to fail! Set min.insync.replicas to ${params.req.replicationFactor - 1}.`);
|
|
171
|
+
}
|
|
172
|
+
return warnings;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Generate optimization recommendations
|
|
176
|
+
*/
|
|
177
|
+
generateRecommendations(params) {
|
|
178
|
+
const recommendations = [];
|
|
179
|
+
// Partition strategy
|
|
180
|
+
if (params.req.avgPartitionsPerTopic < 10) {
|
|
181
|
+
recommendations.push(`💡 Consider increasing partitions per topic to 12-24 for better parallelism and future growth.`);
|
|
182
|
+
}
|
|
183
|
+
if (params.req.avgPartitionsPerTopic > 100) {
|
|
184
|
+
recommendations.push(`💡 ${params.req.avgPartitionsPerTopic} partitions per topic is high. ` +
|
|
185
|
+
`Consider splitting into multiple topics or reducing partition count unless you need extreme parallelism.`);
|
|
186
|
+
}
|
|
187
|
+
// Hardware recommendations
|
|
188
|
+
if (params.diskPerBrokerGB > 5000) {
|
|
189
|
+
recommendations.push(`💡 Large disk requirement (${params.diskPerBrokerGB} GB). Consider using tiered storage (Kafka 2.8+) ` +
|
|
190
|
+
`to offload old data to S3/Azure Blob/GCS.`);
|
|
191
|
+
}
|
|
192
|
+
if (params.ramGB > 128) {
|
|
193
|
+
recommendations.push(`💡 High RAM requirement (${params.ramGB} GB). Ensure your infrastructure supports this. ` +
|
|
194
|
+
`Consider AWS i3en/i4i instances or equivalent.`);
|
|
195
|
+
}
|
|
196
|
+
// Replication recommendations
|
|
197
|
+
if (!params.req.minInsyncReplicas) {
|
|
198
|
+
recommendations.push(`💡 Set min.insync.replicas=2 for production (currently not configured). ` +
|
|
199
|
+
`This prevents data loss if a broker fails.`);
|
|
200
|
+
}
|
|
201
|
+
if (params.req.replicationFactor < 3) {
|
|
202
|
+
recommendations.push(`💡 Increase replication factor to 3 for production durability (currently ${params.req.replicationFactor}).`);
|
|
203
|
+
}
|
|
204
|
+
// Broker count recommendations
|
|
205
|
+
if (params.brokerCount % 3 !== 0 && params.brokerCount > 3) {
|
|
206
|
+
recommendations.push(`💡 Broker count (${params.brokerCount}) is not a multiple of 3. ` +
|
|
207
|
+
`Consider using ${Math.ceil(params.brokerCount / 3) * 3} brokers for better rack awareness.`);
|
|
208
|
+
}
|
|
209
|
+
return recommendations;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Calculate disk IOPS requirements
|
|
213
|
+
*/
|
|
214
|
+
calculateDiskIOPS(writeThroughputMBps, replicationFactor) {
|
|
215
|
+
// Assume 4KB blocks
|
|
216
|
+
const writeIOPS = (writeThroughputMBps * 1024) / 4;
|
|
217
|
+
const replicationWriteIOPS = writeIOPS * (replicationFactor - 1); // Follower writes
|
|
218
|
+
const readIOPS = writeIOPS * 0.5; // Conservative (depends on consumer lag)
|
|
219
|
+
const totalWriteIOPS = writeIOPS + replicationWriteIOPS;
|
|
220
|
+
const totalReadIOPS = readIOPS;
|
|
221
|
+
let recommendation = '';
|
|
222
|
+
if (totalWriteIOPS > 50000) {
|
|
223
|
+
recommendation = 'Use NVMe SSDs (500K+ IOPS)';
|
|
224
|
+
}
|
|
225
|
+
else if (totalWriteIOPS > 10000) {
|
|
226
|
+
recommendation = 'Use Premium SSD (20K-64K IOPS)';
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
recommendation = 'Standard SSD (3K-6K IOPS) sufficient';
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
readIOPS: Math.ceil(totalReadIOPS),
|
|
233
|
+
writeIOPS: Math.ceil(totalWriteIOPS),
|
|
234
|
+
recommendation
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
//# sourceMappingURL=sizing.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sizing.js","sourceRoot":"","sources":["../../../../../plugins/specweave-kafka/lib/utils/sizing.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAqDH,MAAM,OAAO,uBAAuB;IAApC;QACE,2DAA2D;QAC1C,8BAAyB,GAAG,IAAI,CAAC;QACjC,gCAA2B,GAAG,EAAE,CAAC,CAAC,wBAAwB;QAC1D,+BAA0B,GAAG,EAAE,CAAC;QAChC,qBAAgB,GAAG,GAAG,CAAC,CAAC,yCAAyC;QAEjE,gBAAW,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC/C,yBAAoB,GAAG,CAAC,CAAC,CAAC,2BAA2B;QACrD,2BAAsB,GAAG,CAAC,CAAC,CAAC,2BAA2B;IAkT1E,CAAC;IAhTC;;OAEG;IACH,SAAS,CAAC,GAAuB;QAC/B,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,CAAC,oBAAoB,IAAI,GAAG,CAAC;QAClD,MAAM,UAAU,GAAG,GAAG,CAAC,qBAAqB,IAAI,GAAG,CAAC;QAEpD,oCAAoC;QACpC,MAAM,SAAS,GAAG,GAAG,CAAC,mBAAmB,GAAG,YAAY,CAAC;QACzD,MAAM,QAAQ,GAAG,GAAG,CAAC,kBAAkB,GAAG,YAAY,CAAC;QAEvD,oCAAoC;QACpC,MAAM,eAAe,GAAG,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,qBAAqB,CAAC;QAEnE,+CAA+C;QAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAC/B,CAAC,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAC/G,CAAC;QACF,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAC9B,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,GAAG,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAC7G,CAAC;QAEF,oDAAoD;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAEzF,0CAA0C;QAC1C,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,cAAc,EAAE,oBAAoB,CAAC,CAAC;QAElF,wCAAwC;QACxC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAE3D,iEAAiE;QACjE,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,CAAC;QAED,kCAAkC;QAClC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC,CAAC;QAErE,8BAA8B;QAC9B,MAAM,iBAAiB,GAAG,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QAE7E,iCAAiC;QACjC,MAAM,cAAc,GAAG,CAAC,iBAAiB,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;QACtE,MAAM,wBAAwB,GAAG,cAAc,GAAG,GAAG,CAAC,iBAAiB,CAAC;QAExE,4CAA4C;QAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAC/B,CAAC,wBAAwB,GAAG,WAAW,CAAC,GAAG,UAAU,CACtD,CAAC;QAEF,2BAA2B;QAC3B,MAAM,kBAAkB,GAAG,CAAC,mBAAmB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,IAAI,CAAC;QACpF,MAAM,mBAAmB,GAAG,CAAC,mBAAmB,GAAG,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,IAAI,CAAC;QAC/G,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,kBAAkB,GAAG,mBAAmB,CAAC,CAAC;QAErF,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAE1D,qCAAqC;QACrC,MAAM,WAAW,GAAG,IAAI,CAAC,yBAAyB,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAErF,uBAAuB;QACvB,MAAM,4BAA4B,GAAG,WAAW,GAAG,IAAI,CAAC,2BAA2B,GAAG,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QACjH,MAAM,2BAA2B,GAAG,WAAW,GAAG,IAAI,CAAC,0BAA0B,GAAG,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QAC/G,MAAM,qBAAqB,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QAEzF,oBAAoB;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACrC,mBAAmB;YACnB,WAAW;YACX,GAAG;YACH,eAAe;SAChB,CAAC,CAAC;QAEH,2BAA2B;QAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,uBAAuB,CAAC;YACnD,mBAAmB;YACnB,WAAW;YACX,GAAG;YACH,KAAK;YACL,eAAe;SAChB,CAAC,CAAC;QAEH,OAAO;YACL,WAAW;YACX,eAAe;YACf,mBAAmB;YACnB,QAAQ;YACR,KAAK;YACL,MAAM,EAAE,eAAe;YACvB,WAAW;YACX,4BAA4B;YAC5B,2BAA2B;YAC3B,qBAAqB;YACrB,iBAAiB;YACjB,sBAAsB,EAAE,cAAc;YACtC,wBAAwB;YACxB,QAAQ;YACR,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,yBAAyB,CAAC,SAAiB,EAAE,QAAgB,EAAE,WAAmB;QACxF,0CAA0C;QAC1C,MAAM,sBAAsB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,2BAA2B;QACzE,MAAM,gBAAgB,GAAG,SAAS,GAAG,QAAQ,GAAG,sBAAsB,CAAC;QACvE,MAAM,aAAa,GAAG,gBAAgB,GAAG,WAAW,CAAC;QAErD,+BAA+B;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAEnD,sDAAsD;QACtD,IAAI,IAAI,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QACxB,IAAI,IAAI,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC1B,IAAI,IAAI,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC1B,IAAI,IAAI,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,mBAA2B,EAAE,MAAc;QACjE,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,+BAA+B;QAElD,oEAAoE;QACpE,IAAI,mBAAmB,GAAG,IAAI,EAAE,CAAC;YAC/B,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;aAAM,IAAI,mBAAmB,GAAG,IAAI,EAAE,CAAC;YACtC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QAED,iEAAiE;QACjE,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;YACnB,SAAS,IAAI,EAAE,CAAC,CAAC,gBAAgB;QACnC,CAAC;aAAM,IAAI,MAAM,GAAG,IAAI,EAAE,CAAC;YACzB,SAAS,IAAI,CAAC,CAAC,CAAC,YAAY;QAC9B,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,MAKxB;QACC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,2BAA2B;QAC3B,IAAI,MAAM,CAAC,mBAAmB,GAAG,IAAI,EAAE,CAAC;YACtC,QAAQ,CAAC,IAAI,CACX,OAAO,MAAM,CAAC,mBAAmB,2DAA2D;gBAC5F,uCAAuC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,CACrH,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,mBAAmB,GAAG,GAAG,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CACX,OAAO,MAAM,CAAC,mBAAmB,sCAAsC;gBACvE,qDAAqD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,WAAW,CAC/G,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,IAAI,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CACX,gBAAgB,MAAM,CAAC,WAAW,kCAAkC,MAAM,CAAC,GAAG,CAAC,iBAAiB,KAAK;gBACrG,qBAAqB,MAAM,CAAC,GAAG,CAAC,iBAAiB,WAAW,CAC7D,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACxD,QAAQ,CAAC,IAAI,CACX,kEAAkE;gBAClE,wBAAwB,MAAM,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,WAAW,CACpE,CAAC;QACJ,CAAC;QAED,qBAAqB;QACrB,IAAI,MAAM,CAAC,eAAe,GAAG,KAAK,EAAE,CAAC;YACnC,QAAQ,CAAC,IAAI,CACX,OAAO,MAAM,CAAC,eAAe,gCAAgC;gBAC7D,2FAA2F,CAC5F,CAAC;QACJ,CAAC;QAED,uBAAuB;QACvB,IAAI,MAAM,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACjG,QAAQ,CAAC,IAAI,CACX,qCAAqC,MAAM,CAAC,GAAG,CAAC,iBAAiB,4BAA4B,MAAM,CAAC,GAAG,CAAC,iBAAiB,KAAK;gBAC9H,8DAA8D,MAAM,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,GAAG,CAClG,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,MAM/B;QACC,MAAM,eAAe,GAAa,EAAE,CAAC;QAErC,qBAAqB;QACrB,IAAI,MAAM,CAAC,GAAG,CAAC,qBAAqB,GAAG,EAAE,EAAE,CAAC;YAC1C,eAAe,CAAC,IAAI,CAClB,gGAAgG,CACjG,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,GAAG,CAAC,qBAAqB,GAAG,GAAG,EAAE,CAAC;YAC3C,eAAe,CAAC,IAAI,CAClB,MAAM,MAAM,CAAC,GAAG,CAAC,qBAAqB,iCAAiC;gBACvE,0GAA0G,CAC3G,CAAC;QACJ,CAAC;QAED,2BAA2B;QAC3B,IAAI,MAAM,CAAC,eAAe,GAAG,IAAI,EAAE,CAAC;YAClC,eAAe,CAAC,IAAI,CAClB,8BAA8B,MAAM,CAAC,eAAe,mDAAmD;gBACvG,2CAA2C,CAC5C,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC;YACvB,eAAe,CAAC,IAAI,CAClB,4BAA4B,MAAM,CAAC,KAAK,kDAAkD;gBAC1F,gDAAgD,CACjD,CAAC;QACJ,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YAClC,eAAe,CAAC,IAAI,CAClB,0EAA0E;gBAC1E,4CAA4C,CAC7C,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;YACrC,eAAe,CAAC,IAAI,CAClB,4EAA4E,MAAM,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAC7G,CAAC;QACJ,CAAC;QAED,+BAA+B;QAC/B,IAAI,MAAM,CAAC,WAAW,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YAC3D,eAAe,CAAC,IAAI,CAClB,oBAAoB,MAAM,CAAC,WAAW,4BAA4B;gBAClE,kBAAkB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,CAAC,qCAAqC,CAC7F,CAAC;QACJ,CAAC;QAED,OAAO,eAAe,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,mBAA2B,EAAE,iBAAyB;QAKtE,oBAAoB;QACpB,MAAM,SAAS,GAAG,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,oBAAoB,GAAG,SAAS,GAAG,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB;QACpF,MAAM,QAAQ,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC,yCAAyC;QAE3E,MAAM,cAAc,GAAG,SAAS,GAAG,oBAAoB,CAAC;QACxD,MAAM,aAAa,GAAG,QAAQ,CAAC;QAE/B,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,IAAI,cAAc,GAAG,KAAK,EAAE,CAAC;YAC3B,cAAc,GAAG,4BAA4B,CAAC;QAChD,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,EAAE,CAAC;YAClC,cAAc,GAAG,gCAAgC,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,sCAAsC,CAAC;QAC1D,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC;YAClC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;YACpC,cAAc;SACf,CAAC;IACJ,CAAC;CACF"}
|