theprotocol-sdk 0.8.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. theprotocol_sdk-0.9.0/.gitignore +3 -0
  2. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/PKG-INFO +11 -1
  3. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/README.md +10 -0
  4. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/Dockerfile +2 -0
  5. theprotocol_sdk-0.9.0/proxy/compose.migrate.yaml +33 -0
  6. theprotocol_sdk-0.9.0/proxy/migrations/README.md +25 -0
  7. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/README.md +66 -0
  8. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/fake/make_export.py +40 -0
  9. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/fake/make_target.py +26 -0
  10. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/fake/target_postgresql.sql +10 -0
  11. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/fake/target_sqlite.sql +9 -0
  12. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/migration.yaml +30 -0
  13. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/proxy.yaml +40 -0
  14. theprotocol_sdk-0.9.0/proxy/migrations/csv-to-postgres/sample/orders.csv +361 -0
  15. theprotocol_sdk-0.9.0/proxy/migrations/make_samples.py +56 -0
  16. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/README.md +58 -0
  17. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/fake/make_legacy.py +36 -0
  18. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/fake/make_target.py +26 -0
  19. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/fake/target_postgresql.sql +9 -0
  20. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/fake/target_sqlite.sql +8 -0
  21. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/migration.yaml +38 -0
  22. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/proxy.yaml +42 -0
  23. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-postgres/sample/customers.jsonl +480 -0
  24. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-rest/README.md +53 -0
  25. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-rest/fake/make_legacy.py +36 -0
  26. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-rest/fake/products_api.py +123 -0
  27. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-rest/migration.yaml +35 -0
  28. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-rest/proxy.yaml +34 -0
  29. theprotocol_sdk-0.9.0/proxy/migrations/sql-to-rest/sample/products.jsonl +240 -0
  30. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/lyceum/README.md +2 -2
  31. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/lyceum/proxy.yaml +1 -1
  32. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/meridian/README.md +2 -2
  33. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/meridian/proxy.yaml +1 -1
  34. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/trunk/proxy.yaml +1 -1
  35. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/pyproject.toml +1 -1
  36. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/__init__.py +1 -1
  37. theprotocol_sdk-0.9.0/src/theprotocol/proxy/catalog.py +80 -0
  38. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/cli.py +20 -3
  39. theprotocol_sdk-0.9.0/src/theprotocol/proxy/mapping.py +397 -0
  40. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/__init__.py +8 -0
  41. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/cli.py +164 -0
  42. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/commands.py +1012 -0
  43. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/destinations.py +200 -0
  44. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/guard.py +176 -0
  45. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/params.py +181 -0
  46. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/reconcile.py +396 -0
  47. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/reports.py +182 -0
  48. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/sample.py +132 -0
  49. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/schema.py +297 -0
  50. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/scratch.py +91 -0
  51. theprotocol_sdk-0.9.0/src/theprotocol/proxy/migrate/skeletons.py +152 -0
  52. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/plugins.py +7 -1
  53. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/quantum/plugins.py +1 -1
  54. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/runtime.py +86 -10
  55. theprotocol_sdk-0.9.0/src/theprotocol/proxy/sinks/__init__.py +15 -0
  56. theprotocol_sdk-0.9.0/src/theprotocol/proxy/sinks/_json.py +57 -0
  57. theprotocol_sdk-0.9.0/src/theprotocol/proxy/sinks/file_out.py +302 -0
  58. theprotocol_sdk-0.9.0/src/theprotocol/proxy/sinks/rest.py +136 -0
  59. theprotocol_sdk-0.9.0/src/theprotocol/proxy/sinks/sql_upsert.py +583 -0
  60. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sources/filedrop.py +19 -0
  61. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sources/sqlpoll.py +23 -6
  62. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/spec.py +26 -3
  63. theprotocol_sdk-0.9.0/tests/data/mapping_vectors.json +2780 -0
  64. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/mqtt_broker.sh +0 -0
  65. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/sftp_server/run.sh +0 -0
  66. theprotocol_sdk-0.9.0/tests/test_proxy_catalog.py +94 -0
  67. theprotocol_sdk-0.9.0/tests/test_proxy_mapping_transforms.py +391 -0
  68. theprotocol_sdk-0.9.0/tests/test_proxy_mapping_vectors.py +100 -0
  69. theprotocol_sdk-0.9.0/tests/test_proxy_migrate_destinations.py +232 -0
  70. theprotocol_sdk-0.9.0/tests/test_proxy_migrate_failfast.py +151 -0
  71. theprotocol_sdk-0.9.0/tests/test_proxy_migrate_kit.py +209 -0
  72. theprotocol_sdk-0.9.0/tests/test_proxy_migrate_registry.py +339 -0
  73. theprotocol_sdk-0.9.0/tests/test_proxy_migrate_run.py +430 -0
  74. theprotocol_sdk-0.9.0/tests/test_proxy_migrate_sample.py +105 -0
  75. theprotocol_sdk-0.9.0/tests/test_proxy_migrations_starters.py +516 -0
  76. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_recipes.py +8 -2
  77. theprotocol_sdk-0.9.0/tests/test_proxy_sink_file_out.py +247 -0
  78. theprotocol_sdk-0.9.0/tests/test_proxy_sink_rest.py +178 -0
  79. theprotocol_sdk-0.9.0/tests/test_proxy_sink_sql_upsert.py +656 -0
  80. theprotocol_sdk-0.9.0/tests/test_proxy_sinks_decimal.py +241 -0
  81. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_filedrop.py +126 -0
  82. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_sqlpoll.py +106 -0
  83. theprotocol_sdk-0.9.0/tests/test_proxy_status_report.py +234 -0
  84. theprotocol_sdk-0.8.0/src/theprotocol/proxy/mapping.py +0 -95
  85. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/LICENSE +0 -0
  86. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/examples/simple_agent.py +0 -0
  87. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/Dockerfile.dockerignore +0 -0
  88. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/README.md +0 -0
  89. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/compose.yaml +0 -0
  90. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/proxy.example.yaml +0 -0
  91. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/proxy.outbound.example.yaml +0 -0
  92. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/proxy.quantum.example.yaml +0 -0
  93. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/README.md +0 -0
  94. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/bazaar/README.md +0 -0
  95. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/bazaar/fake/fake_shop.py +0 -0
  96. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/bazaar/fake/mock_wms.py +0 -0
  97. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/bazaar/proxy.yaml +0 -0
  98. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/dynamo/README.md +0 -0
  99. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/dynamo/fake/fake_meters.py +0 -0
  100. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/dynamo/proxy.yaml +0 -0
  101. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/hansa/README.md +0 -0
  102. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/hansa/fake/fake_tos.py +0 -0
  103. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/hansa/fake/mock_erp.py +0 -0
  104. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/hansa/proxy.yaml +0 -0
  105. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/keystone/README.md +0 -0
  106. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/keystone/fake/fake_logger.py +0 -0
  107. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/keystone/proxy.yaml +0 -0
  108. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/lyceum/fake/fake_sis.py +0 -0
  109. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/meridian/fake/fake_oms.py +0 -0
  110. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/scrivener/README.md +0 -0
  111. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/scrivener/fake/fake_pms.py +0 -0
  112. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/scrivener/proxy.yaml +0 -0
  113. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/trunk/README.md +0 -0
  114. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/proxy/recipes/trunk/fake/fake_mediation.py +0 -0
  115. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/agent/__init__.py +0 -0
  116. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/agent/base.py +0 -0
  117. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/agent/caller_errors.py +0 -0
  118. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/agent/decorators.py +0 -0
  119. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/agent/router.py +0 -0
  120. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/agent/task_store.py +0 -0
  121. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/agent/well_known.py +0 -0
  122. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/auth/__init__.py +0 -0
  123. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/auth/did_jwt.py +0 -0
  124. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/auth/jws_card.py +0 -0
  125. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/auth/jwt_verifier.py +0 -0
  126. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/authz.py +0 -0
  127. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/__init__.py +0 -0
  128. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/acp.py +0 -0
  129. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/anp/__init__.py +0 -0
  130. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/anp/auth.py +0 -0
  131. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/anp/bridge.py +0 -0
  132. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/anp/did_wba.py +0 -0
  133. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/anp/translation.py +0 -0
  134. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/base.py +0 -0
  135. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/google_a2a.py +0 -0
  136. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/bridges/mcp.py +0 -0
  137. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/client/__init__.py +0 -0
  138. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/client/a2a_client.py +0 -0
  139. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/client/credentials.py +0 -0
  140. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/dids.py +0 -0
  141. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/exceptions.py +0 -0
  142. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/exchange/__init__.py +0 -0
  143. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/exchange/client.py +0 -0
  144. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/forge.py +0 -0
  145. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/guild.py +0 -0
  146. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/messenger.py +0 -0
  147. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/models/__init__.py +0 -0
  148. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/models/a2a_protocol.py +0 -0
  149. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/models/agent_card.py +0 -0
  150. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/models/v1_compat.py +0 -0
  151. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/mtls.py +0 -0
  152. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/payment/__init__.py +0 -0
  153. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/payment/authenticator.py +0 -0
  154. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/payment/client.py +0 -0
  155. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/payment/mtls_client.py +0 -0
  156. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/payment/verifier.py +0 -0
  157. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/__init__.py +0 -0
  158. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/actions_builtin.py +0 -0
  159. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/evidence.py +0 -0
  160. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/identity.py +0 -0
  161. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/logs.py +0 -0
  162. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/outbox.py +0 -0
  163. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/quantum/__init__.py +0 -0
  164. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/quantum/backends.py +0 -0
  165. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/quantum/card.py +0 -0
  166. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/quantum/jobs.py +0 -0
  167. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/quantum/meter.py +0 -0
  168. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/quantum/qasm.py +0 -0
  169. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/registry.py +0 -0
  170. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/service.py +0 -0
  171. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sinks_builtin.py +0 -0
  172. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sources/__init__.py +0 -0
  173. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sources/logtail.py +0 -0
  174. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sources/mqtt.py +0 -0
  175. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sources/sftp.py +0 -0
  176. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/sources_builtin.py +0 -0
  177. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/proxy/testing.py +0 -0
  178. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/quantum/__init__.py +0 -0
  179. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/quantum/models.py +0 -0
  180. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/quantum/quantum_workload_v1.0.schema.json +0 -0
  181. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/quantum/receipts.py +0 -0
  182. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/quantum/verify.py +0 -0
  183. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/registry/__init__.py +0 -0
  184. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/registry/card_verify.py +0 -0
  185. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/registry/cost_composition.py +0 -0
  186. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/transfer/__init__.py +0 -0
  187. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/src/theprotocol/transfer/client.py +0 -0
  188. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/mqtt_fake.py +0 -0
  189. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/proxy_fake.py +0 -0
  190. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/registry/__init__.py +0 -0
  191. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/registry/test_card_verify.py +0 -0
  192. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/registry/test_cost_composition.py +0 -0
  193. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/sftp_server/sshd_config +0 -0
  194. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_a2a_client_token.py +0 -0
  195. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_a2a_client_v1_card.py +0 -0
  196. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_agent.py +0 -0
  197. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_artifact_roundtrip.py +0 -0
  198. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_auth.py +0 -0
  199. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_authz.py +0 -0
  200. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_bridge_acp.py +0 -0
  201. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_bridge_anp.py +0 -0
  202. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_bridge_google_a2a.py +0 -0
  203. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_bridge_mcp.py +0 -0
  204. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_caller_errors_v0653.py +0 -0
  205. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_credentials.py +0 -0
  206. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_dependency_binding.py +0 -0
  207. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_did_jwt.py +0 -0
  208. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_dids.py +0 -0
  209. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_exchange_launch.py +0 -0
  210. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_forge.py +0 -0
  211. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_guild.py +0 -0
  212. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_jws_card.py +0 -0
  213. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_messenger.py +0 -0
  214. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_models.py +0 -0
  215. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_mtls_client_presents_cert.py +0 -0
  216. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_mtls_enroll.py +0 -0
  217. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_actions.py +0 -0
  218. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_builtins.py +0 -0
  219. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_cli.py +0 -0
  220. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_core.py +0 -0
  221. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_events.py +0 -0
  222. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_plugins.py +0 -0
  223. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_quantum_backend.py +0 -0
  224. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_quantum_card.py +0 -0
  225. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_quantum_cli.py +0 -0
  226. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_quantum_meter.py +0 -0
  227. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_quantum_plugins.py +0 -0
  228. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_sinks.py +0 -0
  229. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_logtail.py +0 -0
  230. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_mqtt.py +0 -0
  231. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_mqtt_broker.py +0 -0
  232. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_sftp.py +0 -0
  233. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_sftp_server.py +0 -0
  234. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_sftp_transport.py +0 -0
  235. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_source_sqlpoll_engines.py +0 -0
  236. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_proxy_spec.py +0 -0
  237. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_quantum_receipts.py +0 -0
  238. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_quantum_workload.py +0 -0
  239. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_release_version.py +0 -0
  240. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_transfer.py +0 -0
  241. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_v1_compat.py +0 -0
  242. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_verifier_reuse.py +0 -0
  243. {theprotocol_sdk-0.8.0 → theprotocol_sdk-0.9.0}/tests/test_well_known.py +0 -0
@@ -0,0 +1,3 @@
1
+ # CLAUDE.md-class doctrine copies carry live credentials and stay out of git,
2
+ # same posture as the root CLAUDE.md (gitignored since March).
3
+ rescued_tmp_20260810/f1288a98/claude_new.md
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: theprotocol-sdk
3
- Version: 0.8.0
3
+ Version: 0.9.0
4
4
  Summary: Build sovereign AI agents with DID identity on TheProtocol. Speaks the A2A protocol (v1.0) natively, with bridges for Google A2A, ANP, and MCP. Works with Vertex AI, AWS AgentCore, LangGraph, CrewAI.
5
5
  Project-URL: Homepage, https://theprotocol.cloud
6
6
  Project-URL: Documentation, https://docs.theprotocol.cloud
@@ -263,6 +263,16 @@ Connectors are plugins on a published interface (`theprotocol.proxy.plugins`) wi
263
263
 
264
264
  ## Changelog
265
265
 
266
+ **0.9.0** (V0664: Migrate a system; uploaded to PyPI with V0664's roll)
267
+ - `tp-proxy migrate`: a migration from an old system (system A) to a new one (system B) as a kit, a folder with `migration.yaml` (the manifest, schema `theprotocol.migration/v1`, its parameters typed), `proxy.yaml` (the proxy spec that reads A and writes B), a README and a sample. The commands: `new` (a skeleton for sql, file or rest to sql, file or rest), `check`, `plan` (the inventory: rows and fields per source, mapping coverage), `dry-run` (into scratch targets under `KIT/.migrate/scratch/`, on the kit's sample with `--sample`, no account and no system needed), `load` (exactly once, resumable, only after `--confirm-destinations`: every destination is listed with who named it, and a template never writes anywhere you did not name), `reconcile` (both sides read back and compared by key; a zoned time in UTC, a number by value), `rollback` (the SQL targets' rows put back as they were before the load), and `certify` (one `migration.certificate` attestation carrying every report's hash and the kit's content hash). `plan`, `dry-run`, `load` and `reconcile` each write a report under `KIT/.migrate/`, with its own sha256. A command checks every source and writer first, ends with its reason when it cannot go on, and prints a progress line on a long run. An agent identity is needed only for `certify`, `publish`, `fork` and a paid `get`.
268
+ - The template library, on a registry with it switched on: `publish` (free or priced, forkable with a share of the fork's sales for the parent's author), `get` (a kit is written only when its content hash holds), and `fork` (the fork's slug is the parent's with `-fork`, or `--slug`; a retry returns the same fork). `certify` says whether the library counted the certificate.
269
+ - Three writers, as proxy sinks: `sql_upsert` (PostgreSQL, MySQL, SQL Server and SQLite; one atomic upsert per record, the row's state before the first write kept, so a rollback can restore it), `rest` (the mapped fields as the body, the record key as the Idempotency-Key) and `file_out` (CSV or JSON Lines with a `manifest.json` of rows and sha256, rotation, an SFTP upload of closed files). A literal password, bearer or SFTP credential in their config is refused at load.
270
+ - Field transforms in a mapping (value nodes, pure, no eval): `null_if`, `trim`, `case`, `regex` with `group`, `map` with `map_default`, `as` (text, int, decimal, float, bool, date, datetime, json, with `format` and `tz`), `round` (half even on a decimal), `concat` with `sep`, `default` and `required`. A refusal names the mapping's field and the rule, never the record's value; an unknown rule is refused when the spec loads. Existing mappings map exactly as before.
271
+ - `sqlpoll` can be ordered by its `key` alone, without a `cursor`.
272
+ - The file drop source (`filedrop`) reads a folder again. It read only the files present when the proxy started: a file dropped later was never read by a running proxy. A new `poll_seconds` option lists the folder every that many seconds (a file not settled yet is looked at again on the next pass; a row appended to a read file is read; none is read twice). Absent or 0 keeps what it did, one pass and finish, which is what `tp-proxy run --once` needs. The meridian and lyceum recipes set `poll_seconds: 30`.
273
+ - Three free starter kits, not in this package: in the repository under `theprotocol-sdk/proxy/migrations/`, and free in EU's template library (`tp-proxy migrate get <id> --registry https://eu.theprotocol.cloud`): `sql-to-postgres`, `csv-to-postgres` and `sql-to-rest` (with a stand-in products API). Each runs `tp-proxy migrate dry-run <kit> --sample` with no account and no system, and refuses a few sample rows on purpose to show its reasons.
274
+ - `tp-proxy catalog --json` prints every built-in plugin with its config schema, the spec's core schema, the migration manifest schema and the value-node vocabulary (the console builds its forms from it). A proxy can report its status to its registry (`service.report_every_s`, off unless set; names and counts only).
275
+
266
276
  **0.8.0** (V0659 and V0660: TheProtocol Proxy and the quantum workload pieces; uploaded to PyPI with V0660's roll)
267
277
  - TheProtocol Proxy: the module `theprotocol.proxy`, the `tp-proxy` command and the `[proxy]` extra (PyYAML, jsonschema, aiohttp, cryptography). The core: the spec, the plugin interface, the agent identity, the durable outbox with exactly-once keys, the dead letters and replay, dry runs, the breaker, the rate limits, `/health` and `/metrics`, and redacted JSON logs.
268
278
  - The proxy's built-in parts. Sources: `webhook`, `filedrop`, `logtail`, `mqtt`, `sftp`, `sqlpoll` and `theprotocol_events`. Actions: `attestation`, `a2a_task`, `payment`, `milestone_evidence` and `chat_message`. Sinks: `webhook`, `journal`, `calendar`, `email_digest` and `evidence`. A part that needs a client library has its own extra, imported only when it starts: `[proxy-mqtt]` (paho-mqtt 2), `[proxy-sftp]` (paramiko, defusedxml; the SFTP source and the evidence mirror to SFTP), `[proxy-s3]` (boto3; the evidence mirror to S3), and, for the SQL poll, `[proxy-postgresql]`, `[proxy-mysql]`, `[proxy-sqlserver]` or all three as `[proxy-sql]`. Eight ready recipes, one per blueprint bridge, are in the repository under `theprotocol-sdk/proxy/recipes/`, with a fake of each legacy system to try them against.
@@ -161,6 +161,16 @@ Connectors are plugins on a published interface (`theprotocol.proxy.plugins`) wi
161
161
 
162
162
  ## Changelog
163
163
 
164
+ **0.9.0** (V0664: Migrate a system; uploaded to PyPI with V0664's roll)
165
+ - `tp-proxy migrate`: a migration from an old system (system A) to a new one (system B) as a kit, a folder with `migration.yaml` (the manifest, schema `theprotocol.migration/v1`, its parameters typed), `proxy.yaml` (the proxy spec that reads A and writes B), a README and a sample. The commands: `new` (a skeleton for sql, file or rest to sql, file or rest), `check`, `plan` (the inventory: rows and fields per source, mapping coverage), `dry-run` (into scratch targets under `KIT/.migrate/scratch/`, on the kit's sample with `--sample`, no account and no system needed), `load` (exactly once, resumable, only after `--confirm-destinations`: every destination is listed with who named it, and a template never writes anywhere you did not name), `reconcile` (both sides read back and compared by key; a zoned time in UTC, a number by value), `rollback` (the SQL targets' rows put back as they were before the load), and `certify` (one `migration.certificate` attestation carrying every report's hash and the kit's content hash). `plan`, `dry-run`, `load` and `reconcile` each write a report under `KIT/.migrate/`, with its own sha256. A command checks every source and writer first, ends with its reason when it cannot go on, and prints a progress line on a long run. An agent identity is needed only for `certify`, `publish`, `fork` and a paid `get`.
166
+ - The template library, on a registry with it switched on: `publish` (free or priced, forkable with a share of the fork's sales for the parent's author), `get` (a kit is written only when its content hash holds), and `fork` (the fork's slug is the parent's with `-fork`, or `--slug`; a retry returns the same fork). `certify` says whether the library counted the certificate.
167
+ - Three writers, as proxy sinks: `sql_upsert` (PostgreSQL, MySQL, SQL Server and SQLite; one atomic upsert per record, the row's state before the first write kept, so a rollback can restore it), `rest` (the mapped fields as the body, the record key as the Idempotency-Key) and `file_out` (CSV or JSON Lines with a `manifest.json` of rows and sha256, rotation, an SFTP upload of closed files). A literal password, bearer or SFTP credential in their config is refused at load.
168
+ - Field transforms in a mapping (value nodes, pure, no eval): `null_if`, `trim`, `case`, `regex` with `group`, `map` with `map_default`, `as` (text, int, decimal, float, bool, date, datetime, json, with `format` and `tz`), `round` (half even on a decimal), `concat` with `sep`, `default` and `required`. A refusal names the mapping's field and the rule, never the record's value; an unknown rule is refused when the spec loads. Existing mappings map exactly as before.
169
+ - `sqlpoll` can be ordered by its `key` alone, without a `cursor`.
170
+ - The file drop source (`filedrop`) reads a folder again. It read only the files present when the proxy started: a file dropped later was never read by a running proxy. A new `poll_seconds` option lists the folder every that many seconds (a file not settled yet is looked at again on the next pass; a row appended to a read file is read; none is read twice). Absent or 0 keeps what it did, one pass and finish, which is what `tp-proxy run --once` needs. The meridian and lyceum recipes set `poll_seconds: 30`.
171
+ - Three free starter kits, not in this package: in the repository under `theprotocol-sdk/proxy/migrations/`, and free in EU's template library (`tp-proxy migrate get <id> --registry https://eu.theprotocol.cloud`): `sql-to-postgres`, `csv-to-postgres` and `sql-to-rest` (with a stand-in products API). Each runs `tp-proxy migrate dry-run <kit> --sample` with no account and no system, and refuses a few sample rows on purpose to show its reasons.
172
+ - `tp-proxy catalog --json` prints every built-in plugin with its config schema, the spec's core schema, the migration manifest schema and the value-node vocabulary (the console builds its forms from it). A proxy can report its status to its registry (`service.report_every_s`, off unless set; names and counts only).
173
+
164
174
  **0.8.0** (V0659 and V0660: TheProtocol Proxy and the quantum workload pieces; uploaded to PyPI with V0660's roll)
165
175
  - TheProtocol Proxy: the module `theprotocol.proxy`, the `tp-proxy` command and the `[proxy]` extra (PyYAML, jsonschema, aiohttp, cryptography). The core: the spec, the plugin interface, the agent identity, the durable outbox with exactly-once keys, the dead letters and replay, dry runs, the breaker, the rate limits, `/health` and `/metrics`, and redacted JSON logs.
166
176
  - The proxy's built-in parts. Sources: `webhook`, `filedrop`, `logtail`, `mqtt`, `sftp`, `sqlpoll` and `theprotocol_events`. Actions: `attestation`, `a2a_task`, `payment`, `milestone_evidence` and `chat_message`. Sinks: `webhook`, `journal`, `calendar`, `email_digest` and `evidence`. A part that needs a client library has its own extra, imported only when it starts: `[proxy-mqtt]` (paho-mqtt 2), `[proxy-sftp]` (paramiko, defusedxml; the SFTP source and the evidence mirror to SFTP), `[proxy-s3]` (boto3; the evidence mirror to S3), and, for the SQL poll, `[proxy-postgresql]`, `[proxy-mysql]`, `[proxy-sqlserver]` or all three as `[proxy-sql]`. Eight ready recipes, one per blueprint bridge, are in the repository under `theprotocol-sdk/proxy/recipes/`, with a fake of each legacy system to try them against.
@@ -3,6 +3,8 @@
3
3
  # Built from theprotocol-sdk/ (the build context), for both platforms:
4
4
  # docker buildx build --platform linux/amd64,linux/arm64 -f proxy/Dockerfile -t theprotocol-proxy:dev .
5
5
  # An evidence export mirrored to SFTP or S3 needs its client in the image: --build-arg EXTRAS=proxy,proxy-sftp,proxy-s3
6
+ # `tp-proxy migrate` is in every image; a migration needs the drivers of the systems it reads and writes:
7
+ # --build-arg EXTRAS=proxy,proxy-sql,proxy-sftp (PostgreSQL, MySQL, SQL Server, SFTP; see compose.migrate.yaml)
6
8
  ARG EXTRAS=proxy
7
9
  FROM python:3.12-slim AS build
8
10
  ARG EXTRAS
@@ -0,0 +1,33 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # A migration run with `tp-proxy migrate` in the proxy image (V0664, "Migrate a system"). The kit is a folder beside this
3
+ # file, mounted read-write at /kit: every step writes its report to /kit/.migrate/. Give the image's user (uid 10001)
4
+ # write access to it first: sudo chown -R 10001 ./kit
5
+ # docker compose -f compose.migrate.yaml build
6
+ # docker compose -f compose.migrate.yaml run --rm migrate check /kit
7
+ # docker compose -f compose.migrate.yaml run --rm migrate dry-run /kit --sample # no account, no system needed
8
+ # docker compose -f compose.migrate.yaml run --rm migrate plan /kit
9
+ # docker compose -f compose.migrate.yaml run --rm migrate load /kit --confirm-destinations
10
+ # docker compose -f compose.migrate.yaml run --rm migrate reconcile /kit
11
+ # The systems' credentials come from .env.secret (the kit's proxy.yaml names them env:...); never write them into the
12
+ # kit. A step's parameters: --set NAME=VALUE, or --params /kit/params.env.
13
+ services:
14
+ migrate:
15
+ build:
16
+ context: ..
17
+ dockerfile: proxy/Dockerfile
18
+ args:
19
+ EXTRAS: proxy,proxy-sql,proxy-sftp
20
+ image: theprotocol-proxy:migrate
21
+ entrypoint: ["tp-proxy", "migrate"]
22
+ command: ["check", "/kit"]
23
+ env_file:
24
+ - path: .env.secret
25
+ required: false
26
+ volumes:
27
+ - ./kit:/kit
28
+ read_only: true
29
+ tmpfs: [/tmp]
30
+ cap_drop: [ALL]
31
+ security_opt: ["no-new-privileges:true"]
32
+ healthcheck:
33
+ disable: true
@@ -0,0 +1,25 @@
1
+ # Migration templates: free starters
2
+
3
+ Three kits for `tp-proxy migrate` (V0664, "Migrate a system"), each a folder holding `migration.yaml` (the manifest,
4
+ schema `theprotocol.migration/v1`), `proxy.yaml` (the proxy spec that reads the old system and writes the new one),
5
+ `README.md`, a synthetic `sample/` (invented rows, a few refused on purpose so a dry run shows its reasons) and
6
+ stand-ins for both systems in `fake/`. Each one runs a dry run on its sample with no account and no system at all:
7
+
8
+ ```
9
+ # the kits are not in the pip package: clone the repository for them, or take the same kits free from EU's library
10
+ # with tp-proxy migrate get <id> --registry https://eu.theprotocol.cloud --out <kit>
11
+ pip install "theprotocol-sdk[proxy]"
12
+ tp-proxy migrate check proxy/migrations/<kit>
13
+ tp-proxy migrate dry-run proxy/migrations/<kit> --sample
14
+ ```
15
+
16
+ | kit | from | to | rollback |
17
+ |---|---|---|---|
18
+ | [sql-to-postgres/](sql-to-postgres/README.md) | a customers table (PostgreSQL, MySQL, SQL Server or SQLite) | `public.customers` on PostgreSQL, upserted by the old id; `dd.mm.yyyy` dates, status codes, padded names | auto |
19
+ | [csv-to-postgres/](csv-to-postgres/README.md) | a folder (or SFTP folder) of CSV order exports | `public.orders` on PostgreSQL; amounts rounded to 2, local Berlin times to UTC | auto |
20
+ | [sql-to-rest/](sql-to-rest/README.md) | a products table | a REST API (`POST /products`, the record key as Idempotency-Key), read back with `GET /products` | manual |
21
+
22
+ The samples are made by `make_samples.py` (a fixed seed; run it again after changing it).
23
+
24
+ Your own: `tp-proxy migrate new <name> --from sql|file --to sql|rest` writes a kit skeleton; copy one of these and
25
+ change its mapping; `tp-proxy migrate publish` puts it in the template library, free or for a price, forkable.
@@ -0,0 +1,66 @@
1
+ # csv-to-postgres: order exports (CSV files) to PostgreSQL
2
+
3
+ A free starter template for `tp-proxy migrate`: `migration.yaml` and `proxy.yaml` here, a synthetic export in
4
+ `sample/`, stand-ins for both systems in `fake/`.
5
+
6
+ **What it moves.** Every row of the CSV order exports in one folder into `public.orders` on PostgreSQL, upserted by
7
+ the order number (an export read again moves nothing twice). On the way: the amount becomes an exact decimal rounded
8
+ to 2 places (half to even), the old system's local time (`YYYY-MM-DD HH:MM`, Europe/Berlin unless `SOURCE_TZ` says
9
+ otherwise) becomes UTC, the currency is trimmed and upper-cased. A row whose amount is no number or whose time is in
10
+ another format is refused, named in the report, and not written.
11
+
12
+ **What it needs.** Read access to the export folder. On PostgreSQL, the table (`fake/target_postgresql.sql`) and a
13
+ login allowed to INSERT, UPDATE, SELECT and DELETE it and nothing else: `pip install
14
+ "theprotocol-sdk[proxy-postgresql]"`, its password as a reference (`--set TARGET_PASSWORD=env:TARGET_DB_PASSWORD`; a dry run needs none). The
15
+ exports must be complete before the run (it reads the folder once). On an SFTP server instead of a folder: install
16
+ `proxy-sftp`, replace the `exports` source with
17
+
18
+ ```yaml
19
+ exports:
20
+ type: sftp
21
+ config: {host: '${SFTP_HOST}', username: '${SFTP_USER}', private_key: '${SFTP_KEY}', known_hosts: '${SFTP_KNOWN_HOSTS}',
22
+ path: '${EXPORT_DIR}', format: csv, suffix: .csv, id_field: order_no, after_read: leave, poll_seconds: 0}
23
+ ```
24
+
25
+ and declare the four new parameters in `migration.yaml` (`SFTP_KEY` with `secret: true`, given as `file:/path`;
26
+ `SFTP_KNOWN_HOSTS` with no default, an absolute path to the server's host key file).
27
+ Every host and path stays a parameter, so the kit can be published (a template names no destination of its own);
28
+ `after_read: leave` keeps the files where they are, so `reconcile` can read them again, and the account needs read
29
+ access only. The sample stands in for either.
30
+
31
+ **Try it free.** No account and no system at all; the dry run reads the sample and writes only under `.migrate/`:
32
+
33
+ ```
34
+ pip install "theprotocol-sdk[proxy]"
35
+ tp-proxy migrate check proxy/migrations/csv-to-postgres
36
+ tp-proxy migrate dry-run proxy/migrations/csv-to-postgres --sample
37
+ ```
38
+
39
+ The sample has 360 orders, 5 refused on purpose (3 amounts that are no number, 2 times in another format). Then
40
+ against your own exports, still writing nothing to the database: `tp-proxy migrate dry-run
41
+ proxy/migrations/csv-to-postgres --set EXPORT_DIR=/path/to/exports`.
42
+
43
+ **The real run.** `load` first lists every place the rows go and who named it (you, with `--set` or `--params`, or
44
+ the template's own default); read the list, then confirm it with `--confirm-destinations`. A kit fetched from the
45
+ library refuses a destination its template named until you set that parameter yourself. From the SDK's folder (`K` is the kit; a `.migrate/` folder in it keeps the state and the reports).
46
+ Working files (databases, exports, `my.params`) stay outside the kit's folder: a kit holds its own text files only,
47
+ and `load` refuses a folder holding anything else. Rehearse on files first:
48
+
49
+ ```
50
+ K=proxy/migrations/csv-to-postgres; mkdir -p try
51
+ python3 $K/fake/make_export.py $K/sample/orders.csv try/export --files 3
52
+ python3 $K/fake/make_target.py try/target.db
53
+ export TARGET_DB_PASSWORD=unused-by-sqlite
54
+ printf 'EXPORT_DIR=try/export\nTARGET_ENGINE=sqlite\nTARGET_PATH=try/target.db\nTARGET_TABLE=orders\nTARGET_PASSWORD=env:TARGET_DB_PASSWORD\n' > try/my.params
55
+ tp-proxy migrate plan $K --params try/my.params # rows and fields in the folder, fields with no source
56
+ tp-proxy migrate load $K --params try/my.params --confirm-destinations # exactly once, resumable, old rows kept
57
+ tp-proxy migrate reconcile $K --params try/my.params # both sides read back, compared by order number
58
+ ```
59
+
60
+ Then against your own exports and PostgreSQL: a `my.params` with `EXPORT_DIR`, `TARGET_HOST`, `TARGET_PORT`,
61
+ `TARGET_DATABASE`, `TARGET_USER` and the password as a reference, the table made from `fake/target_postgresql.sql`;
62
+ the same three commands, and `tp-proxy migrate certify $K --identity proxy.yaml` for a migration certificate.
63
+
64
+ **Check it.** `reconcile` passes with missing, extra and mismatched all 0 on the time, the amount and the customer.
65
+ The way back: `tp-proxy migrate rollback $K --yes --params try/my.params` puts every order the load changed back as it was
66
+ and deletes the ones that were not there before.
@@ -0,0 +1,40 @@
1
+ #!/usr/bin/env python3
2
+ """A stand-in for the old system's export folder: the sample's orders split into a few CSV files, as a system that
3
+ exports once a month would leave them.
4
+
5
+ python3 fake/make_export.py sample/orders.csv export --files 3
6
+
7
+ Every value is the sample's own (invented).
8
+ """
9
+ import argparse
10
+ import csv
11
+ import io
12
+ from pathlib import Path
13
+
14
+
15
+ def split(sample, folder, files: int = 3) -> int:
16
+ rows = list(csv.DictReader(io.StringIO(Path(sample).read_text(encoding="utf-8"))))
17
+ if not rows:
18
+ return 0
19
+ out = Path(folder)
20
+ out.mkdir(parents=True, exist_ok=True)
21
+ per = -(-len(rows) // files)
22
+ for n in range(files):
23
+ part = rows[n * per:(n + 1) * per]
24
+ if not part:
25
+ break
26
+ buf = io.StringIO()
27
+ w = csv.DictWriter(buf, fieldnames=list(rows[0]), lineterminator="\n")
28
+ w.writeheader()
29
+ w.writerows(part)
30
+ (out / f"orders-export-{n + 1:02d}.csv").write_text(buf.getvalue(), encoding="utf-8")
31
+ return len(rows)
32
+
33
+
34
+ if __name__ == "__main__":
35
+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
36
+ ap.add_argument("sample")
37
+ ap.add_argument("folder")
38
+ ap.add_argument("--files", type=int, default=3)
39
+ a = ap.parse_args()
40
+ print(f"{split(a.sample, a.folder, a.files)} rows written to {a.folder}")
@@ -0,0 +1,26 @@
1
+ #!/usr/bin/env python3
2
+ """The rehearsal target: a SQLite file with the new table's columns (fake/target_sqlite.sql), for TARGET_ENGINE=sqlite.
3
+
4
+ python3 fake/make_target.py target.db
5
+
6
+ Then --set TARGET_ENGINE=sqlite --set TARGET_PATH=target.db --set TARGET_TABLE=orders.
7
+ """
8
+ import sqlite3
9
+ import sys
10
+ from pathlib import Path
11
+
12
+
13
+ def build(db) -> None:
14
+ c = sqlite3.connect(db)
15
+ try:
16
+ c.executescript((Path(__file__).with_name("target_sqlite.sql")).read_text(encoding="utf-8"))
17
+ c.commit()
18
+ finally:
19
+ c.close()
20
+
21
+
22
+ if __name__ == "__main__":
23
+ if len(sys.argv) != 2:
24
+ sys.exit(__doc__)
25
+ build(sys.argv[1])
26
+ print(f"{sys.argv[1]}: the table orders is ready")
@@ -0,0 +1,10 @@
1
+ -- The new table on PostgreSQL. The migration's login needs INSERT, UPDATE, SELECT and DELETE on it, nothing else:
2
+ -- GRANT INSERT, UPDATE, SELECT, DELETE ON public.orders TO tp_migrate;
3
+ CREATE TABLE IF NOT EXISTS public.orders (
4
+ order_no bigint PRIMARY KEY,
5
+ customer_id integer NOT NULL,
6
+ ordered_at timestamptz NOT NULL,
7
+ amount numeric(12, 2) NOT NULL,
8
+ currency char(3) NOT NULL,
9
+ status text NOT NULL
10
+ );
@@ -0,0 +1,9 @@
1
+ -- The rehearsal target (TARGET_ENGINE=sqlite, TARGET_TABLE=orders): the PostgreSQL table's columns in a local file.
2
+ CREATE TABLE IF NOT EXISTS orders (
3
+ order_no INTEGER PRIMARY KEY,
4
+ customer_id INTEGER NOT NULL,
5
+ ordered_at TEXT NOT NULL,
6
+ amount TEXT NOT NULL,
7
+ currency TEXT NOT NULL,
8
+ status TEXT NOT NULL
9
+ );
@@ -0,0 +1,30 @@
1
+ schema: theprotocol.migration/v1
2
+ name: csv-to-postgres
3
+ title: Order exports (CSV files) to PostgreSQL
4
+ summary: A folder of CSV order exports to public.orders, upserted by the order number; amounts rounded to 2, local Berlin times to UTC
5
+ from: {kind: file, what: 'a folder (or an SFTP folder) the old system exports orders into, one CSV file per export'}
6
+ to: {kind: sql, engines: [postgresql], what: public.orders}
7
+ parameters:
8
+ EXPORT_DIR: {description: the folder the old system's CSV exports are in, default: export}
9
+ SOURCE_TZ: {description: the zone the old system wrote its times in, default: Europe/Berlin}
10
+ TARGET_ENGINE: {description: 'postgresql; sqlite writes a local file instead, to rehearse', default: postgresql, enum: [postgresql, sqlite]}
11
+ TARGET_HOST: {description: the new database server, default: localhost}
12
+ TARGET_PORT: {description: its port, default: '5432'}
13
+ TARGET_DATABASE: {description: the new database, default: app}
14
+ TARGET_USER: {description: a login limited to the target table, default: tp_migrate}
15
+ TARGET_PASSWORD: {description: 'its password, as a reference such as env:TARGET_DB_PASSWORD', secret: true}
16
+ TARGET_PATH: {description: 'TARGET_ENGINE sqlite only: the rehearsal file', default: target.db}
17
+ TARGET_TABLE: {description: the new table, default: public.orders}
18
+ load:
19
+ mappings: [orders]
20
+ batch_rows: 1000
21
+ reconcile:
22
+ - mapping: orders
23
+ source: {key: [order_no], fields: [ordered_at, amount, customer_id]}
24
+ target: {key: [order_no], fields: [ordered_at, amount, customer_id]}
25
+ pairs: {ordered_at: ordered_at, amount: amount, customer_id: customer_id}
26
+ sample:
27
+ exports: {file: sample/orders.csv, format: csv}
28
+ cutover:
29
+ checklist: [Stop the old system's exports (or let the last one land), Load again (only new rows move), Reconcile again, Point the readers at the new table]
30
+ rollback: auto
@@ -0,0 +1,40 @@
1
+ version: 1
2
+ name: csv-to-postgres
3
+ # system to system: no registry and no identity (nothing here calls TheProtocol)
4
+ sources:
5
+ exports:
6
+ type: filedrop # an SFTP folder instead: see README.md
7
+ config:
8
+ path: ${EXPORT_DIR}
9
+ format: csv
10
+ suffix: .csv
11
+ id_field: order_no # the old system's own id: an export read twice moves nothing twice
12
+ settle_seconds: 0 # the exports are complete before the migration starts
13
+ poll_seconds: 0 # read the folder once, then finish
14
+ sinks:
15
+ shop:
16
+ type: sql_upsert
17
+ config:
18
+ engine: ${TARGET_ENGINE}
19
+ host: ${TARGET_HOST}
20
+ connect_args: {port: '${TARGET_PORT}'}
21
+ database: ${TARGET_DATABASE}
22
+ user: ${TARGET_USER}
23
+ password: ${TARGET_PASSWORD}
24
+ path: ${TARGET_PATH}
25
+ table: ${TARGET_TABLE}
26
+ key: [order_no]
27
+ on_conflict: update
28
+ preimage: true
29
+ mappings:
30
+ - name: orders
31
+ version: 1
32
+ from: exports
33
+ to: shop
34
+ fields:
35
+ order_no: {path: $.payload.data.order_no, as: int}
36
+ customer_id: {path: $.payload.data.customer_id, as: int}
37
+ ordered_at: {path: $.payload.data.ordered_at, as: datetime, format: '%Y-%m-%d %H:%M', tz: '${SOURCE_TZ}'}
38
+ amount: {path: $.payload.data.amount, as: decimal, round: 2}
39
+ currency: {path: $.payload.data.currency, trim: true, case: upper}
40
+ status: {path: $.payload.data.status, trim: true, case: lower}