python-corekit 0.6.2__tar.gz → 0.6.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. {python_corekit-0.6.2/python_corekit.egg-info → python_corekit-0.6.3}/PKG-INFO +1 -1
  2. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/__init__.py +2 -1
  3. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/connection.py +15 -15
  4. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/operations/base.py +9 -9
  5. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/operations/statements.py +12 -11
  6. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/table.py +6 -3
  7. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/__init__.py +4 -16
  8. python_corekit-0.6.3/corekit/integrations/__init__.py +30 -0
  9. python_corekit-0.6.3/corekit/integrations/adapter.py +72 -0
  10. python_corekit-0.6.3/corekit/integrations/loader.py +41 -0
  11. python_corekit-0.6.3/corekit/integrations/normalizer.py +28 -0
  12. python_corekit-0.6.3/corekit/integrations/pipeline.py +136 -0
  13. python_corekit-0.6.3/corekit/integrations/schemas.py +49 -0
  14. {python_corekit-0.6.2 → python_corekit-0.6.3}/pyproject.toml +1 -1
  15. {python_corekit-0.6.2 → python_corekit-0.6.3/python_corekit.egg-info}/PKG-INFO +1 -1
  16. {python_corekit-0.6.2 → python_corekit-0.6.3}/python_corekit.egg-info/SOURCES.txt +7 -0
  17. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_architecture.py +1 -0
  18. python_corekit-0.6.3/tests/test_integrations.py +87 -0
  19. {python_corekit-0.6.2 → python_corekit-0.6.3}/LICENSE +0 -0
  20. {python_corekit-0.6.2 → python_corekit-0.6.3}/README.md +0 -0
  21. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/__init__.py +0 -0
  22. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/api/__init__.py +0 -0
  23. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/api/application.py +0 -0
  24. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/api/handler.py +0 -0
  25. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/api/lifespan.py +0 -0
  26. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/api/middleware.py +0 -0
  27. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/api/responses.py +0 -0
  28. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/api/routers.py +0 -0
  29. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/concurrency/__init__.py +0 -0
  30. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/concurrency/decorators.py +0 -0
  31. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/concurrency/thread_local.py +0 -0
  32. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/concurrency/worker.py +0 -0
  33. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/config/__init__.py +0 -0
  34. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/config/loader.py +0 -0
  35. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/config/settings.py +0 -0
  36. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/config/sources.py +0 -0
  37. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/__init__.py +0 -0
  38. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/connectable.py +0 -0
  39. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/decorators.py +0 -0
  40. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/redis/__init__.py +0 -0
  41. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/redis/connection.py +0 -0
  42. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/registry.py +0 -0
  43. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/fields/__init__.py +0 -0
  44. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/fields/jsonb.py +0 -0
  45. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/migration/__init__.py +0 -0
  46. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/migration/base.py +0 -0
  47. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/migration/operations.py +0 -0
  48. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/migration/registry.py +0 -0
  49. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/migration/table.py +0 -0
  50. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/operations/__init__.py +0 -0
  51. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/connections/sql/query.py +0 -0
  52. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/crypto/__init__.py +0 -0
  53. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/crypto/constants.py +0 -0
  54. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/crypto/enum.py +0 -0
  55. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/crypto/hasher.py +0 -0
  56. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/__init__.py +0 -0
  57. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/dataset.py +0 -0
  58. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/expressions/__init__.py +0 -0
  59. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/expressions/comparison.py +0 -0
  60. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/expressions/expression.py +0 -0
  61. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/expressions/operator.py +0 -0
  62. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/expressions/target.py +0 -0
  63. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/record.py +0 -0
  64. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/data/stats.py +0 -0
  65. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/decorators/__init__.py +0 -0
  66. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/decorators/exception_handling.py +0 -0
  67. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/decorators/warnings.py +0 -0
  68. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/__init__.py +0 -0
  69. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/client.py +0 -0
  70. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/constants.py +0 -0
  71. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/disk/__init__.py +0 -0
  72. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/disk/constants.py +0 -0
  73. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/disk/inspector.py +0 -0
  74. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/disk/models.py +0 -0
  75. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/docker/watchdog.py +0 -0
  76. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/connection.py +0 -0
  77. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/extract/__init__.py +0 -0
  78. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/extract/extractor.py +0 -0
  79. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/extract/schemas.py +0 -0
  80. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/load/__init__.py +0 -0
  81. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/load/loader.py +0 -0
  82. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/load/schemas.py +0 -0
  83. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/orchestrator.py +0 -0
  84. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/schemas.py +0 -0
  85. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/transform/__init__.py +0 -0
  86. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/transform/schemas.py +0 -0
  87. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/etl/transform/transformer.py +0 -0
  88. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/__init__.py +0 -0
  89. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/enum.py +0 -0
  90. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/frames.py +0 -0
  91. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/models.py +0 -0
  92. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/publisher.py +0 -0
  93. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/reader.py +0 -0
  94. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/sse.py +0 -0
  95. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/events/websocket.py +0 -0
  96. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/__init__.py +0 -0
  97. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/actions.py +0 -0
  98. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/asgi.py +0 -0
  99. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/constants.py +0 -0
  100. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/exceptionator.py +0 -0
  101. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/fingerprint.py +0 -0
  102. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/guard.py +0 -0
  103. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/jobs.py +0 -0
  104. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptionator/models.py +0 -0
  105. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptions/__init__.py +0 -0
  106. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptions/base.py +0 -0
  107. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptions/custom/__init__.py +0 -0
  108. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptions/enum.py +0 -0
  109. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptions/http/__init__.py +0 -0
  110. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/exceptions/types.py +0 -0
  111. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/files/__init__.py +0 -0
  112. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/files/base.py +0 -0
  113. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/files/disk.py +0 -0
  114. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/files/enum.py +0 -0
  115. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/files/json.py +0 -0
  116. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/files/toml.py +0 -0
  117. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/html/__init__.py +0 -0
  118. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/html/client.py +0 -0
  119. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/html/models.py +0 -0
  120. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/html/parser.py +0 -0
  121. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/html/scraper.py +0 -0
  122. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/html/settings.py +0 -0
  123. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/html/state.py +0 -0
  124. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/__init__.py +0 -0
  125. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/api.py +0 -0
  126. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/client.py +0 -0
  127. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/exceptions.py +0 -0
  128. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/exponential_backoff.py +0 -0
  129. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/response.py +0 -0
  130. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/status.py +0 -0
  131. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/http/stream.py +0 -0
  132. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/jobs/__init__.py +0 -0
  133. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/jobs/registry.py +0 -0
  134. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/jobs/runner.py +0 -0
  135. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/jobs/task.py +0 -0
  136. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/__init__.py +0 -0
  137. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/client.py +0 -0
  138. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/enum.py +0 -0
  139. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/events.py +0 -0
  140. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/messages.py +0 -0
  141. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/prompts/__init__.py +0 -0
  142. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/prompts/enum.py +0 -0
  143. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/prompts/exceptions.py +0 -0
  144. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/prompts/loader.py +0 -0
  145. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/prompts/template.py +0 -0
  146. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/protocols.py +0 -0
  147. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/streaming.py +0 -0
  148. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/tools/__init__.py +0 -0
  149. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/tools/base.py +0 -0
  150. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/tools/detection.py +0 -0
  151. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/tools/loop.py +0 -0
  152. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/tools/registry.py +0 -0
  153. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/llm/wire.py +0 -0
  154. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/log_monitor/__init__.py +0 -0
  155. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/log_monitor/constants.py +0 -0
  156. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/log_monitor/models.py +0 -0
  157. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/log_monitor/service.py +0 -0
  158. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/notifications/__init__.py +0 -0
  159. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/notifications/base.py +0 -0
  160. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/notifications/models.py +0 -0
  161. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/__init__.py +0 -0
  162. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/benchmarkable.py +0 -0
  163. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/loggable.py +0 -0
  164. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/request_context.py +0 -0
  165. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/timing/__init__.py +0 -0
  166. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/timing/constants.py +0 -0
  167. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/timing/split.py +0 -0
  168. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/observability/timing/timer.py +0 -0
  169. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/process/__init__.py +0 -0
  170. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/process/runner.py +0 -0
  171. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/py.typed +0 -0
  172. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/registry/__init__.py +0 -0
  173. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/registry/ordered.py +0 -0
  174. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/registry/registry.py +0 -0
  175. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/__init__.py +0 -0
  176. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/dataclasses/__init__.py +0 -0
  177. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/enum.py +0 -0
  178. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/models/__init__.py +0 -0
  179. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/models/arbitrary.py +0 -0
  180. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/models/date_models.py +0 -0
  181. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/pydantic/__init__.py +0 -0
  182. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/pydantic/fields.py +0 -0
  183. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/size.py +0 -0
  184. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/types.py +0 -0
  185. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/schemas/version.py +0 -0
  186. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/serialization/__init__.py +0 -0
  187. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/serialization/enum.py +0 -0
  188. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/serialization/pickle_file.py +0 -0
  189. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/serialization/serializable.py +0 -0
  190. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/serialization/serializer.py +0 -0
  191. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/__init__.py +0 -0
  192. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/coercion.py +0 -0
  193. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/collections.py +0 -0
  194. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/ids.py +0 -0
  195. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/parsers.py +0 -0
  196. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/payload.py +0 -0
  197. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/raise_exc.py +0 -0
  198. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/text.py +0 -0
  199. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/time.py +0 -0
  200. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/validators.py +0 -0
  201. {python_corekit-0.6.2 → python_corekit-0.6.3}/corekit/utils/void.py +0 -0
  202. {python_corekit-0.6.2 → python_corekit-0.6.3}/python_corekit.egg-info/dependency_links.txt +0 -0
  203. {python_corekit-0.6.2 → python_corekit-0.6.3}/python_corekit.egg-info/requires.txt +0 -0
  204. {python_corekit-0.6.2 → python_corekit-0.6.3}/python_corekit.egg-info/top_level.txt +0 -0
  205. {python_corekit-0.6.2 → python_corekit-0.6.3}/setup.cfg +0 -0
  206. {python_corekit-0.6.2 → python_corekit-0.6.3}/setup.py +0 -0
  207. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_api.py +0 -0
  208. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_application.py +0 -0
  209. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_concurrency.py +0 -0
  210. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_config.py +0 -0
  211. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_connections.py +0 -0
  212. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_data.py +0 -0
  213. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_docker.py +0 -0
  214. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_etl.py +0 -0
  215. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_events.py +0 -0
  216. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_exceptionator.py +0 -0
  217. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_exceptions.py +0 -0
  218. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_expressions.py +0 -0
  219. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_expressions_sqlalchemy.py +0 -0
  220. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_files.py +0 -0
  221. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_html.py +0 -0
  222. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_imports.py +0 -0
  223. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_imports_are_top_level.py +0 -0
  224. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_jobs.py +0 -0
  225. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_lifespan.py +0 -0
  226. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_llm.py +0 -0
  227. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_log_monitor.py +0 -0
  228. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_middleware_stack.py +0 -0
  229. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_migration.py +0 -0
  230. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_notifications.py +0 -0
  231. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_observability.py +0 -0
  232. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_operations.py +0 -0
  233. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_ordered_registry.py +0 -0
  234. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_process.py +0 -0
  235. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_redis.py +0 -0
  236. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_registry.py +0 -0
  237. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_request_context.py +0 -0
  238. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_requests.py +0 -0
  239. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_schemas_arbitrary.py +0 -0
  240. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_schemas_enum.py +0 -0
  241. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_schemas_size.py +0 -0
  242. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_serialization.py +0 -0
  243. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_sql.py +0 -0
  244. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils.py +0 -0
  245. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils_coercion.py +0 -0
  246. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils_collections.py +0 -0
  247. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils_ids.py +0 -0
  248. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils_parsers.py +0 -0
  249. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils_payload.py +0 -0
  250. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils_text.py +0 -0
  251. {python_corekit-0.6.2 → python_corekit-0.6.3}/tests/test_utils_time.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: python-corekit
3
- Version: 0.6.2
3
+ Version: 0.6.3
4
4
  Summary: Shared foundations for Python projects: logging, benchmarking, registries, FastAPI application and routers, SQL statements and migrations, background tasks, ETL, LLM chat/tool primitives, and HTML scraping
5
5
  Author: Steven Jacobsen
6
6
  License-Expression: MIT
@@ -16,7 +16,7 @@ from corekit.connections.sql.operations import (
16
16
  Update,
17
17
  )
18
18
  from corekit.connections.sql.query import Query
19
- from corekit.connections.sql.table import NamedTable, TableExportItem
19
+ from corekit.connections.sql.table import NamedTable, TableExportItem, TTable
20
20
 
21
21
  __all__ = [
22
22
  "Conditional",
@@ -35,5 +35,6 @@ __all__ = [
35
35
  "Select",
36
36
  "SQLConnection",
37
37
  "TableExportItem",
38
+ "TTable",
38
39
  "Update",
39
40
  ]
@@ -26,7 +26,7 @@ from corekit.config import get_settings
26
26
  from corekit.connections import Connectable
27
27
  from corekit.connections.sql.operations import DqlOperation, Operation, Select
28
28
  from corekit.connections.sql.query import Query
29
- from corekit.connections.sql.table import NamedTable
29
+ from corekit.connections.sql.table import TTable
30
30
 
31
31
  __all__ = ["PaginatedResult", "SQLConnection"]
32
32
 
@@ -261,7 +261,7 @@ class SQLConnection(Connectable):
261
261
  self.session.merge(item)
262
262
  self.session.commit()
263
263
 
264
- def delete(self, table: type[NamedTable], identifier: str) -> None:
264
+ def delete(self, table: type[TTable], identifier: str) -> None:
265
265
  """
266
266
  Delete a row by id.
267
267
  """
@@ -308,19 +308,19 @@ class SQLConnection(Connectable):
308
308
  result = self.select(statement)
309
309
  return None if result is None else result.first()
310
310
 
311
- def fetch_all(self, table: type[NamedTable]) -> Iterator[NamedTable]:
311
+ def fetch_all(self, table: type[TTable]) -> Iterator[TTable]:
312
312
  """
313
313
  Every row in a table.
314
314
  """
315
315
  return self._exec(select(table))
316
316
 
317
- def fetch_one_by_id(self, table: type[NamedTable], identifier: str) -> NamedTable | None:
317
+ def fetch_one_by_id(self, table: type[TTable], identifier: str) -> TTable | None:
318
318
  """
319
319
  A single row by id.
320
320
  """
321
321
  return self.first(select(table).where(table.id == identifier))
322
322
 
323
- def fetch_one_by_condition(self, table: type[NamedTable], key: Any, value: Any) -> NamedTable | None:
323
+ def fetch_one_by_condition(self, table: type[TTable], key: Any, value: Any) -> TTable | None:
324
324
  """
325
325
  The first row where a column equals a value.
326
326
 
@@ -330,13 +330,13 @@ class SQLConnection(Connectable):
330
330
  column = getattr(table, key) if isinstance(key, str) else key
331
331
  return self.first(select(table).where(column == value))
332
332
 
333
- def exists(self, table: type[NamedTable], identifier: str) -> bool:
333
+ def exists(self, table: type[TTable], identifier: str) -> bool:
334
334
  """
335
335
  Whether a row with this id exists.
336
336
  """
337
337
  return self.fetch_one_by_id(table, identifier) is not None
338
338
 
339
- def fetch_or_create_one(self, table: type[NamedTable], identifier: str) -> NamedTable:
339
+ def fetch_or_create_one(self, table: type[TTable], identifier: str) -> TTable:
340
340
  """
341
341
  Fetch a row by id, inserting a default one if it is missing.
342
342
  """
@@ -346,13 +346,13 @@ class SQLConnection(Connectable):
346
346
  self.insert(item)
347
347
  return item
348
348
 
349
- def fetch_many_by_query(self, query: Query) -> list[NamedTable]:
349
+ def fetch_many_by_query(self, query: Query[TTable]) -> list[TTable]:
350
350
  """
351
351
  Every row matching a Query.
352
352
  """
353
353
  return list(self._exec(query.build()))
354
354
 
355
- def delete_many_by_query(self, query: Query) -> int:
355
+ def delete_many_by_query(self, query: Query[TTable]) -> int:
356
356
  """
357
357
  Delete every row matching a Query.
358
358
 
@@ -366,7 +366,7 @@ class SQLConnection(Connectable):
366
366
  self.session.commit()
367
367
  return len(items)
368
368
 
369
- def execute(self, operation: Operation) -> Any:
369
+ def execute(self, operation: Operation[Any]) -> Any:
370
370
  """
371
371
  Run an operation, committing if it writes.
372
372
 
@@ -380,19 +380,19 @@ class SQLConnection(Connectable):
380
380
  self.session.commit()
381
381
  return result
382
382
 
383
- def fetch(self, operation: DqlOperation) -> list[NamedTable]:
383
+ def fetch(self, operation: DqlOperation[TTable]) -> list[TTable]:
384
384
  """
385
385
  Every row an operation selects.
386
386
  """
387
387
  return list(self.execute(operation))
388
388
 
389
- def fetch_one_by_query(self, query: Query) -> NamedTable | None:
389
+ def fetch_one_by_query(self, query: Query[TTable]) -> TTable | None:
390
390
  """
391
391
  The first row matching a Query.
392
392
  """
393
393
  return self.first(query.build())
394
394
 
395
- def count(self, table: type[NamedTable]) -> int:
395
+ def count(self, table: type[TTable]) -> int:
396
396
  """
397
397
  The number of rows in a table.
398
398
  """
@@ -401,13 +401,13 @@ class SQLConnection(Connectable):
401
401
 
402
402
  def paginate(
403
403
  self,
404
- table: type[NamedTable],
404
+ table: type[TTable],
405
405
  offset: int = 0,
406
406
  limit: int = 50,
407
407
  *,
408
408
  order_by: Any = None,
409
409
  desc: bool = False,
410
- ) -> PaginatedResult:
410
+ ) -> PaginatedResult[TTable]:
411
411
  """
412
412
  A page of rows, together with the total row count.
413
413
 
@@ -3,17 +3,17 @@ Statement primitives: the shapes a database operation can take.
3
3
  """
4
4
 
5
5
  from abc import ABC, abstractmethod
6
- from typing import Any, ClassVar
6
+ from typing import Any, ClassVar, Generic, Self
7
7
 
8
8
  from pydantic import BaseModel, Field
9
9
 
10
- from corekit.connections.sql.table import NamedTable
10
+ from corekit.connections.sql.table import TTable
11
11
  from corekit.data.expressions import Expression
12
12
 
13
13
  __all__ = ["Conditional", "DdlOperation", "DmlOperation", "DqlOperation", "Operation"]
14
14
 
15
15
 
16
- class Operation(BaseModel, ABC):
16
+ class Operation(BaseModel, ABC, Generic[TTable]):
17
17
  """
18
18
  A statement waiting to be executed.
19
19
 
@@ -28,7 +28,7 @@ class Operation(BaseModel, ABC):
28
28
  #: Conditional writes (update, delete) and insert both set this; reads do not.
29
29
  writes: ClassVar[bool] = False
30
30
 
31
- table: type[NamedTable]
31
+ table: type[TTable]
32
32
 
33
33
  @abstractmethod
34
34
  def build(self) -> Any:
@@ -40,7 +40,7 @@ class Operation(BaseModel, ABC):
40
40
  return f"{type(self).__name__}({self.table.__name__})"
41
41
 
42
42
 
43
- class Conditional(Operation, ABC):
43
+ class Conditional(Operation[TTable], ABC, Generic[TTable]):
44
44
  """
45
45
  An operation narrowed by conditions, e.g. everything but an insert.
46
46
 
@@ -51,7 +51,7 @@ class Conditional(Operation, ABC):
51
51
 
52
52
  conditions: list[Any] = Field(default_factory=list)
53
53
 
54
- def where(self, condition: Any) -> "Conditional":
54
+ def where(self, condition: Any) -> Self:
55
55
  """
56
56
  Narrow this operation. Repeated calls combine with AND.
57
57
  """
@@ -75,13 +75,13 @@ class Conditional(Operation, ABC):
75
75
  return statement
76
76
 
77
77
 
78
- class DqlOperation(Conditional, ABC):
78
+ class DqlOperation(Conditional[TTable], ABC, Generic[TTable]):
79
79
  """
80
80
  A read. Returns rows and changes nothing.
81
81
  """
82
82
 
83
83
 
84
- class DmlOperation(Conditional, ABC):
84
+ class DmlOperation(Conditional[TTable], ABC, Generic[TTable]):
85
85
  """
86
86
  A conditional write against rows: update or delete.
87
87
 
@@ -92,7 +92,7 @@ class DmlOperation(Conditional, ABC):
92
92
  writes: ClassVar[bool] = True
93
93
 
94
94
 
95
- class DdlOperation(Operation, ABC):
95
+ class DdlOperation(Operation[TTable], ABC, Generic[TTable]):
96
96
  """
97
97
  A change to the schema itself.
98
98
 
@@ -2,7 +2,7 @@
2
2
  The statements an application runs: select, insert, update and delete.
3
3
  """
4
4
 
5
- from typing import Any, ClassVar
5
+ from typing import Any, ClassVar, Generic, Self
6
6
 
7
7
  from pydantic import Field
8
8
  from sqlmodel import delete as _delete
@@ -12,11 +12,12 @@ from sqlmodel import select as _select
12
12
  from sqlmodel import update as _update
13
13
 
14
14
  from corekit.connections.sql.operations.base import DmlOperation, DqlOperation, Operation
15
+ from corekit.connections.sql.table import TTable
15
16
 
16
17
  __all__ = ["Delete", "Insert", "Select", "Update"]
17
18
 
18
19
 
19
- class Select(DqlOperation):
20
+ class Select(DqlOperation[TTable], Generic[TTable]):
20
21
  """
21
22
  Read rows, optionally ordered, offset and capped.
22
23
 
@@ -37,7 +38,7 @@ class Select(DqlOperation):
37
38
  offset_count: int = 0
38
39
  for_update_lock: bool = False
39
40
 
40
- def order_by(self, field: Any, desc: bool = False) -> "Select":
41
+ def order_by(self, field: Any, desc: bool = False) -> Self:
41
42
  """
42
43
  Order results by a field.
43
44
  """
@@ -45,7 +46,7 @@ class Select(DqlOperation):
45
46
  self.order_by_desc = desc
46
47
  return self
47
48
 
48
- def limit(self, limit: int) -> "Select":
49
+ def limit(self, limit: int) -> Self:
49
50
  """
50
51
  Cap the number of rows returned.
51
52
 
@@ -54,14 +55,14 @@ class Select(DqlOperation):
54
55
  self.limit_count = limit
55
56
  return self
56
57
 
57
- def offset(self, offset: int) -> "Select":
58
+ def offset(self, offset: int) -> Self:
58
59
  """
59
60
  Skip the first ``offset`` rows.
60
61
  """
61
62
  self.offset_count = offset
62
63
  return self
63
64
 
64
- def for_update(self) -> "Select":
65
+ def for_update(self) -> Self:
65
66
  """
66
67
  Lock the matching rows until the transaction ends.
67
68
  """
@@ -92,7 +93,7 @@ class Select(DqlOperation):
92
93
  return statement
93
94
 
94
95
 
95
- class Delete(DmlOperation):
96
+ class Delete(DmlOperation[TTable], Generic[TTable]):
96
97
  """
97
98
  Delete every row matching the conditions, in one statement.
98
99
 
@@ -114,7 +115,7 @@ class Delete(DmlOperation):
114
115
  return self._apply_conditions(_delete(self.table))
115
116
 
116
117
 
117
- class Update(DmlOperation):
118
+ class Update(DmlOperation[TTable], Generic[TTable]):
118
119
  """
119
120
  Set columns on every row matching the conditions, in one statement.
120
121
 
@@ -125,7 +126,7 @@ class Update(DmlOperation):
125
126
 
126
127
  values: dict[str, Any] = Field(default_factory=dict)
127
128
 
128
- def set(self, **values: Any) -> "Update":
129
+ def set(self, **values: Any) -> Self:
129
130
  """
130
131
  Columns to write, as keyword arguments.
131
132
  """
@@ -146,7 +147,7 @@ class Update(DmlOperation):
146
147
  return self._apply_conditions(_update(self.table)).values(**self.values)
147
148
 
148
149
 
149
- class Insert(Operation):
150
+ class Insert(Operation[TTable], Generic[TTable]):
150
151
  """
151
152
  Add rows. A write, so ``SQLConnection.execute`` commits it.
152
153
 
@@ -163,7 +164,7 @@ class Insert(Operation):
163
164
 
164
165
  rows: list[dict[str, Any]] = Field(default_factory=list)
165
166
 
166
- def add(self, **values: Any) -> "Insert":
167
+ def add(self, **values: Any) -> Self:
167
168
  """
168
169
  Append one row.
169
170
  """
@@ -3,14 +3,14 @@ Base table type.
3
3
  """
4
4
 
5
5
  import logging
6
- from typing import Any, ClassVar
6
+ from typing import Any, ClassVar, Self, TypeVar
7
7
 
8
8
  from pydantic import BaseModel
9
9
  from sqlmodel import SQLModel
10
10
 
11
11
  from corekit.registry import SmartRegistry
12
12
 
13
- __all__ = ["NamedTable", "TableExportItem"]
13
+ __all__ = ["NamedTable", "TableExportItem", "TTable"]
14
14
 
15
15
 
16
16
  class TableExportItem(BaseModel):
@@ -85,7 +85,7 @@ class NamedTable(SQLModel):
85
85
  return name or cls.__name__
86
86
 
87
87
  @classmethod
88
- def create_default(cls, identifier: str, **kwargs: Any) -> "NamedTable":
88
+ def create_default(cls, identifier: str, **kwargs: Any) -> Self:
89
89
  """
90
90
  Build an instance carrying only its identifier and any supplied fields.
91
91
  """
@@ -98,3 +98,6 @@ class NamedTable(SQLModel):
98
98
  This row as a portable record, tagged with its database table name.
99
99
  """
100
100
  return TableExportItem(table_name=self.table_name, data=self.model_dump())
101
+
102
+
103
+ TTable = TypeVar("TTable", bound=NamedTable)
@@ -1,21 +1,9 @@
1
1
  """
2
- ETL pipelines.
2
+ ETL pipelines (legacy).
3
3
 
4
- Implement an extractor, a transformer and a loader, then declare them on an
5
- orchestrator::
6
-
7
- class UserPipeline(BaseETLOrchestrator):
8
- '''
9
- Loads users from the API into the warehouse.
10
- '''
11
- extractor = UserExtractor
12
- transformer = UserTransformer
13
- loader = UserLoader
14
-
15
- await UserPipeline().run()
16
-
17
- The run loop streams, so memory use is bounded by the batch size rather than by
18
- how much data the source holds.
4
+ Prefer ``corekit.integrations`` for new work: ``BaseAdapter`` / ``BaseNormalizer``
5
+ / ``BaseLoader`` / ``BasePipeline``. This package remains for existing tests and
6
+ imports; new vendors should not extend Extractor/Transformer naming.
19
7
  """
20
8
 
21
9
  from corekit.etl.connection import BaseConnection, ConnectionDetails, ETLSource
@@ -0,0 +1,30 @@
1
+ """
2
+ Integration framework: clients, adapters, normalizers, loaders, pipelines.
3
+
4
+ Layering::
5
+
6
+ BaseApiClient — wire + auth to the external system
7
+ BaseNormalizer — raw payload → concrete BaseModel (never dict-out)
8
+ BaseAdapter — named ops (interactive) + optional bulk sources (sync)
9
+ BaseLoader — persist normalized items into the host system
10
+ BasePipeline — stream adapter bulk → loader (nightly / full sync)
11
+
12
+ Interactive Integrations call ``adapter.run(...)``. Catalog syncs call
13
+ ``pipeline.run()``. Do not route point lookups through the pipeline.
14
+ """
15
+
16
+ from corekit.integrations.adapter import BaseAdapter
17
+ from corekit.integrations.loader import BaseLoader
18
+ from corekit.integrations.normalizer import BaseNormalizer
19
+ from corekit.integrations.pipeline import BasePipeline
20
+ from corekit.integrations.schemas import LoadableBatch, NormalizedItem, OperationClass
21
+
22
+ __all__ = [
23
+ "BaseAdapter",
24
+ "BaseLoader",
25
+ "BaseNormalizer",
26
+ "BasePipeline",
27
+ "LoadableBatch",
28
+ "NormalizedItem",
29
+ "OperationClass",
30
+ ]
@@ -0,0 +1,72 @@
1
+ """
2
+ Vendor adapter: named operations plus optional bulk sources for sync pipelines.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from abc import ABC, abstractmethod
8
+ from collections.abc import AsyncIterable, Callable
9
+ from typing import Any
10
+
11
+ from pydantic import BaseModel
12
+
13
+ from corekit.http import BadRequestException, BaseApiClient
14
+ from corekit.integrations.normalizer import BaseNormalizer
15
+ from corekit.integrations.schemas import NormalizedItem, OperationClass
16
+ from corekit.observability import Loggable
17
+
18
+
19
+ class BaseAdapter(Loggable, ABC):
20
+ """
21
+ Integration surface for one external system.
22
+
23
+ - **Named ops** (``run``): interactive request/response (search, create, …).
24
+ - **Bulk sources** (``iter_bulk``): streaming catalog/sync pulls for pipelines.
25
+
26
+ Always constructed with a ``BaseApiClient`` and a ``BaseNormalizer``. Ops and
27
+ bulk sources return ``BaseModel`` instances — never raw dicts.
28
+ """
29
+
30
+ def __init__(self, client: BaseApiClient, normalizer: BaseNormalizer[Any]) -> None:
31
+ super().__init__()
32
+ self.client = client
33
+ self.normalizer = normalizer
34
+
35
+ @abstractmethod
36
+ def operation_classes(self) -> dict[str, OperationClass]:
37
+ """
38
+ Map operation key → read/write class.
39
+ """
40
+
41
+ @property
42
+ def operations(self) -> frozenset[str]:
43
+ return frozenset(self.operation_classes())
44
+
45
+ def operation_class(self, operation: str) -> OperationClass | None:
46
+ return self.operation_classes().get(operation)
47
+
48
+ def require_operation(self, operation: str) -> OperationClass:
49
+ op_class = self.operation_class(operation)
50
+ if op_class is None:
51
+ raise BadRequestException(detail=f"Unknown operation '{operation}'")
52
+ return op_class
53
+
54
+ @abstractmethod
55
+ async def run(self, operation: str, payload: dict[str, Any], **context: Any) -> BaseModel:
56
+ """
57
+ Execute a named operation. Return a normalized ``BaseModel`` (never secrets).
58
+ """
59
+
60
+ def bulk_sources(self) -> list[Callable[..., AsyncIterable[NormalizedItem]]]:
61
+ """
62
+ Optional streaming sources for full sync. Default: none.
63
+ """
64
+ return []
65
+
66
+ async def iter_bulk(self, *args: Any, **kwargs: Any) -> AsyncIterable[NormalizedItem]:
67
+ """
68
+ Yield normalized items from every bulk source.
69
+ """
70
+ for source in self.bulk_sources():
71
+ async for item in source(*args, **kwargs):
72
+ yield item
@@ -0,0 +1,41 @@
1
+ """
2
+ Persist normalized items into the host system.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from abc import ABC, abstractmethod
8
+
9
+ from corekit.integrations.schemas import LoadableBatch, NormalizedItem
10
+ from corekit.observability import Loggable
11
+
12
+
13
+ class BaseLoader(Loggable, ABC):
14
+ """
15
+ Batches normalized items and writes them to a destination (DB, namespace, …).
16
+ """
17
+
18
+ def __init__(self) -> None:
19
+ super().__init__()
20
+ self.batch: LoadableBatch = LoadableBatch()
21
+
22
+ def _should_load(self, min_ops: int) -> bool:
23
+ return bool(len(self.batch)) and len(self.batch) >= min_ops
24
+
25
+ def add_item(self, item: NormalizedItem) -> None:
26
+ self.batch.add_item(item)
27
+
28
+ async def flush(self) -> None:
29
+ await self.load(min_ops=0)
30
+
31
+ async def load(self, min_ops: int) -> None:
32
+ if not self._should_load(min_ops):
33
+ return
34
+ await self._load(self.batch)
35
+ self.batch.clear()
36
+
37
+ @abstractmethod
38
+ async def _load(self, batch: LoadableBatch) -> None:
39
+ """
40
+ Write a batch to its destination.
41
+ """
@@ -0,0 +1,28 @@
1
+ """
2
+ Translate raw external payloads into known Pydantic models.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from abc import ABC, abstractmethod
8
+ from typing import Any, Generic, TypeVar
9
+
10
+ from pydantic import BaseModel
11
+
12
+ from corekit.observability import Loggable
13
+
14
+ TModel = TypeVar("TModel", bound=BaseModel)
15
+
16
+
17
+ class BaseNormalizer(Loggable, ABC, Generic[TModel]):
18
+ """
19
+ Pure translation from vendor/raw payloads to a concrete ``BaseModel``.
20
+
21
+ Output is always structured. Dict-in/dict-out is not a normalizer.
22
+ """
23
+
24
+ @abstractmethod
25
+ def normalize(self, raw: Any) -> TModel:
26
+ """
27
+ Return a typed model downstream code can rely on.
28
+ """
@@ -0,0 +1,136 @@
1
+ """
2
+ Bulk sync: stream adapter bulk sources into a loader.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from dataclasses import dataclass
8
+ from typing import Any, ClassVar
9
+
10
+ from corekit.integrations.adapter import BaseAdapter
11
+ from corekit.integrations.loader import BaseLoader
12
+ from corekit.observability import Loggable
13
+ from corekit.registry import SmartRegistry
14
+
15
+ __all__ = ["BasePipeline"]
16
+
17
+ DEFAULT_BATCH_SIZE = 100
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class PipelineStage:
22
+ """
23
+ One declared stage on a pipeline class.
24
+ """
25
+
26
+ attribute: str
27
+ base_class: type
28
+
29
+ @property
30
+ def label(self) -> str:
31
+ return self.base_class.__name__.removeprefix("Base")
32
+
33
+ def declared_on(self, cls: type) -> Any:
34
+ return cls.__dict__.get(self.attribute)
35
+
36
+ def validate(self, cls: type) -> None:
37
+ stage = getattr(cls, self.attribute, None)
38
+ if stage is None:
39
+ raise TypeError(f"{cls.__name__} is missing its {self.attribute}. Set {self.attribute} = Your{self.label}.")
40
+ if not (isinstance(stage, type) and issubclass(stage, self.base_class)):
41
+ raise TypeError(
42
+ f"{cls.__name__}.{self.attribute} must be a {self.base_class.__name__} subclass, "
43
+ f"got {stage!r}. Assign the class itself, not an instance."
44
+ )
45
+
46
+
47
+ class BasePipeline(Loggable):
48
+ """
49
+ Runs adapter bulk sync into a loader.
50
+
51
+ Interactive named ops stay on ``BaseAdapter.run`` — this class is for
52
+ nightly/full pulls only.
53
+ """
54
+
55
+ adapter: ClassVar[type[BaseAdapter] | None] = None
56
+ loader: ClassVar[type[BaseLoader] | None] = None
57
+ batch_size: ClassVar[int] = DEFAULT_BATCH_SIZE
58
+
59
+ __registry__: SmartRegistry = SmartRegistry()
60
+ __binding_only__: ClassVar[bool] = False
61
+
62
+ STAGES: ClassVar[list[PipelineStage]] = [
63
+ PipelineStage(attribute="adapter", base_class=BaseAdapter),
64
+ PipelineStage(attribute="loader", base_class=BaseLoader),
65
+ ]
66
+
67
+ def __init_subclass__(cls, **kwargs: Any) -> None:
68
+ super().__init_subclass__(**kwargs)
69
+ if cls.__dict__.get("__binding_only__"):
70
+ return
71
+ if not any(stage.declared_on(cls) for stage in cls.STAGES):
72
+ return
73
+ for stage in cls.STAGES:
74
+ stage.validate(cls)
75
+ BasePipeline.__registry__[cls.__name__] = cls
76
+
77
+ def __class_getitem__(cls, item: tuple[type, ...]) -> Any:
78
+ if not isinstance(item, tuple) or len(item) != 2:
79
+ raise TypeError(f"{cls.__name__}[...] takes exactly two stages: {cls.__name__}[Adapter, Loader]")
80
+ bound = {stage.attribute: value for stage, value in zip(cls.STAGES, item, strict=True)}
81
+ return type(f"{cls.__name__}_{item[0].__name__}", (cls,), bound)
82
+
83
+ @classmethod
84
+ def get_pipeline_types(cls) -> SmartRegistry:
85
+ return cls.__registry__
86
+
87
+ @classmethod
88
+ def get_pipeline_by_name(cls, name: str) -> type[BasePipeline] | None:
89
+ return cls.__registry__.get(name)
90
+
91
+ def __init__(
92
+ self,
93
+ adapter: BaseAdapter | None = None,
94
+ loader: BaseLoader | None = None,
95
+ batch_size: int | None = None,
96
+ on_item_error: str = "raise",
97
+ ) -> None:
98
+ if on_item_error not in ("raise", "skip"):
99
+ raise ValueError("on_item_error must be 'raise' or 'skip'")
100
+ super().__init__()
101
+ self.batch_size = batch_size if batch_size is not None else type(self).batch_size
102
+ self.on_item_error = on_item_error
103
+ self._adapter = adapter
104
+ self._loader = loader
105
+
106
+ def _build(self, override: Any, name: str) -> Any:
107
+ if override is not None:
108
+ return override
109
+ stage_class = getattr(type(self), name, None)
110
+ if stage_class is None:
111
+ raise TypeError(f"{type(self).__name__} has no {name}. Declare one, or pass {name}=...")
112
+ return stage_class()
113
+
114
+ async def run(self) -> None:
115
+ """
116
+ Stream ``adapter.iter_bulk()`` into the loader until exhausted.
117
+ """
118
+ adapter = self._build(self._adapter, "adapter")
119
+ loader = self._build(self._loader, "loader")
120
+
121
+ count = 0
122
+ skipped = 0
123
+ async for item in adapter.iter_bulk():
124
+ try:
125
+ loader.add_item(item)
126
+ except Exception:
127
+ if self.on_item_error == "raise":
128
+ raise
129
+ skipped += 1
130
+ self.exception(f"{type(self).__name__} skipped an item")
131
+ continue
132
+ await loader.load(min_ops=self.batch_size)
133
+ count += 1
134
+
135
+ await loader.flush()
136
+ self.info(f"{type(self).__name__} processed {count} items, skipped {skipped}")