pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (404) hide show
  1. api/main.py +27 -1
  2. api/models/docs.py +68 -0
  3. api/models/evolution.py +117 -0
  4. api/models/tracking.py +111 -0
  5. api/models/validation.py +46 -6
  6. api/routes/v1/__init__.py +14 -1
  7. api/routes/v1/docs.py +187 -0
  8. api/routes/v1/evolution.py +337 -0
  9. api/routes/v1/templates.py +211 -27
  10. api/routes/v1/tracking.py +301 -0
  11. api/routes/v1/validation.py +68 -31
  12. pycharter/__init__.py +268 -58
  13. pycharter/data/templates/contract/template_coercion_rules.yaml +57 -0
  14. pycharter/data/templates/contract/template_contract.yaml +122 -0
  15. pycharter/data/templates/contract/template_metadata.yaml +68 -0
  16. pycharter/data/templates/contract/template_schema.yaml +100 -0
  17. pycharter/data/templates/contract/template_validation_rules.yaml +75 -0
  18. pycharter/data/templates/etl/README.md +224 -0
  19. pycharter/data/templates/etl/extract_cloud_azure.yaml +24 -0
  20. pycharter/data/templates/etl/extract_cloud_gcs.yaml +25 -0
  21. pycharter/data/templates/etl/extract_cloud_s3.yaml +30 -0
  22. pycharter/data/templates/etl/extract_database.yaml +34 -0
  23. pycharter/data/templates/etl/extract_database_ssh.yaml +40 -0
  24. pycharter/data/templates/etl/extract_file_csv.yaml +21 -0
  25. pycharter/data/templates/etl/extract_file_glob.yaml +25 -0
  26. pycharter/data/templates/etl/extract_file_json.yaml +24 -0
  27. pycharter/data/templates/etl/extract_file_parquet.yaml +20 -0
  28. pycharter/data/templates/etl/extract_http_paginated.yaml +79 -0
  29. pycharter/data/templates/etl/extract_http_path_params.yaml +38 -0
  30. pycharter/data/templates/etl/extract_http_simple.yaml +62 -0
  31. pycharter/data/templates/etl/load_cloud_azure.yaml +24 -0
  32. pycharter/data/templates/etl/load_cloud_gcs.yaml +22 -0
  33. pycharter/data/templates/etl/load_cloud_s3.yaml +27 -0
  34. pycharter/data/templates/etl/load_file.yaml +34 -0
  35. pycharter/data/templates/etl/load_insert.yaml +18 -0
  36. pycharter/data/templates/etl/load_postgresql.yaml +39 -0
  37. pycharter/data/templates/etl/load_sqlite.yaml +21 -0
  38. pycharter/data/templates/etl/load_truncate_and_load.yaml +20 -0
  39. pycharter/data/templates/etl/load_upsert.yaml +25 -0
  40. pycharter/data/templates/etl/load_with_dlq.yaml +34 -0
  41. pycharter/data/templates/etl/load_with_ssh_tunnel.yaml +35 -0
  42. pycharter/data/templates/etl/pipeline_http_to_db.yaml +75 -0
  43. pycharter/data/templates/etl/transform_combined.yaml +48 -0
  44. pycharter/data/templates/etl/transform_custom_function.yaml +58 -0
  45. pycharter/data/templates/etl/transform_jsonata.yaml +51 -0
  46. pycharter/data/templates/etl/transform_simple.yaml +59 -0
  47. pycharter/db/schemas/.ipynb_checkpoints/data_contract-checkpoint.py +160 -0
  48. pycharter/docs_generator/__init__.py +43 -0
  49. pycharter/docs_generator/generator.py +465 -0
  50. pycharter/docs_generator/renderers.py +247 -0
  51. pycharter/etl_generator/__init__.py +168 -80
  52. pycharter/etl_generator/builder.py +121 -0
  53. pycharter/etl_generator/config_loader.py +394 -0
  54. pycharter/etl_generator/config_validator.py +418 -0
  55. pycharter/etl_generator/context.py +132 -0
  56. pycharter/etl_generator/expression.py +499 -0
  57. pycharter/etl_generator/extractors/__init__.py +30 -0
  58. pycharter/etl_generator/extractors/base.py +70 -0
  59. pycharter/etl_generator/extractors/cloud_storage.py +530 -0
  60. pycharter/etl_generator/extractors/database.py +221 -0
  61. pycharter/etl_generator/extractors/factory.py +185 -0
  62. pycharter/etl_generator/extractors/file.py +475 -0
  63. pycharter/etl_generator/extractors/http.py +895 -0
  64. pycharter/etl_generator/extractors/streaming.py +57 -0
  65. pycharter/etl_generator/loaders/__init__.py +41 -0
  66. pycharter/etl_generator/loaders/base.py +35 -0
  67. pycharter/etl_generator/loaders/cloud.py +87 -0
  68. pycharter/etl_generator/loaders/cloud_storage_loader.py +275 -0
  69. pycharter/etl_generator/loaders/database.py +274 -0
  70. pycharter/etl_generator/loaders/factory.py +180 -0
  71. pycharter/etl_generator/loaders/file.py +72 -0
  72. pycharter/etl_generator/loaders/file_loader.py +130 -0
  73. pycharter/etl_generator/pipeline.py +743 -0
  74. pycharter/etl_generator/protocols.py +54 -0
  75. pycharter/etl_generator/result.py +63 -0
  76. pycharter/etl_generator/schemas/__init__.py +49 -0
  77. pycharter/etl_generator/transformers/__init__.py +49 -0
  78. pycharter/etl_generator/transformers/base.py +63 -0
  79. pycharter/etl_generator/transformers/config.py +45 -0
  80. pycharter/etl_generator/transformers/custom_function.py +101 -0
  81. pycharter/etl_generator/transformers/jsonata_transformer.py +56 -0
  82. pycharter/etl_generator/transformers/operations.py +218 -0
  83. pycharter/etl_generator/transformers/pipeline.py +54 -0
  84. pycharter/etl_generator/transformers/simple_operations.py +131 -0
  85. pycharter/quality/__init__.py +25 -0
  86. pycharter/quality/tracking/__init__.py +64 -0
  87. pycharter/quality/tracking/collector.py +318 -0
  88. pycharter/quality/tracking/exporters.py +238 -0
  89. pycharter/quality/tracking/models.py +194 -0
  90. pycharter/quality/tracking/store.py +385 -0
  91. pycharter/runtime_validator/__init__.py +20 -7
  92. pycharter/runtime_validator/builder.py +328 -0
  93. pycharter/runtime_validator/validator.py +311 -7
  94. pycharter/runtime_validator/validator_core.py +61 -0
  95. pycharter/schema_evolution/__init__.py +61 -0
  96. pycharter/schema_evolution/compatibility.py +270 -0
  97. pycharter/schema_evolution/diff.py +496 -0
  98. pycharter/schema_evolution/models.py +201 -0
  99. pycharter/shared/__init__.py +56 -0
  100. pycharter/shared/errors.py +296 -0
  101. pycharter/shared/protocols.py +234 -0
  102. {pycharter-0.0.22.dist-info → pycharter-0.0.24.dist-info}/METADATA +146 -26
  103. pycharter-0.0.24.dist-info/RECORD +543 -0
  104. {pycharter-0.0.22.dist-info → pycharter-0.0.24.dist-info}/WHEEL +1 -1
  105. ui/static/404/index.html +1 -1
  106. ui/static/404.html +1 -1
  107. ui/static/__next.__PAGE__.txt +1 -1
  108. ui/static/__next._full.txt +1 -1
  109. ui/static/__next._head.txt +1 -1
  110. ui/static/__next._index.txt +1 -1
  111. ui/static/__next._tree.txt +1 -1
  112. ui/static/_next/static/chunks/26dfc590f7714c03.js +1 -0
  113. ui/static/_next/static/chunks/34d289e6db2ef551.js +1 -0
  114. ui/static/_next/static/chunks/99508d9d5869cc27.js +1 -0
  115. ui/static/_next/static/chunks/b313c35a6ba76574.js +1 -0
  116. ui/static/_not-found/__next._full.txt +1 -1
  117. ui/static/_not-found/__next._head.txt +1 -1
  118. ui/static/_not-found/__next._index.txt +1 -1
  119. ui/static/_not-found/__next._not-found.__PAGE__.txt +1 -1
  120. ui/static/_not-found/__next._not-found.txt +1 -1
  121. ui/static/_not-found/__next._tree.txt +1 -1
  122. ui/static/_not-found/index.html +1 -1
  123. ui/static/_not-found/index.txt +1 -1
  124. ui/static/contracts/__next._full.txt +2 -2
  125. ui/static/contracts/__next._head.txt +1 -1
  126. ui/static/contracts/__next._index.txt +1 -1
  127. ui/static/contracts/__next._tree.txt +1 -1
  128. ui/static/contracts/__next.contracts.__PAGE__.txt +2 -2
  129. ui/static/contracts/__next.contracts.txt +1 -1
  130. ui/static/contracts/index.html +1 -1
  131. ui/static/contracts/index.txt +2 -2
  132. ui/static/documentation/__next._full.txt +1 -1
  133. ui/static/documentation/__next._head.txt +1 -1
  134. ui/static/documentation/__next._index.txt +1 -1
  135. ui/static/documentation/__next._tree.txt +1 -1
  136. ui/static/documentation/__next.documentation.__PAGE__.txt +1 -1
  137. ui/static/documentation/__next.documentation.txt +1 -1
  138. ui/static/documentation/index.html +2 -2
  139. ui/static/documentation/index.txt +1 -1
  140. ui/static/index.html +1 -1
  141. ui/static/index.txt +1 -1
  142. ui/static/metadata/__next._full.txt +1 -1
  143. ui/static/metadata/__next._head.txt +1 -1
  144. ui/static/metadata/__next._index.txt +1 -1
  145. ui/static/metadata/__next._tree.txt +1 -1
  146. ui/static/metadata/__next.metadata.__PAGE__.txt +1 -1
  147. ui/static/metadata/__next.metadata.txt +1 -1
  148. ui/static/metadata/index.html +1 -1
  149. ui/static/metadata/index.txt +1 -1
  150. ui/static/quality/__next._full.txt +2 -2
  151. ui/static/quality/__next._head.txt +1 -1
  152. ui/static/quality/__next._index.txt +1 -1
  153. ui/static/quality/__next._tree.txt +1 -1
  154. ui/static/quality/__next.quality.__PAGE__.txt +2 -2
  155. ui/static/quality/__next.quality.txt +1 -1
  156. ui/static/quality/index.html +2 -2
  157. ui/static/quality/index.txt +2 -2
  158. ui/static/rules/__next._full.txt +1 -1
  159. ui/static/rules/__next._head.txt +1 -1
  160. ui/static/rules/__next._index.txt +1 -1
  161. ui/static/rules/__next._tree.txt +1 -1
  162. ui/static/rules/__next.rules.__PAGE__.txt +1 -1
  163. ui/static/rules/__next.rules.txt +1 -1
  164. ui/static/rules/index.html +1 -1
  165. ui/static/rules/index.txt +1 -1
  166. ui/static/schemas/__next._full.txt +1 -1
  167. ui/static/schemas/__next._head.txt +1 -1
  168. ui/static/schemas/__next._index.txt +1 -1
  169. ui/static/schemas/__next._tree.txt +1 -1
  170. ui/static/schemas/__next.schemas.__PAGE__.txt +1 -1
  171. ui/static/schemas/__next.schemas.txt +1 -1
  172. ui/static/schemas/index.html +1 -1
  173. ui/static/schemas/index.txt +1 -1
  174. ui/static/settings/__next._full.txt +1 -1
  175. ui/static/settings/__next._head.txt +1 -1
  176. ui/static/settings/__next._index.txt +1 -1
  177. ui/static/settings/__next._tree.txt +1 -1
  178. ui/static/settings/__next.settings.__PAGE__.txt +1 -1
  179. ui/static/settings/__next.settings.txt +1 -1
  180. ui/static/settings/index.html +1 -1
  181. ui/static/settings/index.txt +1 -1
  182. ui/static/static/404/index.html +1 -1
  183. ui/static/static/404.html +1 -1
  184. ui/static/static/__next.__PAGE__.txt +1 -1
  185. ui/static/static/__next._full.txt +2 -2
  186. ui/static/static/__next._head.txt +1 -1
  187. ui/static/static/__next._index.txt +2 -2
  188. ui/static/static/__next._tree.txt +2 -2
  189. ui/static/static/_next/static/chunks/13d4a0fbd74c1ee4.js +1 -0
  190. ui/static/static/_next/static/chunks/2edb43b48432ac04.js +441 -0
  191. ui/static/static/_next/static/chunks/d2363397e1b2bcab.css +1 -0
  192. ui/static/static/_next/static/chunks/f7d1a90dd75d2572.js +1 -0
  193. ui/static/static/_not-found/__next._full.txt +2 -2
  194. ui/static/static/_not-found/__next._head.txt +1 -1
  195. ui/static/static/_not-found/__next._index.txt +2 -2
  196. ui/static/static/_not-found/__next._not-found.__PAGE__.txt +1 -1
  197. ui/static/static/_not-found/__next._not-found.txt +1 -1
  198. ui/static/static/_not-found/__next._tree.txt +2 -2
  199. ui/static/static/_not-found/index.html +1 -1
  200. ui/static/static/_not-found/index.txt +2 -2
  201. ui/static/static/contracts/__next._full.txt +3 -3
  202. ui/static/static/contracts/__next._head.txt +1 -1
  203. ui/static/static/contracts/__next._index.txt +2 -2
  204. ui/static/static/contracts/__next._tree.txt +2 -2
  205. ui/static/static/contracts/__next.contracts.__PAGE__.txt +2 -2
  206. ui/static/static/contracts/__next.contracts.txt +1 -1
  207. ui/static/static/contracts/index.html +1 -1
  208. ui/static/static/contracts/index.txt +3 -3
  209. ui/static/static/documentation/__next._full.txt +3 -3
  210. ui/static/static/documentation/__next._head.txt +1 -1
  211. ui/static/static/documentation/__next._index.txt +2 -2
  212. ui/static/static/documentation/__next._tree.txt +2 -2
  213. ui/static/static/documentation/__next.documentation.__PAGE__.txt +2 -2
  214. ui/static/static/documentation/__next.documentation.txt +1 -1
  215. ui/static/static/documentation/index.html +2 -2
  216. ui/static/static/documentation/index.txt +3 -3
  217. ui/static/static/index.html +1 -1
  218. ui/static/static/index.txt +2 -2
  219. ui/static/static/metadata/__next._full.txt +2 -2
  220. ui/static/static/metadata/__next._head.txt +1 -1
  221. ui/static/static/metadata/__next._index.txt +2 -2
  222. ui/static/static/metadata/__next._tree.txt +2 -2
  223. ui/static/static/metadata/__next.metadata.__PAGE__.txt +1 -1
  224. ui/static/static/metadata/__next.metadata.txt +1 -1
  225. ui/static/static/metadata/index.html +1 -1
  226. ui/static/static/metadata/index.txt +2 -2
  227. ui/static/static/quality/__next._full.txt +2 -2
  228. ui/static/static/quality/__next._head.txt +1 -1
  229. ui/static/static/quality/__next._index.txt +2 -2
  230. ui/static/static/quality/__next._tree.txt +2 -2
  231. ui/static/static/quality/__next.quality.__PAGE__.txt +1 -1
  232. ui/static/static/quality/__next.quality.txt +1 -1
  233. ui/static/static/quality/index.html +2 -2
  234. ui/static/static/quality/index.txt +2 -2
  235. ui/static/static/rules/__next._full.txt +2 -2
  236. ui/static/static/rules/__next._head.txt +1 -1
  237. ui/static/static/rules/__next._index.txt +2 -2
  238. ui/static/static/rules/__next._tree.txt +2 -2
  239. ui/static/static/rules/__next.rules.__PAGE__.txt +1 -1
  240. ui/static/static/rules/__next.rules.txt +1 -1
  241. ui/static/static/rules/index.html +1 -1
  242. ui/static/static/rules/index.txt +2 -2
  243. ui/static/static/schemas/__next._full.txt +2 -2
  244. ui/static/static/schemas/__next._head.txt +1 -1
  245. ui/static/static/schemas/__next._index.txt +2 -2
  246. ui/static/static/schemas/__next._tree.txt +2 -2
  247. ui/static/static/schemas/__next.schemas.__PAGE__.txt +1 -1
  248. ui/static/static/schemas/__next.schemas.txt +1 -1
  249. ui/static/static/schemas/index.html +1 -1
  250. ui/static/static/schemas/index.txt +2 -2
  251. ui/static/static/settings/__next._full.txt +2 -2
  252. ui/static/static/settings/__next._head.txt +1 -1
  253. ui/static/static/settings/__next._index.txt +2 -2
  254. ui/static/static/settings/__next._tree.txt +2 -2
  255. ui/static/static/settings/__next.settings.__PAGE__.txt +1 -1
  256. ui/static/static/settings/__next.settings.txt +1 -1
  257. ui/static/static/settings/index.html +1 -1
  258. ui/static/static/settings/index.txt +2 -2
  259. ui/static/static/static/.gitkeep +0 -0
  260. ui/static/static/static/404/index.html +1 -0
  261. ui/static/static/static/404.html +1 -0
  262. ui/static/static/static/__next.__PAGE__.txt +10 -0
  263. ui/static/static/static/__next._full.txt +30 -0
  264. ui/static/static/static/__next._head.txt +7 -0
  265. ui/static/static/static/__next._index.txt +9 -0
  266. ui/static/static/static/__next._tree.txt +2 -0
  267. ui/static/static/static/_next/static/chunks/222442f6da32302a.js +1 -0
  268. ui/static/static/static/_next/static/chunks/247eb132b7f7b574.js +1 -0
  269. ui/static/static/static/_next/static/chunks/297d55555b71baba.js +1 -0
  270. ui/static/static/static/_next/static/chunks/2ab439ce003cd691.js +1 -0
  271. ui/static/static/static/_next/static/chunks/414e77373f8ff61c.js +1 -0
  272. ui/static/static/static/_next/static/chunks/49ca65abd26ae49e.js +1 -0
  273. ui/static/static/static/_next/static/chunks/652ad0aa26265c47.js +2 -0
  274. ui/static/static/static/_next/static/chunks/9667e7a3d359eb39.js +1 -0
  275. ui/static/static/static/_next/static/chunks/9c23f44fff36548a.js +1 -0
  276. ui/static/static/static/_next/static/chunks/a6dad97d9634a72d.js +1 -0
  277. ui/static/static/static/_next/static/chunks/b32a0963684b9933.js +4 -0
  278. ui/static/static/static/_next/static/chunks/c69f6cba366bd988.js +1 -0
  279. ui/static/static/static/_next/static/chunks/db913959c675cea6.js +1 -0
  280. ui/static/static/static/_next/static/chunks/f061a4be97bfc3b3.js +1 -0
  281. ui/static/static/static/_next/static/chunks/f2e7afeab1178138.js +1 -0
  282. ui/static/static/static/_next/static/chunks/ff1a16fafef87110.js +1 -0
  283. ui/static/static/static/_next/static/chunks/turbopack-ffcb7ab6794027ef.js +3 -0
  284. ui/static/static/static/_next/static/tNTkVW6puVXC4bAm4WrHl/_buildManifest.js +11 -0
  285. ui/static/static/static/_next/static/tNTkVW6puVXC4bAm4WrHl/_ssgManifest.js +1 -0
  286. ui/static/static/static/_not-found/__next._full.txt +17 -0
  287. ui/static/static/static/_not-found/__next._head.txt +7 -0
  288. ui/static/static/static/_not-found/__next._index.txt +9 -0
  289. ui/static/static/static/_not-found/__next._not-found.__PAGE__.txt +5 -0
  290. ui/static/static/static/_not-found/__next._not-found.txt +4 -0
  291. ui/static/static/static/_not-found/__next._tree.txt +2 -0
  292. ui/static/static/static/_not-found/index.html +1 -0
  293. ui/static/static/static/_not-found/index.txt +17 -0
  294. ui/static/static/static/contracts/__next._full.txt +21 -0
  295. ui/static/static/static/contracts/__next._head.txt +7 -0
  296. ui/static/static/static/contracts/__next._index.txt +9 -0
  297. ui/static/static/static/contracts/__next._tree.txt +2 -0
  298. ui/static/static/static/contracts/__next.contracts.__PAGE__.txt +9 -0
  299. ui/static/static/static/contracts/__next.contracts.txt +4 -0
  300. ui/static/static/static/contracts/index.html +1 -0
  301. ui/static/static/static/contracts/index.txt +21 -0
  302. ui/static/static/static/documentation/__next._full.txt +21 -0
  303. ui/static/static/static/documentation/__next._head.txt +7 -0
  304. ui/static/static/static/documentation/__next._index.txt +9 -0
  305. ui/static/static/static/documentation/__next._tree.txt +2 -0
  306. ui/static/static/static/documentation/__next.documentation.__PAGE__.txt +9 -0
  307. ui/static/static/static/documentation/__next.documentation.txt +4 -0
  308. ui/static/static/static/documentation/index.html +93 -0
  309. ui/static/static/static/documentation/index.txt +21 -0
  310. ui/static/static/static/index.html +1 -0
  311. ui/static/static/static/index.txt +30 -0
  312. ui/static/static/static/metadata/__next._full.txt +21 -0
  313. ui/static/static/static/metadata/__next._head.txt +7 -0
  314. ui/static/static/static/metadata/__next._index.txt +9 -0
  315. ui/static/static/static/metadata/__next._tree.txt +2 -0
  316. ui/static/static/static/metadata/__next.metadata.__PAGE__.txt +9 -0
  317. ui/static/static/static/metadata/__next.metadata.txt +4 -0
  318. ui/static/static/static/metadata/index.html +1 -0
  319. ui/static/static/static/metadata/index.txt +21 -0
  320. ui/static/static/static/quality/__next._full.txt +21 -0
  321. ui/static/static/static/quality/__next._head.txt +7 -0
  322. ui/static/static/static/quality/__next._index.txt +9 -0
  323. ui/static/static/static/quality/__next._tree.txt +2 -0
  324. ui/static/static/static/quality/__next.quality.__PAGE__.txt +9 -0
  325. ui/static/static/static/quality/__next.quality.txt +4 -0
  326. ui/static/static/static/quality/index.html +2 -0
  327. ui/static/static/static/quality/index.txt +21 -0
  328. ui/static/static/static/rules/__next._full.txt +21 -0
  329. ui/static/static/static/rules/__next._head.txt +7 -0
  330. ui/static/static/static/rules/__next._index.txt +9 -0
  331. ui/static/static/static/rules/__next._tree.txt +2 -0
  332. ui/static/static/static/rules/__next.rules.__PAGE__.txt +9 -0
  333. ui/static/static/static/rules/__next.rules.txt +4 -0
  334. ui/static/static/static/rules/index.html +1 -0
  335. ui/static/static/static/rules/index.txt +21 -0
  336. ui/static/static/static/schemas/__next._full.txt +21 -0
  337. ui/static/static/static/schemas/__next._head.txt +7 -0
  338. ui/static/static/static/schemas/__next._index.txt +9 -0
  339. ui/static/static/static/schemas/__next._tree.txt +2 -0
  340. ui/static/static/static/schemas/__next.schemas.__PAGE__.txt +9 -0
  341. ui/static/static/static/schemas/__next.schemas.txt +4 -0
  342. ui/static/static/static/schemas/index.html +1 -0
  343. ui/static/static/static/schemas/index.txt +21 -0
  344. ui/static/static/static/settings/__next._full.txt +21 -0
  345. ui/static/static/static/settings/__next._head.txt +7 -0
  346. ui/static/static/static/settings/__next._index.txt +9 -0
  347. ui/static/static/static/settings/__next._tree.txt +2 -0
  348. ui/static/static/static/settings/__next.settings.__PAGE__.txt +9 -0
  349. ui/static/static/static/settings/__next.settings.txt +4 -0
  350. ui/static/static/static/settings/index.html +1 -0
  351. ui/static/static/static/settings/index.txt +21 -0
  352. ui/static/static/static/validation/__next._full.txt +21 -0
  353. ui/static/static/static/validation/__next._head.txt +7 -0
  354. ui/static/static/static/validation/__next._index.txt +9 -0
  355. ui/static/static/static/validation/__next._tree.txt +2 -0
  356. ui/static/static/static/validation/__next.validation.__PAGE__.txt +9 -0
  357. ui/static/static/static/validation/__next.validation.txt +4 -0
  358. ui/static/static/static/validation/index.html +1 -0
  359. ui/static/static/static/validation/index.txt +21 -0
  360. ui/static/static/validation/__next._full.txt +2 -2
  361. ui/static/static/validation/__next._head.txt +1 -1
  362. ui/static/static/validation/__next._index.txt +2 -2
  363. ui/static/static/validation/__next._tree.txt +2 -2
  364. ui/static/static/validation/__next.validation.__PAGE__.txt +1 -1
  365. ui/static/static/validation/__next.validation.txt +1 -1
  366. ui/static/static/validation/index.html +1 -1
  367. ui/static/static/validation/index.txt +2 -2
  368. ui/static/validation/__next._full.txt +2 -2
  369. ui/static/validation/__next._head.txt +1 -1
  370. ui/static/validation/__next._index.txt +1 -1
  371. ui/static/validation/__next._tree.txt +1 -1
  372. ui/static/validation/__next.validation.__PAGE__.txt +2 -2
  373. ui/static/validation/__next.validation.txt +1 -1
  374. ui/static/validation/index.html +1 -1
  375. ui/static/validation/index.txt +2 -2
  376. pycharter/data/templates/template_coercion_rules.yaml +0 -15
  377. pycharter/data/templates/template_contract.yaml +0 -587
  378. pycharter/data/templates/template_metadata.yaml +0 -38
  379. pycharter/data/templates/template_schema.yaml +0 -22
  380. pycharter/data/templates/template_transform_advanced.yaml +0 -50
  381. pycharter/data/templates/template_transform_simple.yaml +0 -59
  382. pycharter/data/templates/template_validation_rules.yaml +0 -29
  383. pycharter/etl_generator/extraction.py +0 -916
  384. pycharter/etl_generator/factory.py +0 -174
  385. pycharter/etl_generator/orchestrator.py +0 -1650
  386. pycharter/integrations/__init__.py +0 -19
  387. pycharter/integrations/kafka.py +0 -178
  388. pycharter/integrations/streaming.py +0 -100
  389. pycharter-0.0.22.dist-info/RECORD +0 -358
  390. {pycharter-0.0.22.dist-info → pycharter-0.0.24.dist-info}/entry_points.txt +0 -0
  391. {pycharter-0.0.22.dist-info → pycharter-0.0.24.dist-info}/licenses/LICENSE +0 -0
  392. {pycharter-0.0.22.dist-info → pycharter-0.0.24.dist-info}/top_level.txt +0 -0
  393. /ui/static/_next/static/{0rYA78L88aUyD2Uh38hhX → 2gKjNv6YvE6BcIdFthBLs}/_buildManifest.js +0 -0
  394. /ui/static/_next/static/{0rYA78L88aUyD2Uh38hhX → 2gKjNv6YvE6BcIdFthBLs}/_ssgManifest.js +0 -0
  395. /ui/static/static/_next/static/{tNTkVW6puVXC4bAm4WrHl → 0rYA78L88aUyD2Uh38hhX}/_buildManifest.js +0 -0
  396. /ui/static/static/_next/static/{tNTkVW6puVXC4bAm4WrHl → 0rYA78L88aUyD2Uh38hhX}/_ssgManifest.js +0 -0
  397. /ui/static/{_next → static/_next}/static/chunks/c4fa4f4114b7c352.js +0 -0
  398. /ui/static/static/{_next → static/_next}/static/chunks/4e310fe5005770a3.css +0 -0
  399. /ui/static/{_next → static/static/_next}/static/chunks/5e04d10c4a7b58a3.js +0 -0
  400. /ui/static/static/{_next → static/_next}/static/chunks/5fc14c00a2779dc5.js +0 -0
  401. /ui/static/{_next → static/static/_next}/static/chunks/75d88a058d8ffaa6.js +0 -0
  402. /ui/static/{_next → static/static/_next}/static/chunks/8c89634cf6bad76f.js +0 -0
  403. /ui/static/static/{_next → static/_next}/static/chunks/b584574fdc8ab13e.js +0 -0
  404. /ui/static/static/{_next → static/_next}/static/chunks/d5989c94d3614b3a.js +0 -0
@@ -1,587 +0,0 @@
1
- schema:
2
- type: object
3
- title: Aircraft
4
- version: 1.0.0
5
- properties:
6
- metadata:
7
- description: Metadata when the record is processed
8
- title: Metadata
9
- type: object
10
- REGISTRATION:
11
- description: Registration of the Aircraft
12
- examples:
13
- - BHNR
14
- - LD33F
15
- maxLength: 10
16
- title: Registration
17
- type: string
18
- coercion: coerce_to_string
19
- validations:
20
- max_length:
21
- threshold: 10
22
- non_empty_string: null
23
- VALID_SINCE:
24
- description: Start of validity
25
- format: date-time
26
- title: Valid Since
27
- type: string
28
- coercion: coerce_to_datetime
29
- VALID_UNTIL:
30
- description: End of validity
31
- format: date-time
32
- title: Valid Until
33
- type: string
34
- coercion: coerce_to_datetime
35
- AC_OPERATOR:
36
- description: Aircraft Operator
37
- examples:
38
- - LD
39
- - CX
40
- maxLength: 3
41
- title: Ac Operator
42
- type: string
43
- coercion: coerce_to_string
44
- validations:
45
- max_length:
46
- threshold: 3
47
- non_empty_string: null
48
- AC_OWNER:
49
- description: Owner of the aircraft
50
- examples:
51
- - LD
52
- - CX
53
- maxLength: 3
54
- title: Ac Owner
55
- type: string
56
- coercion: coerce_to_string
57
- validations:
58
- max_length:
59
- threshold: 3
60
- non_empty_string: null
61
- AC_SUBTYPE:
62
- description: Subtype of the aircraft
63
- maxLength: 3
64
- title: Ac Subtype
65
- type: string
66
- coercion: coerce_to_string
67
- validations:
68
- max_length:
69
- threshold: 3
70
- non_empty_string: null
71
- AC_LOGICAL_NO:
72
- description: Logical number of the Aircraft
73
- title: Ac Logical No
74
- type: integer
75
- coercion: coerce_to_integer
76
- AC_STATE:
77
- description: Status of Aircraft(''R''=real and ''O''=overflow)
78
- enum:
79
- - R
80
- - O
81
- maxLength: 1
82
- title: Ac State
83
- type: string
84
- coercion: coerce_to_string
85
- validations:
86
- only_allow:
87
- allowed_values:
88
- - R
89
- - O
90
- DRY_OPERATING_WGT:
91
- description: Dry Operating Weight (kg)
92
- title: Dry Operating Wgt
93
- type: integer
94
- coercion: coerce_to_integer
95
- validations:
96
- greater_than_or_equal_to:
97
- threshold: 0
98
- MAX_TAKEOFF_WGT:
99
- description: Max Takeoff Weight (kg)
100
- title: Max Takeoff Wgt
101
- type: integer
102
- coercion: coerce_to_integer
103
- validations:
104
- greater_than_or_equal_to:
105
- threshold: 0
106
- CARGO_CAPACITY:
107
- description: Cargo Capacity (kg)
108
- title: Cargo Capacity
109
- type: integer
110
- coercion: coerce_to_integer
111
- validations:
112
- greater_than_or_equal_to:
113
- threshold: 0
114
- FUEL_CAPACITY:
115
- description: Fuel Capacity (kg)
116
- title: Fuel Capacity
117
- type: integer
118
- coercion: coerce_to_integer
119
- validations:
120
- greater_than_or_equal_to:
121
- threshold: 0
122
- AVG_FUEL_CONSUMP:
123
- description: Average Fuel Consumption (kg per hour)
124
- title: Avg Fuel Consump
125
- type: integer
126
- coercion: coerce_to_integer
127
- validations:
128
- greater_than_or_equal_to:
129
- threshold: 0
130
- AC_INDEX:
131
- description: Aircraft index
132
- title: Ac Index
133
- type: integer
134
- coercion: coerce_to_integer
135
- CREWSIZE_COCKPIT:
136
- description: Number of Cockpit Crewmembers
137
- title: Crewsize Cockpit
138
- type: integer
139
- coercion: coerce_to_integer
140
- validations:
141
- greater_than_or_equal_to:
142
- threshold: 0
143
- CREWSIZE_CABIN:
144
- description: Number of Cabin Crewmembers
145
- title: Crewsize Cabin
146
- type: integer
147
- coercion: coerce_to_integer
148
- validations:
149
- greater_than_or_equal_to:
150
- threshold: 0
151
- STD_VERSION:
152
- description: Aircraft Standard Version
153
- maxLength: 20
154
- title: Std Version
155
- type: string
156
- coercion: coerce_to_string
157
- validations:
158
- max_length:
159
- threshold: 20
160
- non_empty_string: null
161
- AP_RESTRICTION:
162
- description: Airport Restriction - e.g. Political Reason, Performance(''Y''
163
- or ''N'')
164
- examples:
165
- - N
166
- - Y
167
- - R
168
- maxLength: 1
169
- title: Ap Restriction
170
- type: string
171
- coercion: coerce_to_string
172
- validations:
173
- max_length:
174
- threshold: 1
175
- non_empty_string: null
176
- AC_OWNER_NAME:
177
- description: Full Name of Owner
178
- maxLength: 30
179
- title: Ac Owner Name
180
- type: string
181
- coercion: coerce_to_string
182
- validations:
183
- max_length:
184
- threshold: 30
185
- non_empty_string: null
186
- AC_SUBTYPE_NAME:
187
- description: Full Name of Subtype
188
- maxLength: 30
189
- title: Ac Subtype Name
190
- type: string
191
- coercion: coerce_to_string
192
- validations:
193
- max_length:
194
- threshold: 30
195
- non_empty_string: null
196
- AC_CATEGORY:
197
- description: SSIM-Category of Aircraft
198
- maxLength: 1
199
- title: Ac Category
200
- type: string
201
- coercion: coerce_to_string
202
- validations:
203
- max_length:
204
- threshold: 1
205
- non_empty_string: null
206
- REMARK:
207
- anyOf:
208
- - maxLength: 80
209
- type: string
210
- - type: 'null'
211
- default: null
212
- description: Additional remarks
213
- title: Remark
214
- coercion: coerce_to_nullable_string
215
- validations:
216
- max_length:
217
- threshold: 80
218
- FUEL_MEASURE_UNIT:
219
- description: Unit for fuel
220
- maxLength: 3
221
- title: Fuel Measure Unit
222
- type: string
223
- coercion: coerce_to_string
224
- validations:
225
- max_length:
226
- threshold: 3
227
- non_empty_string: null
228
- AC_CALLSIGN:
229
- anyOf:
230
- - maxLength: 5
231
- type: string
232
- - type: 'null'
233
- default: null
234
- description: Callsign of aircraft
235
- title: Ac Callsign
236
- coercion: coerce_to_nullable_string
237
- validations:
238
- max_length:
239
- threshold: 5
240
- RADIO:
241
- anyOf:
242
- - maxLength: 1
243
- type: string
244
- - type: 'null'
245
- default: null
246
- description: Radio type
247
- title: Radio
248
- coercion: coerce_to_nullable_string
249
- validations:
250
- max_length:
251
- threshold: 1
252
- NOISE:
253
- anyOf:
254
- - type: integer
255
- - type: 'null'
256
- default: null
257
- description: ICAO noise chapter
258
- title: Noise
259
- coercion: coerce_to_nullable_integer
260
- validations:
261
- greater_than_or_equal_to:
262
- threshold: 0
263
- PHONE:
264
- anyOf:
265
- - maxLength: 20
266
- type: string
267
- - type: 'null'
268
- default: null
269
- description: Onboard mobile phone number
270
- title: Phone
271
- coercion: coerce_to_nullable_string
272
- validations:
273
- max_length:
274
- threshold: 20
275
- SPECIAL_EQUIPMENT:
276
- anyOf:
277
- - maxLength: 40
278
- type: string
279
- - type: 'null'
280
- default: null
281
- description: Special equipment
282
- title: Special Equipment
283
- coercion: coerce_to_nullable_string
284
- validations:
285
- max_length:
286
- threshold: 40
287
- ACARS:
288
- anyOf:
289
- - maxLength: 1
290
- type: string
291
- - type: 'null'
292
- default: null
293
- description: ACARS type
294
- enum:
295
- - V
296
- - H
297
- - S
298
- title: Acars
299
- coercion: coerce_to_nullable_string
300
- validations:
301
- only_allow:
302
- allowed_values:
303
- - V
304
- - H
305
- - S
306
- ALT_REG:
307
- anyOf:
308
- - maxLength: 10
309
- type: string
310
- - type: 'null'
311
- default: null
312
- description: Alternate Registration
313
- title: Alt Reg
314
- coercion: coerce_to_nullable_string
315
- validations:
316
- max_length:
317
- threshold: 10
318
- RECORD_ID:
319
- default: 0
320
- description: Record ID
321
- title: Record Id
322
- type: integer
323
- coercion: coerce_to_integer
324
- LAST_UPDATE:
325
- anyOf:
326
- - format: date-time
327
- type: string
328
- - type: 'null'
329
- default: null
330
- description: Timestamp of last update
331
- title: Last Update
332
- coercion: coerce_to_nullable_datetime
333
- LAST_UPDATE_USER_ID:
334
- anyOf:
335
- - maxLength: 32
336
- type: string
337
- - type: 'null'
338
- default: null
339
- description: User ID of last update
340
- title: Last Update User Id
341
- coercion: coerce_to_nullable_string
342
- validations:
343
- max_length:
344
- threshold: 32
345
- STD_VERSION_ALT_1:
346
- anyOf:
347
- - maxLength: 20
348
- type: string
349
- - type: 'null'
350
- default: null
351
- description: Alternative Aircraft Version 1
352
- title: Std Version Alt 1
353
- coercion: coerce_to_nullable_string
354
- validations:
355
- max_length:
356
- threshold: 20
357
- STD_VERSION_ALT_2:
358
- anyOf:
359
- - maxLength: 20
360
- type: string
361
- - type: 'null'
362
- default: null
363
- description: Alternative Aircraft Version 2
364
- title: Std Version Alt 2
365
- coercion: coerce_to_nullable_string
366
- validations:
367
- max_length:
368
- threshold: 20
369
- STD_VERSION_ALT_3:
370
- anyOf:
371
- - maxLength: 20
372
- type: string
373
- - type: 'null'
374
- default: null
375
- description: Alternative Aircraft Version 3
376
- title: Std Version Alt 3
377
- coercion: coerce_to_nullable_string
378
- validations:
379
- max_length:
380
- threshold: 20
381
- STD_VERSION_ALT_4:
382
- anyOf:
383
- - maxLength: 20
384
- type: string
385
- - type: 'null'
386
- default: null
387
- description: Alternative Aircraft Version 4
388
- title: Std Version Alt 4
389
- coercion: coerce_to_nullable_string
390
- validations:
391
- max_length:
392
- threshold: 20
393
- ILS_EQUIPMENT:
394
- default: I
395
- description: aircraft equipment supporting ILS CAT approaches(I, II, IIIa, IIIb)
396
- enum:
397
- - I
398
- - II
399
- - IIIa
400
- - IIIb
401
- maxLength: 4
402
- title: Ils Equipment
403
- type: string
404
- coercion: coerce_to_string
405
- validations:
406
- only_allow:
407
- allowed_values:
408
- - I
409
- - II
410
- - IIIa
411
- - IIIb
412
- AUTOLAND:
413
- default: N
414
- description: Autoland capability indicator
415
- enum:
416
- - Y
417
- - N
418
- maxLength: 1
419
- title: Autoland
420
- type: string
421
- coercion: coerce_to_string
422
- validations:
423
- only_allow:
424
- allowed_values:
425
- - Y
426
- - N
427
- HOMEBASE:
428
- anyOf:
429
- - maxLength: 3
430
- type: string
431
- - type: 'null'
432
- default: null
433
- description: Homebase of the aircraft (IATA CODE)
434
- title: Homebase
435
- coercion: coerce_to_nullable_string
436
- validations:
437
- max_length:
438
- threshold: 3
439
- required:
440
- - metadata
441
- - REGISTRATION
442
- - VALID_SINCE
443
- - VALID_UNTIL
444
- - AC_OPERATOR
445
- - AC_OWNER
446
- - AC_SUBTYPE
447
- - AC_LOGICAL_NO
448
- - AC_STATE
449
- - DRY_OPERATING_WGT
450
- - MAX_TAKEOFF_WGT
451
- - CARGO_CAPACITY
452
- - FUEL_CAPACITY
453
- - AVG_FUEL_CONSUMP
454
- - AC_INDEX
455
- - CREWSIZE_COCKPIT
456
- - CREWSIZE_CABIN
457
- - STD_VERSION
458
- - AP_RESTRICTION
459
- - AC_OWNER_NAME
460
- - AC_SUBTYPE_NAME
461
- - AC_CATEGORY
462
- - FUEL_MEASURE_UNIT
463
- metadata:
464
- title: Aircraft
465
- status: active
466
- type: object
467
- description: Default metadata for Aircraft
468
- validation_title: Aircraft
469
- created_at: '2025-08-28T04:10:45.905705+00:00'
470
- updated_at: '2025-08-28T04:10:45.905712+00:00'
471
- created_by: IOC DE
472
- updated_by: IOC DE
473
- data_feed_association:
474
- - title: Aircraft
475
- type: Operational
476
- pulls_from:
477
- - NetlineOpsReplica
478
- pushes_to:
479
- - SCR
480
- - Fleetwise
481
- system_sources:
482
- - NetlineBase
483
- dependencies:
484
- - AircraftType
485
- - Operator
486
- - Airport
487
- business_owners:
488
- - IOC
489
- domain: NetlineOps configuration
490
- data_quality:
491
- accuracy: 0.0
492
- monitoring_enabled: false
493
- sla: null
494
- total_records: 905
495
- last_updated: '2024-01-01T00:00:00'
496
- governance_rules:
497
- data_retention:
498
- days: 2555
499
- reason: Regulatory compliance for aircraft records
500
- pii_fields:
501
- fields: []
502
- sensitivity: Internal
503
- access_control:
504
- read:
505
- - operations-team
506
- - data-team
507
- write:
508
- - operations-team
509
- business_rules:
510
- validity_period:
511
- description: VALID_SINCE must be before VALID_UNTIL
512
- enforced: true
513
- weight_consistency:
514
- description: MAX_TAKEOFF_WGT should be >= DRY_OPERATING_WGT
515
- enforced: false
516
- serve:
517
- api:
518
- type: GET
519
- endpoint: /v1/aircraft
520
- rate_limit: 120/min
521
- cache_ttl: 3600
522
- confluence_page: https://cathaypacific-prod.atlassian.net/wiki/spaces/DEN/pages/2631027294902/Aircraft
523
- params:
524
- - name: registration
525
- in: query
526
- required: false
527
- description: Filter by registration
528
- schema:
529
- type: string
530
- example: BLXN
531
- - name: effective_date
532
- in: query
533
- required: false
534
- description: Filter by effective date in ISO format, default will be request
535
- timestamp
536
- example: '2025-03-29T00:00:00Z'
537
- schema:
538
- type: string
539
- format: date-time
540
- indexes:
541
- - cluster: ioc-de-{env}
542
- database: internal
543
- collection: replica_nlo_aircraft_serve
544
- indexes:
545
- - aircraft_unique_index:
546
- registration: 1
547
- valid_since: 1
548
- unique: true
549
- background: true
550
- - cluster: ioc-de-{env}
551
- database: internal
552
- collection: replica_nlo_aircraft_staging
553
- indexes:
554
- - aircraft_unique_index:
555
- registration: 1
556
- valid_since: 1
557
- unique: true
558
- background: true
559
- changelog:
560
- - version: 1.0.0
561
- date: '2025-08-28T04:10:45.905715+00:00'
562
- author: IOC
563
- type: initial
564
- changes:
565
- - 'Initial metadata collected from upstream for: Aircraft'
566
- ownership:
567
- owner: operations-team
568
- team: data-engineering
569
- additional_info: {}
570
- governance_rules:
571
- data_retention:
572
- days: 2555
573
- reason: Regulatory compliance for aircraft records
574
- pii_fields:
575
- fields: []
576
- sensitivity: Internal
577
- access_control:
578
- read:
579
- - operations-team
580
- - data-team
581
- write:
582
- - operations-team
583
- versions:
584
- schema: 1.0.0
585
- coercion_rules: 1.0.0
586
- validation_rules: 1.0.0
587
- metadata: 1.0.0
@@ -1,38 +0,0 @@
1
- # Metadata Template
2
- # Complete metadata definition for your dataset
3
-
4
- # Basic Information
5
- title: my_metadata
6
- version: 1.0.0
7
- status: active
8
- type: object
9
- description: Metadata for your dataset
10
-
11
- # Audit Information
12
- created_at: '2025-01-01T00:00:00.000000+00:00'
13
- updated_at: '2025-01-01T00:00:00.000000+00:00'
14
- created_by: your_username
15
- updated_by: your_username
16
-
17
- # Data Lineage
18
- pulls_from: []
19
- pushes_to: []
20
- system_sources: []
21
-
22
- # Ownership
23
- business_owners: []
24
- bu_sme: []
25
- it_application_owners: []
26
- it_sme: []
27
- domains: []
28
-
29
- # Governance Rules
30
- governance_rules:
31
- data_retention:
32
- days: 365
33
- reason: Standard data retention policy
34
- pii_fields:
35
- fields: []
36
- business_criticality: "tier_3"
37
- data_classification: "internal"
38
- monitored: false
@@ -1,22 +0,0 @@
1
- type: object
2
- title: my_schema
3
- version: 1.0.0
4
- properties:
5
- id:
6
- type: integer
7
- description: Unique identifier
8
- title: ID
9
- name:
10
- type: string
11
- description: Name of the record
12
- title: Name
13
- maxLength: 100
14
- created_at:
15
- type: string
16
- format: date-time
17
- description: Creation timestamp
18
- title: Created At
19
- required:
20
- - id
21
- - name
22
- - created_at
@@ -1,50 +0,0 @@
1
- # Advanced Transformation Template
2
- # Use this template for complex transformations with JSONata and custom functions
3
-
4
- title: advanced_transformation
5
- description: Advanced transformation with JSONata and custom functions
6
- version: 1.0.0
7
-
8
- # Step 1: Simple operations (optional but recommended for basic cleanup)
9
- transform:
10
- rename:
11
- oldName: new_name
12
- convert:
13
- price: float
14
- timestamp: datetime
15
- defaults:
16
- source: "api"
17
-
18
- # Step 2: JSONata for complex transformations
19
- jsonata:
20
- expression: |
21
- $.{
22
- "ticker": symbol,
23
- "price_change": price - previousClose,
24
- "price_change_pct": ((price - previousClose) / previousClose) * 100,
25
- "is_positive": price > previousClose,
26
- "formatted_date": $fromMillis(timestamp),
27
- "metadata": {
28
- "source": source,
29
- "processed_at": $now()
30
- }
31
- }
32
- mode: "record" # Apply to each record individually
33
- # mode: "batch" # Apply to entire dataset
34
-
35
- # Step 3: Custom Python function for business logic
36
- custom_function:
37
- # Option 1: Using module and function
38
- module: "myproject.transforms"
39
- function: "optimize_portfolio"
40
- mode: "batch"
41
- kwargs:
42
- method: "min_volatility"
43
- solver: "ipopt"
44
- constraints:
45
- max_weight: 0.1
46
- min_weight: 0.01
47
-
48
- # Option 2: Using callable path (alternative)
49
- # callable: "myproject.transforms.optimize_portfolio"
50
- # mode: "batch"