keboola-cli 0.63.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (306) hide show
  1. keboola_agent_cli/__init__.py +34 -0
  2. keboola_agent_cli/__main__.py +5 -0
  3. keboola_agent_cli/_ui_dist/assets/arc-DhFYIddx.js +2 -0
  4. keboola_agent_cli/_ui_dist/assets/arc-DhFYIddx.js.map +1 -0
  5. keboola_agent_cli/_ui_dist/assets/architecture-7EHR7CIX-hNCijx_H.js +1 -0
  6. keboola_agent_cli/_ui_dist/assets/architectureDiagram-3BPJPVTR-C6hUlprM.js +37 -0
  7. keboola_agent_cli/_ui_dist/assets/architectureDiagram-3BPJPVTR-C6hUlprM.js.map +1 -0
  8. keboola_agent_cli/_ui_dist/assets/array-BifhSqXX.js +2 -0
  9. keboola_agent_cli/_ui_dist/assets/array-BifhSqXX.js.map +1 -0
  10. keboola_agent_cli/_ui_dist/assets/blockDiagram-GPEHLZMM-DC7qY9i4.js +133 -0
  11. keboola_agent_cli/_ui_dist/assets/blockDiagram-GPEHLZMM-DC7qY9i4.js.map +1 -0
  12. keboola_agent_cli/_ui_dist/assets/c4Diagram-AAUBKEIU-5Lh44evt.js +11 -0
  13. keboola_agent_cli/_ui_dist/assets/c4Diagram-AAUBKEIU-5Lh44evt.js.map +1 -0
  14. keboola_agent_cli/_ui_dist/assets/channel-DBMrXlxx.js +2 -0
  15. keboola_agent_cli/_ui_dist/assets/channel-DBMrXlxx.js.map +1 -0
  16. keboola_agent_cli/_ui_dist/assets/chunk-2J33WTMH-Coy82EBh.js +2 -0
  17. keboola_agent_cli/_ui_dist/assets/chunk-2J33WTMH-Coy82EBh.js.map +1 -0
  18. keboola_agent_cli/_ui_dist/assets/chunk-3OPIFGDE-BQC5CRHI.js +63 -0
  19. keboola_agent_cli/_ui_dist/assets/chunk-3OPIFGDE-BQC5CRHI.js.map +1 -0
  20. keboola_agent_cli/_ui_dist/assets/chunk-4BX2VUAB-DUuEt70o.js +2 -0
  21. keboola_agent_cli/_ui_dist/assets/chunk-4BX2VUAB-DUuEt70o.js.map +1 -0
  22. keboola_agent_cli/_ui_dist/assets/chunk-55IACEB6-BvR-6chF.js +2 -0
  23. keboola_agent_cli/_ui_dist/assets/chunk-55IACEB6-BvR-6chF.js.map +1 -0
  24. keboola_agent_cli/_ui_dist/assets/chunk-5ZQYHXKU-BjcTN7ul.js +3 -0
  25. keboola_agent_cli/_ui_dist/assets/chunk-5ZQYHXKU-BjcTN7ul.js.map +1 -0
  26. keboola_agent_cli/_ui_dist/assets/chunk-727SXJPM-C0zxqqRN.js +207 -0
  27. keboola_agent_cli/_ui_dist/assets/chunk-727SXJPM-C0zxqqRN.js.map +1 -0
  28. keboola_agent_cli/_ui_dist/assets/chunk-AQP2D5EJ-CXf7rIlZ.js +232 -0
  29. keboola_agent_cli/_ui_dist/assets/chunk-AQP2D5EJ-CXf7rIlZ.js.map +1 -0
  30. keboola_agent_cli/_ui_dist/assets/chunk-BSJP7CBP-Oj_FO9Q7.js +2 -0
  31. keboola_agent_cli/_ui_dist/assets/chunk-BSJP7CBP-Oj_FO9Q7.js.map +1 -0
  32. keboola_agent_cli/_ui_dist/assets/chunk-CSCIHK7Q-CcTsLrFc.js +124 -0
  33. keboola_agent_cli/_ui_dist/assets/chunk-CSCIHK7Q-CcTsLrFc.js.map +1 -0
  34. keboola_agent_cli/_ui_dist/assets/chunk-FMBD7UC4-FH-zLkkW.js +16 -0
  35. keboola_agent_cli/_ui_dist/assets/chunk-FMBD7UC4-FH-zLkkW.js.map +1 -0
  36. keboola_agent_cli/_ui_dist/assets/chunk-L5ZTLDWV-B1Ky_e7O.js +2 -0
  37. keboola_agent_cli/_ui_dist/assets/chunk-L5ZTLDWV-B1Ky_e7O.js.map +1 -0
  38. keboola_agent_cli/_ui_dist/assets/chunk-ND2GUHAM-BHz1rpbm.js +2 -0
  39. keboola_agent_cli/_ui_dist/assets/chunk-ND2GUHAM-BHz1rpbm.js.map +1 -0
  40. keboola_agent_cli/_ui_dist/assets/chunk-NNHCCRGN-DlpIbxXb.js +160 -0
  41. keboola_agent_cli/_ui_dist/assets/chunk-NNHCCRGN-DlpIbxXb.js.map +1 -0
  42. keboola_agent_cli/_ui_dist/assets/chunk-NZK2D7GU-tnrSoegS.js +2 -0
  43. keboola_agent_cli/_ui_dist/assets/chunk-NZK2D7GU-tnrSoegS.js.map +1 -0
  44. keboola_agent_cli/_ui_dist/assets/chunk-O5CBEL6O-DxxqDH0l.js +71 -0
  45. keboola_agent_cli/_ui_dist/assets/chunk-O5CBEL6O-DxxqDH0l.js.map +1 -0
  46. keboola_agent_cli/_ui_dist/assets/chunk-QZHKN3VN-CSjc2gjj.js +2 -0
  47. keboola_agent_cli/_ui_dist/assets/chunk-QZHKN3VN-CSjc2gjj.js.map +1 -0
  48. keboola_agent_cli/_ui_dist/assets/classDiagram-4FO5ZUOK-BuZcZu85.js +2 -0
  49. keboola_agent_cli/_ui_dist/assets/classDiagram-4FO5ZUOK-BuZcZu85.js.map +1 -0
  50. keboola_agent_cli/_ui_dist/assets/classDiagram-v2-Q7XG4LA2-BuZcZu85.js +2 -0
  51. keboola_agent_cli/_ui_dist/assets/classDiagram-v2-Q7XG4LA2-BuZcZu85.js.map +1 -0
  52. keboola_agent_cli/_ui_dist/assets/cose-bilkent-S5V4N54A-Y0L8LDMa.js +2 -0
  53. keboola_agent_cli/_ui_dist/assets/cose-bilkent-S5V4N54A-Y0L8LDMa.js.map +1 -0
  54. keboola_agent_cli/_ui_dist/assets/cytoscape.esm-C8YCVR3_.js +322 -0
  55. keboola_agent_cli/_ui_dist/assets/cytoscape.esm-C8YCVR3_.js.map +1 -0
  56. keboola_agent_cli/_ui_dist/assets/dagre-BM42HDAG-UZ-9BTqF.js +5 -0
  57. keboola_agent_cli/_ui_dist/assets/dagre-BM42HDAG-UZ-9BTqF.js.map +1 -0
  58. keboola_agent_cli/_ui_dist/assets/dagre-Bx709z4p.js +2 -0
  59. keboola_agent_cli/_ui_dist/assets/dagre-Bx709z4p.js.map +1 -0
  60. keboola_agent_cli/_ui_dist/assets/defaultLocale-C8Fc0cco.js +2 -0
  61. keboola_agent_cli/_ui_dist/assets/defaultLocale-C8Fc0cco.js.map +1 -0
  62. keboola_agent_cli/_ui_dist/assets/diagram-2AECGRRQ-DoDQ60wi.js +44 -0
  63. keboola_agent_cli/_ui_dist/assets/diagram-2AECGRRQ-DoDQ60wi.js.map +1 -0
  64. keboola_agent_cli/_ui_dist/assets/diagram-5GNKFQAL-CMGFxpUs.js +11 -0
  65. keboola_agent_cli/_ui_dist/assets/diagram-5GNKFQAL-CMGFxpUs.js.map +1 -0
  66. keboola_agent_cli/_ui_dist/assets/diagram-KO2AKTUF-1uGDa-Iu.js +4 -0
  67. keboola_agent_cli/_ui_dist/assets/diagram-KO2AKTUF-1uGDa-Iu.js.map +1 -0
  68. keboola_agent_cli/_ui_dist/assets/diagram-LMA3HP47-XtFH7B51.js +25 -0
  69. keboola_agent_cli/_ui_dist/assets/diagram-LMA3HP47-XtFH7B51.js.map +1 -0
  70. keboola_agent_cli/_ui_dist/assets/diagram-OG6HWLK6-B4_Te1T5.js +25 -0
  71. keboola_agent_cli/_ui_dist/assets/diagram-OG6HWLK6-B4_Te1T5.js.map +1 -0
  72. keboola_agent_cli/_ui_dist/assets/dist-Di6zmlv0.js +2 -0
  73. keboola_agent_cli/_ui_dist/assets/dist-Di6zmlv0.js.map +1 -0
  74. keboola_agent_cli/_ui_dist/assets/erDiagram-TEJ5UH35-NjQkrdFt.js +86 -0
  75. keboola_agent_cli/_ui_dist/assets/erDiagram-TEJ5UH35-NjQkrdFt.js.map +1 -0
  76. keboola_agent_cli/_ui_dist/assets/eventmodeling-FCH6USID-BrJMIks8.js +1 -0
  77. keboola_agent_cli/_ui_dist/assets/flowDiagram-I6XJVG4X-CIr8DWl7.js +163 -0
  78. keboola_agent_cli/_ui_dist/assets/flowDiagram-I6XJVG4X-CIr8DWl7.js.map +1 -0
  79. keboola_agent_cli/_ui_dist/assets/ganttDiagram-6RSMTGT7-C1VY_xbQ.js +293 -0
  80. keboola_agent_cli/_ui_dist/assets/ganttDiagram-6RSMTGT7-C1VY_xbQ.js.map +1 -0
  81. keboola_agent_cli/_ui_dist/assets/gitGraph-WXDBUCRP-COacYjo-.js +1 -0
  82. keboola_agent_cli/_ui_dist/assets/gitGraphDiagram-PVQCEYII-DQT8-kg2.js +107 -0
  83. keboola_agent_cli/_ui_dist/assets/gitGraphDiagram-PVQCEYII-DQT8-kg2.js.map +1 -0
  84. keboola_agent_cli/_ui_dist/assets/graphlib-B8gBHxth.js +2 -0
  85. keboola_agent_cli/_ui_dist/assets/graphlib-B8gBHxth.js.map +1 -0
  86. keboola_agent_cli/_ui_dist/assets/index-CMq50kkV.css +1 -0
  87. keboola_agent_cli/_ui_dist/assets/index-D8W97DAz.js +118 -0
  88. keboola_agent_cli/_ui_dist/assets/index-D8W97DAz.js.map +1 -0
  89. keboola_agent_cli/_ui_dist/assets/info-J43DQDTF-DdCTRIzU.js +1 -0
  90. keboola_agent_cli/_ui_dist/assets/infoDiagram-5YYISTIA-C77rsoTp.js +3 -0
  91. keboola_agent_cli/_ui_dist/assets/infoDiagram-5YYISTIA-C77rsoTp.js.map +1 -0
  92. keboola_agent_cli/_ui_dist/assets/init-D6jRqBbL.js +2 -0
  93. keboola_agent_cli/_ui_dist/assets/init-D6jRqBbL.js.map +1 -0
  94. keboola_agent_cli/_ui_dist/assets/ishikawaDiagram-YF4QCWOH-BcTbXaLy.js +71 -0
  95. keboola_agent_cli/_ui_dist/assets/ishikawaDiagram-YF4QCWOH-BcTbXaLy.js.map +1 -0
  96. keboola_agent_cli/_ui_dist/assets/journeyDiagram-JHISSGLW-BejeAJQ_.js +140 -0
  97. keboola_agent_cli/_ui_dist/assets/journeyDiagram-JHISSGLW-BejeAJQ_.js.map +1 -0
  98. keboola_agent_cli/_ui_dist/assets/kanban-definition-UN3LZRKU-BRNz_UrH.js +90 -0
  99. keboola_agent_cli/_ui_dist/assets/kanban-definition-UN3LZRKU-BRNz_UrH.js.map +1 -0
  100. keboola_agent_cli/_ui_dist/assets/katex-C4eR7coU.js +258 -0
  101. keboola_agent_cli/_ui_dist/assets/katex-C4eR7coU.js.map +1 -0
  102. keboola_agent_cli/_ui_dist/assets/line-CzAQKFbJ.js +2 -0
  103. keboola_agent_cli/_ui_dist/assets/line-CzAQKFbJ.js.map +1 -0
  104. keboola_agent_cli/_ui_dist/assets/linear-DUNFFdck.js +2 -0
  105. keboola_agent_cli/_ui_dist/assets/linear-DUNFFdck.js.map +1 -0
  106. keboola_agent_cli/_ui_dist/assets/mermaid-parser.core-CpuBOkFa.js +5 -0
  107. keboola_agent_cli/_ui_dist/assets/mermaid-parser.core-CpuBOkFa.js.map +1 -0
  108. keboola_agent_cli/_ui_dist/assets/mindmap-definition-RKZ34NQL-9EJQNjH0.js +97 -0
  109. keboola_agent_cli/_ui_dist/assets/mindmap-definition-RKZ34NQL-9EJQNjH0.js.map +1 -0
  110. keboola_agent_cli/_ui_dist/assets/ordinal-hYBb2elL.js +2 -0
  111. keboola_agent_cli/_ui_dist/assets/ordinal-hYBb2elL.js.map +1 -0
  112. keboola_agent_cli/_ui_dist/assets/packet-YPE3B663-DLiiw_B2.js +1 -0
  113. keboola_agent_cli/_ui_dist/assets/path-BWPyau1x.js +2 -0
  114. keboola_agent_cli/_ui_dist/assets/path-BWPyau1x.js.map +1 -0
  115. keboola_agent_cli/_ui_dist/assets/pie-LRSECV5Y-CRoO8G1g.js +1 -0
  116. keboola_agent_cli/_ui_dist/assets/pieDiagram-4H26LBE5-XH4cy6Cb.js +31 -0
  117. keboola_agent_cli/_ui_dist/assets/pieDiagram-4H26LBE5-XH4cy6Cb.js.map +1 -0
  118. keboola_agent_cli/_ui_dist/assets/quadrantDiagram-W4KKPZXB-fdhc93U8.js +8 -0
  119. keboola_agent_cli/_ui_dist/assets/quadrantDiagram-W4KKPZXB-fdhc93U8.js.map +1 -0
  120. keboola_agent_cli/_ui_dist/assets/radar-GUYGQ44K-DAlLVJHm.js +1 -0
  121. keboola_agent_cli/_ui_dist/assets/requirementDiagram-4Y6WPE33-a94eP3R9.js +85 -0
  122. keboola_agent_cli/_ui_dist/assets/requirementDiagram-4Y6WPE33-a94eP3R9.js.map +1 -0
  123. keboola_agent_cli/_ui_dist/assets/rough.esm-CSKSodPl.js +2 -0
  124. keboola_agent_cli/_ui_dist/assets/rough.esm-CSKSodPl.js.map +1 -0
  125. keboola_agent_cli/_ui_dist/assets/sankeyDiagram-5OEKKPKP-jcBa02sp.js +41 -0
  126. keboola_agent_cli/_ui_dist/assets/sankeyDiagram-5OEKKPKP-jcBa02sp.js.map +1 -0
  127. keboola_agent_cli/_ui_dist/assets/sequenceDiagram-3UESZ5HK-A5-GGM-e.js +163 -0
  128. keboola_agent_cli/_ui_dist/assets/sequenceDiagram-3UESZ5HK-A5-GGM-e.js.map +1 -0
  129. keboola_agent_cli/_ui_dist/assets/src-ZI-V_AF0.js +2 -0
  130. keboola_agent_cli/_ui_dist/assets/src-ZI-V_AF0.js.map +1 -0
  131. keboola_agent_cli/_ui_dist/assets/stateDiagram-AJRCARHV-BKAA5rqE.js +2 -0
  132. keboola_agent_cli/_ui_dist/assets/stateDiagram-AJRCARHV-BKAA5rqE.js.map +1 -0
  133. keboola_agent_cli/_ui_dist/assets/stateDiagram-v2-BHNVJYJU-DnJwJBsE.js +2 -0
  134. keboola_agent_cli/_ui_dist/assets/stateDiagram-v2-BHNVJYJU-DnJwJBsE.js.map +1 -0
  135. keboola_agent_cli/_ui_dist/assets/timeline-definition-PNZ67QCA-Cy39jp8b.js +121 -0
  136. keboola_agent_cli/_ui_dist/assets/timeline-definition-PNZ67QCA-Cy39jp8b.js.map +1 -0
  137. keboola_agent_cli/_ui_dist/assets/treeView-BLDUP644-DbLYl23-.js +1 -0
  138. keboola_agent_cli/_ui_dist/assets/treemap-LRROVOQU-Bp0eGlOt.js +1 -0
  139. keboola_agent_cli/_ui_dist/assets/vennDiagram-CIIHVFJN-BGECKubd.js +35 -0
  140. keboola_agent_cli/_ui_dist/assets/vennDiagram-CIIHVFJN-BGECKubd.js.map +1 -0
  141. keboola_agent_cli/_ui_dist/assets/wardley-L42UT6IY-D4yH4jqS.js +1 -0
  142. keboola_agent_cli/_ui_dist/assets/wardleyDiagram-YWT4CUSO-D6XRG3cZ.js +79 -0
  143. keboola_agent_cli/_ui_dist/assets/wardleyDiagram-YWT4CUSO-D6XRG3cZ.js.map +1 -0
  144. keboola_agent_cli/_ui_dist/assets/xychartDiagram-2RQKCTM6-DRre-pfZ.js +8 -0
  145. keboola_agent_cli/_ui_dist/assets/xychartDiagram-2RQKCTM6-DRre-pfZ.js.map +1 -0
  146. keboola_agent_cli/_ui_dist/index.html +50 -0
  147. keboola_agent_cli/ai_client.py +83 -0
  148. keboola_agent_cli/auto_update.py +550 -0
  149. keboola_agent_cli/changelog.py +1198 -0
  150. keboola_agent_cli/cli.py +448 -0
  151. keboola_agent_cli/client.py +3422 -0
  152. keboola_agent_cli/commands/__init__.py +0 -0
  153. keboola_agent_cli/commands/_data_app_git.py +343 -0
  154. keboola_agent_cli/commands/_helpers.py +377 -0
  155. keboola_agent_cli/commands/_metadata_input.py +49 -0
  156. keboola_agent_cli/commands/_semantic_layer_crud.py +632 -0
  157. keboola_agent_cli/commands/_semantic_layer_helpers.py +44 -0
  158. keboola_agent_cli/commands/_semantic_layer_reference_data.py +247 -0
  159. keboola_agent_cli/commands/agent.py +968 -0
  160. keboola_agent_cli/commands/branch.py +423 -0
  161. keboola_agent_cli/commands/changelog.py +168 -0
  162. keboola_agent_cli/commands/component.py +216 -0
  163. keboola_agent_cli/commands/config.py +2442 -0
  164. keboola_agent_cli/commands/context.py +1481 -0
  165. keboola_agent_cli/commands/data_app.py +1279 -0
  166. keboola_agent_cli/commands/dev_portal.py +584 -0
  167. keboola_agent_cli/commands/doctor.py +37 -0
  168. keboola_agent_cli/commands/encrypt.py +145 -0
  169. keboola_agent_cli/commands/feature.py +311 -0
  170. keboola_agent_cli/commands/flow.py +948 -0
  171. keboola_agent_cli/commands/http_client.py +157 -0
  172. keboola_agent_cli/commands/init.py +279 -0
  173. keboola_agent_cli/commands/job.py +661 -0
  174. keboola_agent_cli/commands/kai.py +301 -0
  175. keboola_agent_cli/commands/lineage.py +1464 -0
  176. keboola_agent_cli/commands/org.py +292 -0
  177. keboola_agent_cli/commands/permissions.py +360 -0
  178. keboola_agent_cli/commands/project.py +1192 -0
  179. keboola_agent_cli/commands/repl.py +243 -0
  180. keboola_agent_cli/commands/schedule.py +340 -0
  181. keboola_agent_cli/commands/search.py +178 -0
  182. keboola_agent_cli/commands/semantic_layer.py +939 -0
  183. keboola_agent_cli/commands/serve.py +272 -0
  184. keboola_agent_cli/commands/sharing.py +340 -0
  185. keboola_agent_cli/commands/storage.py +2630 -0
  186. keboola_agent_cli/commands/stream.py +266 -0
  187. keboola_agent_cli/commands/sync.py +1277 -0
  188. keboola_agent_cli/commands/tool.py +206 -0
  189. keboola_agent_cli/commands/version.py +186 -0
  190. keboola_agent_cli/commands/workspace.py +635 -0
  191. keboola_agent_cli/config_store.py +582 -0
  192. keboola_agent_cli/constants.py +528 -0
  193. keboola_agent_cli/data_science_client.py +342 -0
  194. keboola_agent_cli/dev_portal_client.py +323 -0
  195. keboola_agent_cli/errors.py +248 -0
  196. keboola_agent_cli/http_base.py +315 -0
  197. keboola_agent_cli/json_utils.py +126 -0
  198. keboola_agent_cli/lib.py +536 -0
  199. keboola_agent_cli/manage_client.py +324 -0
  200. keboola_agent_cli/metastore_client.py +214 -0
  201. keboola_agent_cli/models.py +427 -0
  202. keboola_agent_cli/output.py +1084 -0
  203. keboola_agent_cli/permissions.py +469 -0
  204. keboola_agent_cli/py.typed +3 -0
  205. keboola_agent_cli/result_models.py +271 -0
  206. keboola_agent_cli/server/__init__.py +34 -0
  207. keboola_agent_cli/server/agent_runner.py +1289 -0
  208. keboola_agent_cli/server/agents_store.py +325 -0
  209. keboola_agent_cli/server/app.py +764 -0
  210. keboola_agent_cli/server/auth.py +117 -0
  211. keboola_agent_cli/server/dependencies.py +149 -0
  212. keboola_agent_cli/server/pricing.py +303 -0
  213. keboola_agent_cli/server/routers/__init__.py +1 -0
  214. keboola_agent_cli/server/routers/agents.py +616 -0
  215. keboola_agent_cli/server/routers/ai_chat.py +129 -0
  216. keboola_agent_cli/server/routers/branches.py +133 -0
  217. keboola_agent_cli/server/routers/components.py +48 -0
  218. keboola_agent_cli/server/routers/configs.py +507 -0
  219. keboola_agent_cli/server/routers/data_apps.py +384 -0
  220. keboola_agent_cli/server/routers/dev_portal.py +67 -0
  221. keboola_agent_cli/server/routers/encrypt.py +35 -0
  222. keboola_agent_cli/server/routers/feature.py +179 -0
  223. keboola_agent_cli/server/routers/flows.py +204 -0
  224. keboola_agent_cli/server/routers/health.py +53 -0
  225. keboola_agent_cli/server/routers/jobs.py +175 -0
  226. keboola_agent_cli/server/routers/kai.py +80 -0
  227. keboola_agent_cli/server/routers/lineage.py +226 -0
  228. keboola_agent_cli/server/routers/mcp.py +70 -0
  229. keboola_agent_cli/server/routers/members.py +170 -0
  230. keboola_agent_cli/server/routers/org.py +96 -0
  231. keboola_agent_cli/server/routers/projects.py +106 -0
  232. keboola_agent_cli/server/routers/schedules.py +54 -0
  233. keboola_agent_cli/server/routers/search.py +30 -0
  234. keboola_agent_cli/server/routers/semantic_layer.py +650 -0
  235. keboola_agent_cli/server/routers/sharing.py +86 -0
  236. keboola_agent_cli/server/routers/storage.py +574 -0
  237. keboola_agent_cli/server/routers/stream.py +100 -0
  238. keboola_agent_cli/server/routers/workspaces.py +302 -0
  239. keboola_agent_cli/server/run_broadcaster.py +329 -0
  240. keboola_agent_cli/server/sse.py +25 -0
  241. keboola_agent_cli/services/__init__.py +0 -0
  242. keboola_agent_cli/services/_encryption.py +217 -0
  243. keboola_agent_cli/services/_semantic_layer_cascade.py +147 -0
  244. keboola_agent_cli/services/_semantic_layer_crud.py +382 -0
  245. keboola_agent_cli/services/_semantic_layer_internals.py +1078 -0
  246. keboola_agent_cli/services/_semantic_layer_lookup.py +181 -0
  247. keboola_agent_cli/services/_semantic_layer_reference_data.py +217 -0
  248. keboola_agent_cli/services/_sync_bindings.py +456 -0
  249. keboola_agent_cli/services/_sync_branch.py +191 -0
  250. keboola_agent_cli/services/_sync_bulk.py +228 -0
  251. keboola_agent_cli/services/_sync_clone.py +163 -0
  252. keboola_agent_cli/services/_sync_models.py +97 -0
  253. keboola_agent_cli/services/_sync_push_ops.py +369 -0
  254. keboola_agent_cli/services/_sync_storage.py +376 -0
  255. keboola_agent_cli/services/_sync_writeback.py +167 -0
  256. keboola_agent_cli/services/agent_service.py +458 -0
  257. keboola_agent_cli/services/base.py +175 -0
  258. keboola_agent_cli/services/branch_service.py +588 -0
  259. keboola_agent_cli/services/component_service.py +694 -0
  260. keboola_agent_cli/services/config_service.py +2099 -0
  261. keboola_agent_cli/services/data_app_git_service.py +224 -0
  262. keboola_agent_cli/services/data_app_service.py +2082 -0
  263. keboola_agent_cli/services/deep_lineage_service.py +1322 -0
  264. keboola_agent_cli/services/dev_portal_service.py +345 -0
  265. keboola_agent_cli/services/doctor_service.py +445 -0
  266. keboola_agent_cli/services/encrypt_service.py +87 -0
  267. keboola_agent_cli/services/feature_service.py +268 -0
  268. keboola_agent_cli/services/flow_service.py +769 -0
  269. keboola_agent_cli/services/flow_validation.py +188 -0
  270. keboola_agent_cli/services/http_forwarder_service.py +236 -0
  271. keboola_agent_cli/services/job_idempotency_store.py +285 -0
  272. keboola_agent_cli/services/job_service.py +797 -0
  273. keboola_agent_cli/services/kai_service.py +367 -0
  274. keboola_agent_cli/services/lineage_service.py +274 -0
  275. keboola_agent_cli/services/mcp_service.py +1498 -0
  276. keboola_agent_cli/services/mcp_transport.py +259 -0
  277. keboola_agent_cli/services/member_service.py +593 -0
  278. keboola_agent_cli/services/org_service.py +619 -0
  279. keboola_agent_cli/services/project_service.py +947 -0
  280. keboola_agent_cli/services/repo_validate_service.py +767 -0
  281. keboola_agent_cli/services/schedule_service.py +731 -0
  282. keboola_agent_cli/services/search_service.py +331 -0
  283. keboola_agent_cli/services/semantic_layer_service.py +1497 -0
  284. keboola_agent_cli/services/sharing_service.py +307 -0
  285. keboola_agent_cli/services/storage_service.py +2524 -0
  286. keboola_agent_cli/services/stream_service.py +395 -0
  287. keboola_agent_cli/services/sync_service.py +2244 -0
  288. keboola_agent_cli/services/variables_service.py +447 -0
  289. keboola_agent_cli/services/version_service.py +1038 -0
  290. keboola_agent_cli/services/workspace_service.py +1103 -0
  291. keboola_agent_cli/stream_client.py +217 -0
  292. keboola_agent_cli/sync/__init__.py +1 -0
  293. keboola_agent_cli/sync/branch_mapping.py +174 -0
  294. keboola_agent_cli/sync/clone.py +211 -0
  295. keboola_agent_cli/sync/code_extraction.py +655 -0
  296. keboola_agent_cli/sync/config_format.py +290 -0
  297. keboola_agent_cli/sync/diff_engine.py +566 -0
  298. keboola_agent_cli/sync/git_utils.py +93 -0
  299. keboola_agent_cli/sync/manifest.py +162 -0
  300. keboola_agent_cli/sync/naming.py +90 -0
  301. keboola_agent_cli/sync/secrets.py +62 -0
  302. keboola_agent_cli/sync/sql_split.py +134 -0
  303. keboola_cli-0.63.4.dist-info/METADATA +308 -0
  304. keboola_cli-0.63.4.dist-info/RECORD +306 -0
  305. keboola_cli-0.63.4.dist-info/WHEEL +4 -0
  306. keboola_cli-0.63.4.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,2244 @@
1
+ """Sync service - business logic for project pull/push/status operations.
2
+
3
+ Handles downloading Keboola project configurations to the local filesystem
4
+ in a dev-friendly format (YAML configs), and tracking local changes.
5
+ """
6
+
7
+ import hashlib
8
+ import json
9
+ import logging
10
+ import shutil
11
+ import subprocess
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ import yaml
16
+
17
+ from ..constants import (
18
+ ALWAYS_IGNORED_COMPONENTS,
19
+ BRANCH_MAPPING_FILENAME,
20
+ CONFIG_FILENAME,
21
+ DEFAULT_JOBS_PER_CONFIG,
22
+ DEFAULT_MAX_SAMPLES,
23
+ DEFAULT_SAMPLE_LIMIT,
24
+ KEBOOLA_DIR_NAME,
25
+ MANIFEST_VERSION,
26
+ )
27
+ from ..errors import ConfigError, ErrorCode, KeboolaApiError, SyncConflictError
28
+ from ..sync.code_extraction import extract_code_files, merge_code_files
29
+ from ..sync.config_format import (
30
+ api_config_to_local,
31
+ api_row_to_local,
32
+ classify_component_type,
33
+ local_config_to_api,
34
+ local_row_to_api,
35
+ )
36
+ from ..sync.diff_engine import compute_changeset, compute_row_changeset, config_hash
37
+ from ..sync.git_utils import get_default_branch, is_git_repo
38
+ from ..sync.manifest import (
39
+ Manifest,
40
+ ManifestBranch,
41
+ ManifestConfigRow,
42
+ ManifestConfiguration,
43
+ ManifestGitBranching,
44
+ ManifestNaming,
45
+ ManifestProject,
46
+ load_manifest,
47
+ save_manifest,
48
+ )
49
+ from ..sync.naming import config_path, config_row_path, sanitize_name
50
+ from ._encryption import (
51
+ encrypt_secrets_in_config,
52
+ find_plaintext_secret_keys,
53
+ )
54
+ from ._sync_bindings import resolve_flow_task_bindings, resolve_variable_bindings
55
+ from ._sync_branch import (
56
+ branch_link as _branch_link,
57
+ )
58
+ from ._sync_branch import (
59
+ branch_status as _branch_status,
60
+ )
61
+ from ._sync_branch import (
62
+ branch_unlink as _branch_unlink,
63
+ )
64
+ from ._sync_bulk import (
65
+ diff_all as _bulk_diff_all,
66
+ )
67
+ from ._sync_bulk import (
68
+ pull_all as _bulk_pull_all,
69
+ )
70
+ from ._sync_bulk import (
71
+ push_all as _bulk_push_all,
72
+ )
73
+ from ._sync_clone import clone_project as _clone_project_impl
74
+ from ._sync_models import CreatedConfig, LocalConfigHashes
75
+ from ._sync_push_ops import push_create, push_row_change, push_update
76
+ from ._sync_storage import (
77
+ fetch_jobs_per_config,
78
+ fetch_samples,
79
+ write_per_config_jobs,
80
+ write_storage_metadata,
81
+ )
82
+ from ._sync_writeback import (
83
+ propagate_kbc_metadata,
84
+ writeback_create_config_in_manifest,
85
+ )
86
+ from .base import BaseService
87
+
88
+ logger = logging.getLogger(__name__)
89
+
90
+ # Companion files extracted alongside a config's ``_config.yml`` whose hashes
91
+ # are tracked so ``sync diff`` notices edits to code/description files.
92
+ _EXTRA_HASH_FILENAMES: tuple[str, ...] = (
93
+ "_description.md",
94
+ "transform.sql",
95
+ "transform.py",
96
+ "code.py",
97
+ "pyproject.toml",
98
+ )
99
+
100
+
101
+ def _ensure_within_branch(
102
+ branch_dir: Path,
103
+ config_dir: Path,
104
+ component_id: str,
105
+ config_id: str,
106
+ ) -> None:
107
+ """Reject paths that escape the branch directory (issue #269 sec-01).
108
+
109
+ Resolves both paths and checks that *config_dir* is contained in
110
+ *branch_dir*. Raises ConfigError if not. Defense-in-depth on top of
111
+ ``naming.sanitize_path_segment()`` so a regression in either layer
112
+ cannot turn into a path-traversal write.
113
+ """
114
+ try:
115
+ branch_resolved = branch_dir.resolve()
116
+ config_resolved = config_dir.resolve()
117
+ except OSError as exc:
118
+ raise ConfigError(f"Cannot resolve sync path: {exc}") from exc
119
+ if not config_resolved.is_relative_to(branch_resolved):
120
+ raise ConfigError(
121
+ f"Config path escapes sync workspace (component='{component_id}', "
122
+ f"config_id='{config_id}'). Refusing to write outside "
123
+ f"'{branch_resolved}'. This indicates a malformed API response "
124
+ f"or a regression in path sanitization."
125
+ )
126
+
127
+
128
+ def scan_synced_plaintext_secrets(
129
+ project_root: Path, manifest: Manifest | None = None
130
+ ) -> list[dict[str, Any]]:
131
+ """Find in-sync configs/rows whose local files still hold plaintext #-secrets.
132
+
133
+ Regression guard for issue #378. A config that is *in sync* with the remote
134
+ (local file hash == manifest ``pull_hash``) but whose ``#``-prefixed values
135
+ are NOT ``KBC::``-encrypted means the remote is holding that secret in
136
+ plaintext -- the value passed through the sync baseline unencrypted (written
137
+ by a pre-0.54.0 ``config``/``sync`` path, or pulled from an already-leaked
138
+ remote). Pending local edits (hash != ``pull_hash``) are deliberately
139
+ skipped: there a ``sync push`` on >=0.54.0 encrypts on write, so flagging
140
+ them would be noise.
141
+
142
+ Read-only -- filesystem + manifest only, no API client. Returns one entry
143
+ per affected config/row with the secret *key paths* (never the values).
144
+
145
+ Args:
146
+ project_root: Root of the sync working tree.
147
+ manifest: Already-loaded manifest to reuse (callers like
148
+ :meth:`SyncService.status` load it themselves). Loaded from
149
+ *project_root* when ``None``.
150
+
151
+ Raises:
152
+ FileNotFoundError: if *manifest* is ``None`` and *project_root* has no
153
+ ``.keboola/manifest.json``.
154
+ """
155
+ if manifest is None:
156
+ manifest = load_manifest(project_root)
157
+ warnings: list[dict[str, Any]] = []
158
+
159
+ def _branch_path(branch_id: int | None) -> str:
160
+ if branch_id is not None:
161
+ for b in manifest.branches:
162
+ if b.id == branch_id:
163
+ return b.path
164
+ return manifest.branches[0].path if manifest.branches else "main"
165
+
166
+ def _read_yaml(path: Path) -> dict[str, Any] | None:
167
+ if not path.exists():
168
+ return None
169
+ try:
170
+ parsed = yaml.safe_load(path.read_text(encoding="utf-8"))
171
+ except yaml.YAMLError:
172
+ return None
173
+ return parsed if isinstance(parsed, dict) else None
174
+
175
+ def _in_sync(path: Path, pull_hash: str) -> bool:
176
+ if not pull_hash or not path.exists():
177
+ return False
178
+ return hashlib.sha256(path.read_bytes()).hexdigest() == pull_hash
179
+
180
+ for cfg in manifest.configurations:
181
+ config_dir = project_root / _branch_path(cfg.branch_id) / cfg.path
182
+ config_file = config_dir / CONFIG_FILENAME
183
+ data = _read_yaml(config_file)
184
+ if data is not None and _in_sync(config_file, cfg.metadata.get("pull_hash", "")):
185
+ _name, _desc, configuration = local_config_to_api(data)
186
+ keys = find_plaintext_secret_keys(configuration)
187
+ if keys:
188
+ warnings.append(
189
+ {
190
+ "component_id": cfg.component_id,
191
+ "config_id": cfg.id,
192
+ "path": str(cfg.path),
193
+ "scope": "config",
194
+ "secret_keys": keys,
195
+ }
196
+ )
197
+
198
+ for row in cfg.rows:
199
+ row_file = config_dir / row.path / CONFIG_FILENAME
200
+ row_data = _read_yaml(row_file)
201
+ if row_data is not None and _in_sync(row_file, row.metadata.get("pull_hash", "")):
202
+ _name, _desc, row_cfg = local_row_to_api(row_data, cfg.component_id)
203
+ row_keys = find_plaintext_secret_keys(row_cfg)
204
+ if row_keys:
205
+ warnings.append(
206
+ {
207
+ "component_id": cfg.component_id,
208
+ "config_id": cfg.id,
209
+ "row_id": row.id,
210
+ "path": f"{cfg.path}/{row.path}",
211
+ "scope": "row",
212
+ "secret_keys": row_keys,
213
+ }
214
+ )
215
+
216
+ return warnings
217
+
218
+
219
+ class SyncService(BaseService):
220
+ """Business logic for project sync operations (init, pull, status).
221
+
222
+ Single-project operations only. Uses dependency injection for
223
+ config_store and client_factory following the BaseService pattern.
224
+ """
225
+
226
+ # ------------------------------------------------------------------
227
+ # init
228
+ # ------------------------------------------------------------------
229
+
230
+ def init_sync(
231
+ self,
232
+ alias: str,
233
+ project_root: Path,
234
+ git_branching: bool = False,
235
+ adopt_existing: bool = False,
236
+ ) -> dict[str, Any]:
237
+ """Initialize a sync working directory for a project.
238
+
239
+ Creates the ``.keboola/`` directory with ``manifest.json``.
240
+ Fetches project metadata from the API to populate the manifest.
241
+
242
+ Args:
243
+ alias: Project alias from config store.
244
+ project_root: Root directory for the sync working tree.
245
+ git_branching: Enable git-branching mode.
246
+ adopt_existing: If True and a manifest already exists, validate it
247
+ against the alias's project_id and normalise it (idempotent
248
+ upgrade of a ``kbc``-written manifest) instead of refusing.
249
+
250
+ Returns:
251
+ Dict with initialization stats and created file paths.
252
+
253
+ Raises:
254
+ ConfigError: If the project alias is not found, or if
255
+ ``adopt_existing`` is True but the manifest's project_id does
256
+ not match the alias's project.
257
+ FileExistsError: If manifest already exists and adopt_existing is
258
+ False (use ``sync pull`` to update).
259
+ """
260
+ projects = self.resolve_projects([alias])
261
+ project = projects[alias]
262
+
263
+ keboola_dir = project_root / KEBOOLA_DIR_NAME
264
+ manifest_path = keboola_dir / "manifest.json"
265
+ if manifest_path.exists():
266
+ if adopt_existing:
267
+ return self._adopt_existing_manifest(alias, project_root, project)
268
+ raise FileExistsError(
269
+ f"Manifest already exists at {manifest_path}. "
270
+ "Use 'sync pull' to update, 'sync init --adopt-existing' to adopt a "
271
+ "kbc-written manifest, or delete .keboola/ to reinitialize."
272
+ )
273
+
274
+ # Fetch project info from API
275
+ client = self._client_factory(project.stack_url, project.token)
276
+ with client:
277
+ token_info = client.verify_token()
278
+ branches = client.list_dev_branches()
279
+
280
+ project_id = token_info.project_id
281
+ if project_id is None:
282
+ raise ConfigError("Token verification returned no project ID; cannot build manifest.")
283
+ api_host = project.stack_url.replace("https://", "").rstrip("/")
284
+ default_branch_info = next(
285
+ (b for b in branches if b.get("isDefault")),
286
+ None,
287
+ )
288
+ default_branch_id = default_branch_info["id"] if default_branch_info else None
289
+ default_branch_name = "main"
290
+
291
+ # Git branching setup
292
+ git_branching_config = ManifestGitBranching(enabled=False)
293
+ if git_branching:
294
+ if not is_git_repo(project_root):
295
+ raise ConfigError("Git repository not found. Initialize git first: git init")
296
+ default_branch_name = get_default_branch(project_root)
297
+ git_branching_config = ManifestGitBranching(
298
+ enabled=True,
299
+ defaultBranch=default_branch_name,
300
+ )
301
+
302
+ # Build manifest
303
+ manifest = Manifest(
304
+ version=MANIFEST_VERSION,
305
+ project=ManifestProject(id=project_id, apiHost=api_host),
306
+ allowTargetEnv=True,
307
+ gitBranching=git_branching_config,
308
+ naming=ManifestNaming(),
309
+ branches=[
310
+ ManifestBranch(
311
+ id=default_branch_id,
312
+ path=default_branch_name,
313
+ )
314
+ ]
315
+ if default_branch_id
316
+ else [],
317
+ configurations=[],
318
+ )
319
+
320
+ # Save manifest
321
+ save_manifest(project_root, manifest)
322
+
323
+ created_files = [str(manifest_path)]
324
+
325
+ # Create branch mapping if git-branching mode
326
+ if git_branching:
327
+ mapping = {
328
+ "version": 1,
329
+ "mappings": {
330
+ default_branch_name: {
331
+ "id": None,
332
+ "name": "Main",
333
+ }
334
+ },
335
+ }
336
+ mapping_path = keboola_dir / BRANCH_MAPPING_FILENAME
337
+ mapping_path.write_text(
338
+ json.dumps(mapping, indent=4, ensure_ascii=False) + "\n",
339
+ encoding="utf-8",
340
+ )
341
+ created_files.append(str(mapping_path))
342
+
343
+ return {
344
+ "status": "initialized",
345
+ "project_id": project_id,
346
+ "project_alias": alias,
347
+ "api_host": api_host,
348
+ "git_branching": git_branching,
349
+ "default_branch": default_branch_name,
350
+ "files_created": created_files,
351
+ }
352
+
353
+ def _adopt_existing_manifest(
354
+ self,
355
+ alias: str,
356
+ project_root: Path,
357
+ project: Any,
358
+ ) -> dict[str, Any]:
359
+ """Validate and normalise an existing manifest written by kbc or kbagent.
360
+
361
+ Idempotent: loads the manifest, confirms project_id matches the alias,
362
+ then saves it back through kbagent's serialiser (fills missing optional
363
+ fields with defaults, normalises camelCase keys, preserves all
364
+ existing content).
365
+
366
+ Raises:
367
+ ConfigError: If the manifest's project_id doesn't match the alias.
368
+ """
369
+ existing = load_manifest(project_root)
370
+
371
+ client = self._client_factory(project.stack_url, project.token)
372
+ with client:
373
+ token_info = client.verify_token()
374
+
375
+ if existing.project.id != token_info.project_id:
376
+ raise ConfigError(
377
+ f"Manifest project_id={existing.project.id} does not match alias "
378
+ f"'{alias}' project_id={token_info.project_id}; refusing to overwrite. "
379
+ "Check that --project points to the correct alias."
380
+ )
381
+
382
+ api_host = project.stack_url.replace("https://", "").rstrip("/")
383
+ save_manifest(project_root, existing)
384
+
385
+ return {
386
+ "status": "adopted",
387
+ "project_id": token_info.project_id,
388
+ "project_alias": alias,
389
+ "api_host": api_host,
390
+ "git_branching": existing.git_branching.enabled,
391
+ "default_branch": existing.git_branching.default_branch,
392
+ "files_created": [],
393
+ }
394
+
395
+ # ------------------------------------------------------------------
396
+ # pull
397
+ # ------------------------------------------------------------------
398
+
399
+ def _is_conflict(
400
+ self,
401
+ config_file: Path,
402
+ old_pull_hash: str,
403
+ old_cfg_hash: str,
404
+ api_cfg_hash: str,
405
+ ) -> bool:
406
+ """True iff the file is locally modified AND the remote also changed.
407
+
408
+ A 3-way conflict needs both a stored ``pull_hash`` (the synced file
409
+ state) and a stored ``pull_config_hash`` (the synced remote state);
410
+ without either we cannot prove a conflict, so return False -- be
411
+ conservative, ``--force`` must not abort on incomplete bookkeeping.
412
+ A missing local file is not a content conflict (nothing to lose).
413
+ """
414
+ if not old_pull_hash or not old_cfg_hash:
415
+ return False
416
+ if not config_file.exists():
417
+ return False
418
+ locally_modified = self._file_hash(config_file) != old_pull_hash
419
+ remote_changed = api_cfg_hash != old_cfg_hash
420
+ return locally_modified and remote_changed
421
+
422
+ def _detect_force_pull_conflicts(
423
+ self,
424
+ components: list[dict[str, Any]],
425
+ branch_dir: Path,
426
+ *,
427
+ existing_keys: set[str],
428
+ existing_paths: dict[str, str],
429
+ existing_file_hashes: dict[str, str],
430
+ existing_config_hashes: dict[str, str],
431
+ existing_rows: dict[str, dict[str, str]],
432
+ ) -> list[dict[str, str]]:
433
+ """Return configs/rows a ``--force`` pull would clobber as conflicts.
434
+
435
+ A *conflict* is a config (or row) that is BOTH locally modified (its
436
+ on-disk ``_config.yml`` hash differs from the manifest ``pull_hash``)
437
+ AND changed on the remote since the last pull (the freshly fetched
438
+ config hash differs from ``pull_config_hash``). That is the only case
439
+ where ``--force`` must stop: local and remote have diverged, so neither
440
+ "take remote" nor "keep local" is safe without the user deciding.
441
+
442
+ Configs only locally modified (remote unchanged) are NOT conflicts --
443
+ ``--force`` preserves them so their pending delta stays pushable.
444
+ Brand-new remote configs and configs whose local file is missing are
445
+ skipped (nothing local to lose). Read-only: hashes but writes nothing.
446
+ """
447
+ conflicts: list[dict[str, str]] = []
448
+ for component in components:
449
+ component_id = component.get("id", "")
450
+ if component_id in ALWAYS_IGNORED_COMPONENTS:
451
+ continue
452
+ for cfg in component.get("configurations", []):
453
+ config_id = str(cfg.get("id", ""))
454
+ lookup_key = f"{component_id}/{config_id}"
455
+ if lookup_key not in existing_keys:
456
+ continue # brand-new remote config -- nothing local to lose
457
+
458
+ rel_path = existing_paths.get(lookup_key, "")
459
+ api_cfg_hash = config_hash(api_config_to_local(component_id, cfg, config_id))
460
+ if self._is_conflict(
461
+ branch_dir / rel_path / CONFIG_FILENAME,
462
+ existing_file_hashes.get(lookup_key, ""),
463
+ existing_config_hashes.get(lookup_key, ""),
464
+ api_cfg_hash,
465
+ ):
466
+ conflicts.append(
467
+ {
468
+ "scope": "config",
469
+ "component_id": component_id,
470
+ "config_id": config_id,
471
+ "config_name": str(cfg.get("name", "untitled")),
472
+ "path": rel_path,
473
+ }
474
+ )
475
+
476
+ # Row-level conflicts (same 3-way rule, per row).
477
+ config_dir = branch_dir / rel_path
478
+ for row in cfg.get("rows", []):
479
+ row_id = str(row.get("id", ""))
480
+ existing_row = existing_rows.get(f"{component_id}/{config_id}/{row_id}")
481
+ if not existing_row:
482
+ continue
483
+ row_rel_path = existing_row.get("path", "")
484
+ if self._is_conflict(
485
+ config_dir / row_rel_path / CONFIG_FILENAME,
486
+ existing_row.get("pull_hash", ""),
487
+ existing_row.get("pull_config_hash", ""),
488
+ config_hash(api_row_to_local(row, component_id)),
489
+ ):
490
+ conflicts.append(
491
+ {
492
+ "scope": "row",
493
+ "component_id": component_id,
494
+ "config_id": config_id,
495
+ "config_name": (
496
+ f"{cfg.get('name', 'untitled')}/{row.get('name', 'untitled')}"
497
+ ),
498
+ "path": f"{rel_path}/{row_rel_path}",
499
+ "row_id": row_id,
500
+ }
501
+ )
502
+ return conflicts
503
+
504
+ def pull(
505
+ self,
506
+ alias: str,
507
+ project_root: Path,
508
+ force: bool = False,
509
+ dry_run: bool = False,
510
+ job_limit: int = DEFAULT_JOBS_PER_CONFIG,
511
+ no_storage: bool = False,
512
+ no_jobs: bool = False,
513
+ with_samples: bool = False,
514
+ sample_limit: int = DEFAULT_SAMPLE_LIMIT,
515
+ max_samples: int = DEFAULT_MAX_SAMPLES,
516
+ branch_override: int | None = None,
517
+ ) -> dict[str, Any]:
518
+ """Download all configurations from Keboola to local filesystem.
519
+
520
+ Args:
521
+ alias: Project alias from config store.
522
+ project_root: Root directory of the sync working tree.
523
+ force: If True, overwrite existing local files without checking.
524
+ dry_run: If True, compute what would be pulled but don't write.
525
+ job_limit: Max jobs per config to pull (default 5).
526
+ no_storage: Skip storage metadata download.
527
+ no_jobs: Skip per-config jobs download.
528
+ with_samples: Download table data samples (opt-in).
529
+ sample_limit: Max rows per sample (default 100).
530
+ max_samples: Max number of tables to sample (default 50).
531
+ branch_override: If set, pull from this branch ID rather than
532
+ the resolved active/manifest branch (CLI ``--branch``).
533
+
534
+ Returns:
535
+ Dict with pull statistics (configs, rows, files written).
536
+ """
537
+ projects = self.resolve_projects([alias])
538
+ project = projects[alias]
539
+
540
+ # Load or verify manifest exists
541
+ manifest = load_manifest(project_root)
542
+
543
+ # Determine branch to pull from (git-branching aware)
544
+ branch_id = self._resolve_branch_id(
545
+ project, manifest, project_root, branch_override=branch_override
546
+ )
547
+
548
+ # Fetch all components with configs from API (+ storage metadata + jobs)
549
+ client = self._client_factory(project.stack_url, project.token)
550
+ buckets_data: list[dict[str, Any]] = []
551
+ tables_data: list[dict[str, Any]] = []
552
+ jobs_grouped: list[dict[str, Any]] = []
553
+ samples_data: dict[str, str] = {} # table_id -> CSV string
554
+ with client:
555
+ components = client.list_components_with_configs(branch_id=branch_id)
556
+ self._ensure_branch_registered(manifest, branch_id, client)
557
+
558
+ if not no_storage:
559
+ try:
560
+ buckets_data = client.list_buckets_with_metadata()
561
+ tables_data = client.list_tables_with_metadata()
562
+ except Exception:
563
+ logger.warning("Failed to fetch storage metadata", exc_info=True)
564
+
565
+ if not no_jobs:
566
+ try:
567
+ # API constraint: jobsPerGroup * limit <= 500
568
+ group_limit = min(500 // max(job_limit, 1), 500)
569
+ total_configs = sum(len(comp.get("configurations", [])) for comp in components)
570
+
571
+ if group_limit >= total_configs:
572
+ # Fast path: one grouped API call covers all configs
573
+ jobs_grouped = client.list_jobs_grouped(
574
+ jobs_per_group=job_limit,
575
+ limit=group_limit,
576
+ )
577
+ else:
578
+ # Slow path: too many configs for grouped-jobs limit,
579
+ # fetch per-config via /search/jobs in parallel
580
+ logger.info(
581
+ "Project has %d configs but grouped-jobs limit is %d "
582
+ "(job_limit=%d); falling back to per-config fetching",
583
+ total_configs,
584
+ group_limit,
585
+ job_limit,
586
+ )
587
+ jobs_grouped = fetch_jobs_per_config(self, client, components, job_limit)
588
+ except Exception:
589
+ logger.warning("Failed to fetch jobs", exc_info=True)
590
+
591
+ if with_samples and tables_data:
592
+ samples_data = fetch_samples(client, tables_data, sample_limit, max_samples)
593
+
594
+ # Determine branch directory name
595
+ branch_dir_name = self._find_branch_path(manifest, branch_id)
596
+
597
+ branch_dir = project_root / branch_dir_name
598
+
599
+ # Track stats and change details
600
+ configs_pulled = 0
601
+ rows_pulled = 0
602
+ files_written = 0
603
+ new_configurations: list[ManifestConfiguration] = []
604
+ used_paths: set[str] = set() # detect naming collisions
605
+ pull_details: list[dict[str, str]] = [] # per-config change info
606
+
607
+ # Build lookups for existing manifest state
608
+ existing_paths: dict[str, str] = {
609
+ f"{c.component_id}/{c.id}": c.path for c in manifest.configurations
610
+ }
611
+ existing_keys: set[str] = set(existing_paths.keys())
612
+ existing_config_hashes: dict[str, str] = {
613
+ f"{c.component_id}/{c.id}": c.metadata.get("pull_config_hash", "")
614
+ for c in manifest.configurations
615
+ }
616
+ # Build lookup for file hashes at pull time (to detect local edits)
617
+ existing_file_hashes: dict[str, str] = {
618
+ f"{c.component_id}/{c.id}": c.metadata.get("pull_hash", "")
619
+ for c in manifest.configurations
620
+ }
621
+ # Track branch_id per config to detect branch switches
622
+ existing_branch_ids: dict[str, int] = {
623
+ f"{c.component_id}/{c.id}": c.branch_id for c in manifest.configurations
624
+ }
625
+ # Row-level state lookup, keyed by "{component_id}/{config_id}/{row_id}".
626
+ # Values: dict with path, pull_hash, pull_config_hash (all optional).
627
+ existing_rows: dict[str, dict[str, str]] = {}
628
+ for c in manifest.configurations:
629
+ for r in c.rows:
630
+ row_key = f"{c.component_id}/{c.id}/{r.id}"
631
+ existing_rows[row_key] = {
632
+ "path": r.path,
633
+ "pull_hash": r.metadata.get("pull_hash", ""),
634
+ "pull_config_hash": r.metadata.get("pull_config_hash", ""),
635
+ }
636
+
637
+ # Force-pull conflict guard (force-pull baseline corruption fix).
638
+ # ``--force`` bypasses the "preserve locally-modified files" guard
639
+ # below, so a config edited locally AND changed on the remote since the
640
+ # last pull (a true 3-way conflict) would be silently overwritten -- and
641
+ # when the remote is unchanged, its baseline would instead be re-stamped
642
+ # from the edited file, stranding the un-pushed edits. Detect such
643
+ # conflicts up front and abort BEFORE writing anything (the read-only
644
+ # API fetch above has already happened; nothing is on disk yet), so the
645
+ # user can resolve them. Non-force pull preserves locally-modified
646
+ # files and surfaces conflicts via ``sync diff``, so it needs no abort.
647
+ if force:
648
+ conflicts = self._detect_force_pull_conflicts(
649
+ components,
650
+ branch_dir,
651
+ existing_keys=existing_keys,
652
+ existing_paths=existing_paths,
653
+ existing_file_hashes=existing_file_hashes,
654
+ existing_config_hashes=existing_config_hashes,
655
+ existing_rows=existing_rows,
656
+ )
657
+ if conflicts:
658
+ raise SyncConflictError(conflicts)
659
+
660
+ for component in components:
661
+ component_id = component.get("id", "")
662
+ if component_id in ALWAYS_IGNORED_COMPONENTS:
663
+ continue
664
+ component_type = classify_component_type(component.get("type", "other"))
665
+ configs = component.get("configurations", [])
666
+
667
+ for cfg in configs:
668
+ config_id = str(cfg.get("id", ""))
669
+ config_name = cfg.get("name", "untitled")
670
+
671
+ # Reuse existing path if config is already tracked (stable paths)
672
+ lookup_key = f"{component_id}/{config_id}"
673
+ is_new = lookup_key not in existing_keys
674
+ if lookup_key in existing_paths:
675
+ rel_path = existing_paths[lookup_key]
676
+
677
+ # Auto-rename: detect when remote name changed
678
+ expected_path = config_path(
679
+ manifest.naming.config,
680
+ component_type,
681
+ component_id,
682
+ config_name,
683
+ )
684
+ if rel_path != expected_path and not dry_run:
685
+ rename_target = expected_path
686
+ # Collision: if target already used, add suffix
687
+ if rename_target in used_paths:
688
+ suffix = config_id[:8] if len(config_id) > 8 else config_id
689
+ rename_target = f"{rename_target}-{suffix}"
690
+
691
+ old_dir = branch_dir / rel_path
692
+ new_dir = branch_dir / rename_target
693
+ if old_dir.exists() and not new_dir.exists():
694
+ self._rename_directory(old_dir, new_dir)
695
+ pull_details.append(
696
+ {
697
+ "action": "renamed",
698
+ "component_id": component_id,
699
+ "config_name": config_name,
700
+ "path": rename_target,
701
+ "old_path": rel_path,
702
+ }
703
+ )
704
+ rel_path = rename_target
705
+ logger.info(
706
+ "Renamed config dir: %s -> %s",
707
+ rel_path,
708
+ rename_target,
709
+ )
710
+ else:
711
+ # Generate new filesystem path with collision detection
712
+ rel_path = config_path(
713
+ manifest.naming.config,
714
+ component_type,
715
+ component_id,
716
+ config_name,
717
+ )
718
+ if rel_path in used_paths:
719
+ # Append short config ID suffix to resolve collision
720
+ suffix = config_id[:8] if len(config_id) > 8 else config_id
721
+ rel_path = f"{rel_path}-{suffix}"
722
+ used_paths.add(rel_path)
723
+ config_dir = branch_dir / rel_path
724
+ # Defense-in-depth: refuse to write outside the branch dir
725
+ # (issue #269 sec-01). naming.sanitize_path_segment() should
726
+ # already neutralize traversal; this check guards against any
727
+ # future regression in the sanitizer or template parsing.
728
+ _ensure_within_branch(branch_dir, config_dir, component_id, config_id)
729
+
730
+ # Convert API format to local _config.yml
731
+ local_data = api_config_to_local(component_id, cfg, config_id)
732
+
733
+ # Hash of API-converted data. Stored as pull_config_hash so
734
+ # diff can compare it directly with fresh remote data without
735
+ # a lossy file roundtrip.
736
+ api_cfg_hash = config_hash(local_data)
737
+ pull_cfg_hash = api_cfg_hash
738
+
739
+ # Detect local modifications: if the file hash differs from the
740
+ # pull_hash stored in manifest, the user edited the file -- so
741
+ # preserve it instead of overwriting. This now runs even under
742
+ # ``--force``: a force-pull that reaches here for a modified file
743
+ # has already passed the conflict guard above (so the remote is
744
+ # unchanged), and re-stamping the baseline from the edited file
745
+ # would silently strand the un-pushed edits. Preserving keeps
746
+ # the pending delta visible to ``sync push``.
747
+ locally_modified = False
748
+ if not is_new:
749
+ old_file_hash = existing_file_hashes.get(lookup_key, "")
750
+ if old_file_hash:
751
+ config_file = config_dir / CONFIG_FILENAME
752
+ if config_file.exists():
753
+ current_file_hash = self._file_hash(config_file)
754
+ locally_modified = current_file_hash != old_file_hash
755
+
756
+ remote_unchanged = False # set in else branch; default for locally_modified path
757
+ if locally_modified and not dry_run:
758
+ # Preserve the existing local file -- don't overwrite.
759
+ # Still update manifest entry (remote hash changes, but
760
+ # local file stays as-is).
761
+ file_hash = self._file_hash(config_dir / CONFIG_FILENAME)
762
+ pull_details.append(
763
+ {
764
+ "action": "skipped",
765
+ "component_id": component_id,
766
+ "config_name": config_name,
767
+ "path": rel_path,
768
+ "reason": "locally modified",
769
+ }
770
+ )
771
+ elif locally_modified and dry_run:
772
+ file_hash = ""
773
+ pull_details.append(
774
+ {
775
+ "action": "skipped",
776
+ "component_id": component_id,
777
+ "config_name": config_name,
778
+ "path": rel_path,
779
+ "reason": "locally modified",
780
+ }
781
+ )
782
+ else:
783
+ # Check if remote actually changed since last pull.
784
+ # If pull_config_hash matches AND branch hasn't changed,
785
+ # skip write (idempotent). A branch switch means files
786
+ # live in a different directory, so we must re-write.
787
+ old_cfg_hash = existing_config_hashes.get(lookup_key, "")
788
+ branch_switched = existing_branch_ids.get(lookup_key, branch_id or 0) != (
789
+ branch_id or 0
790
+ )
791
+ remote_unchanged = (
792
+ not is_new
793
+ and not branch_switched
794
+ and old_cfg_hash
795
+ and old_cfg_hash == api_cfg_hash
796
+ )
797
+
798
+ if remote_unchanged:
799
+ # Nothing changed -- reuse existing file hash
800
+ config_file = config_dir / CONFIG_FILENAME
801
+ file_hash = self._file_hash(config_file) if config_file.exists() else ""
802
+ else:
803
+ # Extract code files (SQL, Python) if applicable.
804
+ # This modifies local_data in place (removes
805
+ # blocks/code) and writes separate code files.
806
+ if not dry_run:
807
+ extract_code_files(component_id, local_data, config_dir)
808
+ file_hash = self._write_config_file(config_dir, local_data)
809
+ else:
810
+ file_hash = ""
811
+
812
+ configs_pulled += 1
813
+ files_written += 1
814
+
815
+ if is_new:
816
+ pull_details.append(
817
+ {
818
+ "action": "new",
819
+ "component_id": component_id,
820
+ "config_name": config_name,
821
+ "path": rel_path,
822
+ }
823
+ )
824
+ else:
825
+ pull_details.append(
826
+ {
827
+ "action": "updated",
828
+ "component_id": component_id,
829
+ "config_name": config_name,
830
+ "path": rel_path,
831
+ }
832
+ )
833
+
834
+ # Row-level pull: each row gets its own 3-way diff state
835
+ # (pull_hash, pull_config_hash) so ``sync push`` can detect
836
+ # per-row local modifications and push only changed rows.
837
+ row_manifests: list[ManifestConfigRow] = []
838
+ used_row_paths: set[str] = set()
839
+ for row in cfg.get("rows", []):
840
+ row_id = str(row.get("id", ""))
841
+ row_name = row.get("name", "untitled")
842
+
843
+ # Reuse existing row path (stable) if already tracked.
844
+ row_lookup_key = f"{component_id}/{config_id}/{row_id}"
845
+ existing_row = existing_rows.get(row_lookup_key)
846
+ if existing_row:
847
+ row_rel_path = existing_row["path"]
848
+ else:
849
+ row_rel_path = config_row_path(
850
+ manifest.naming.config_row,
851
+ row_name,
852
+ )
853
+ if row_rel_path in used_row_paths:
854
+ suffix = row_id[:8] if len(row_id) > 8 else row_id
855
+ row_rel_path = f"{row_rel_path}-{suffix}"
856
+ used_row_paths.add(row_rel_path)
857
+ row_dir = config_dir / row_rel_path
858
+
859
+ row_local = api_row_to_local(row, component_id)
860
+ row_api_cfg_hash = config_hash(row_local)
861
+
862
+ row_file = row_dir / CONFIG_FILENAME
863
+ old_row_file_hash = existing_row["pull_hash"] if existing_row else ""
864
+ old_row_cfg_hash = existing_row["pull_config_hash"] if existing_row else ""
865
+ # Runs even under ``--force``: a force-pull reaching a
866
+ # modified row has passed the conflict guard (remote
867
+ # unchanged), so preserve the row rather than re-stamp its
868
+ # baseline from the edited file and strand the edits.
869
+ row_locally_modified = False
870
+ if existing_row and old_row_file_hash and row_file.exists():
871
+ row_locally_modified = self._file_hash(row_file) != old_row_file_hash
872
+
873
+ if row_locally_modified:
874
+ # Preserve local edits; keep old hashes as the 3-way base.
875
+ row_file_hash = old_row_file_hash
876
+ row_pull_cfg_hash = old_row_cfg_hash
877
+ pull_details.append(
878
+ {
879
+ "action": "skipped",
880
+ "component_id": component_id,
881
+ "config_name": f"{config_name}/{row_name}",
882
+ "path": f"{rel_path}/{row_rel_path}",
883
+ "reason": "row locally modified",
884
+ }
885
+ )
886
+ elif (
887
+ existing_row
888
+ and old_row_cfg_hash
889
+ and old_row_cfg_hash == row_api_cfg_hash
890
+ and row_file.exists()
891
+ ):
892
+ # Idempotent: remote unchanged since last pull, file untouched.
893
+ # Guard: row_file.exists() ensures we don't skip writing when
894
+ # the directory is new (e.g. first pull of a dev branch that
895
+ # clones main -- same hash but no file on disk yet).
896
+ row_file_hash = (
897
+ self._file_hash(row_file) if row_file.exists() else old_row_file_hash
898
+ )
899
+ row_pull_cfg_hash = row_api_cfg_hash
900
+ else:
901
+ # New or remote-changed row: write the file.
902
+ if not dry_run:
903
+ self._write_config_file(row_dir, row_local)
904
+ row_file_hash = self._file_hash(row_file) if row_file.exists() else ""
905
+ else:
906
+ row_file_hash = ""
907
+ row_pull_cfg_hash = row_api_cfg_hash
908
+ files_written += 1
909
+ rows_pulled += 1
910
+
911
+ row_manifests.append(
912
+ ManifestConfigRow(
913
+ id=row_id,
914
+ path=row_rel_path,
915
+ metadata={
916
+ "pull_hash": row_file_hash,
917
+ "pull_config_hash": row_pull_cfg_hash,
918
+ },
919
+ )
920
+ )
921
+
922
+ # Record in manifest (store file hash for change detection).
923
+ # For skipped configs: keep existing pull_hash (file untouched)
924
+ # but do NOT update pull_config_hash -- keep the old base so
925
+ # 3-way diff still correctly detects the local modification.
926
+ if locally_modified:
927
+ old_pull_hash = existing_file_hashes.get(lookup_key, file_hash)
928
+ old_cfg_hash = existing_config_hashes.get(lookup_key, pull_cfg_hash)
929
+ cfg_metadata = {
930
+ "pull_hash": old_pull_hash,
931
+ "pull_config_hash": old_cfg_hash,
932
+ }
933
+ else:
934
+ # Compute hashes for all extracted files
935
+ extra_hashes: dict[str, str] = {}
936
+ if not dry_run:
937
+ for fname in [
938
+ "_description.md",
939
+ "transform.sql",
940
+ "transform.py",
941
+ "code.py",
942
+ "pyproject.toml",
943
+ ]:
944
+ fpath = config_dir / fname
945
+ if fpath.exists():
946
+ extra_hashes[fname] = self._file_hash(fpath)
947
+ cfg_metadata = {
948
+ "pull_hash": file_hash,
949
+ "pull_config_hash": pull_cfg_hash,
950
+ "pull_extra_hashes": extra_hashes,
951
+ }
952
+ new_configurations.append(
953
+ ManifestConfiguration(
954
+ branchId=branch_id or 0,
955
+ componentId=component_id,
956
+ id=config_id,
957
+ path=rel_path,
958
+ metadata=cfg_metadata,
959
+ rows=row_manifests,
960
+ )
961
+ )
962
+
963
+ # Detect configs removed from remote (in old manifest but not in new)
964
+ new_keys = {f"{c.component_id}/{c.id}" for c in new_configurations}
965
+ for old_cfg in manifest.configurations:
966
+ old_key = f"{old_cfg.component_id}/{old_cfg.id}"
967
+ if old_key not in new_keys:
968
+ pull_details.append(
969
+ {
970
+ "action": "removed",
971
+ "component_id": old_cfg.component_id,
972
+ "config_name": "",
973
+ "path": old_cfg.path,
974
+ }
975
+ )
976
+
977
+ # Delete orphaned directories for removed configurations
978
+ if not dry_run:
979
+ for detail in pull_details:
980
+ if detail["action"] == "removed" and detail.get("path"):
981
+ orphan_dir = branch_dir / detail["path"]
982
+ if orphan_dir.exists() and orphan_dir.is_dir():
983
+ shutil.rmtree(orphan_dir)
984
+ logger.info("Removed orphaned directory: %s", orphan_dir)
985
+ # Clean up empty parent dirs up to (but not including) branch_dir
986
+ parent = orphan_dir.parent
987
+ while parent != branch_dir and parent.exists():
988
+ if not any(parent.iterdir()):
989
+ parent.rmdir()
990
+ logger.info("Removed empty parent directory: %s", parent)
991
+ parent = parent.parent
992
+ else:
993
+ break
994
+
995
+ # -- Storage metadata (read-only, not tracked in manifest) --
996
+ storage_stats: dict[str, int] = {"buckets": 0, "tables": 0, "samples": 0}
997
+ if not dry_run and buckets_data:
998
+ storage_stats = write_storage_metadata(
999
+ project_root, buckets_data, tables_data, samples_data
1000
+ )
1001
+
1002
+ # -- Per-config jobs (JSONL files next to _config.yml) --
1003
+ jobs_written = 0
1004
+ if not dry_run and jobs_grouped:
1005
+ jobs_written = write_per_config_jobs(branch_dir, new_configurations, jobs_grouped)
1006
+
1007
+ if not dry_run:
1008
+ # Update manifest with pulled configurations
1009
+ manifest.configurations = new_configurations
1010
+ save_manifest(project_root, manifest)
1011
+
1012
+ return {
1013
+ "status": "dry_run" if dry_run else "pulled",
1014
+ "project_alias": alias,
1015
+ "branch_id": branch_id,
1016
+ "branch_dir": branch_dir_name,
1017
+ "configs_pulled": configs_pulled,
1018
+ "rows_pulled": rows_pulled,
1019
+ "files_written": files_written,
1020
+ "jobs_written": jobs_written,
1021
+ "storage": storage_stats,
1022
+ "details": pull_details,
1023
+ }
1024
+
1025
+ # ------------------------------------------------------------------
1026
+ # status
1027
+ # ------------------------------------------------------------------
1028
+
1029
+ def status(self, project_root: Path) -> dict[str, Any]:
1030
+ """Compare local state against the manifest to detect changes.
1031
+
1032
+ Walks the local filesystem and compares against manifest entries
1033
+ to classify configurations as modified, added, deleted, or unchanged.
1034
+
1035
+ Args:
1036
+ project_root: Root directory of the sync working tree.
1037
+
1038
+ Returns:
1039
+ Dict with lists of modified/added/deleted configs, count of
1040
+ unchanged, and ``plaintext_secret_warnings`` -- in-sync configs/rows
1041
+ whose ``#``-secrets are still plaintext on the remote (issue #378).
1042
+ """
1043
+ manifest = load_manifest(project_root)
1044
+
1045
+ modified: list[dict[str, str]] = []
1046
+ deleted: list[dict[str, str]] = []
1047
+ unchanged = 0
1048
+
1049
+ # Check each manifest entry against local files
1050
+ for cfg in manifest.configurations:
1051
+ branch_path = self._find_branch_path(manifest, cfg.branch_id)
1052
+ config_dir = project_root / branch_path / cfg.path
1053
+ config_file = config_dir / CONFIG_FILENAME
1054
+
1055
+ if not config_file.exists():
1056
+ deleted.append(
1057
+ {
1058
+ "component_id": cfg.component_id,
1059
+ "config_id": cfg.id,
1060
+ "path": str(cfg.path),
1061
+ }
1062
+ )
1063
+ continue
1064
+
1065
+ # Compare file hash against the hash stored at pull time.
1066
+ # Check both _config.yml AND extracted code files (transform.sql etc.)
1067
+ # to match the same logic used by diff().
1068
+ current_hash = self._file_hash(config_file)
1069
+ pull_hash = cfg.metadata.get("pull_hash", "")
1070
+ config_unchanged = bool(pull_hash and current_hash == pull_hash)
1071
+
1072
+ extras_unchanged = True
1073
+ stored_extra = cfg.metadata.get("pull_extra_hashes", {})
1074
+ for fname, stored_h in stored_extra.items():
1075
+ fpath = config_dir / fname
1076
+ if fpath.exists():
1077
+ if self._file_hash(fpath) != stored_h:
1078
+ extras_unchanged = False
1079
+ break
1080
+ else:
1081
+ extras_unchanged = False
1082
+ break
1083
+
1084
+ if config_unchanged and extras_unchanged:
1085
+ unchanged += 1
1086
+ else:
1087
+ modified.append(
1088
+ {
1089
+ "component_id": cfg.component_id,
1090
+ "config_id": cfg.id,
1091
+ "path": str(cfg.path),
1092
+ }
1093
+ )
1094
+
1095
+ # Scan for added configs (local files without manifest entry)
1096
+ added = self._find_untracked_configs(project_root, manifest)
1097
+
1098
+ # Flag in-sync configs whose secrets are still plaintext (issue #378).
1099
+ # Best-effort: a file race during the scan must not crash `sync status`
1100
+ # (mirrors the doctor sync_secrets check's defensive guard).
1101
+ try:
1102
+ plaintext_secret_warnings = scan_synced_plaintext_secrets(project_root, manifest)
1103
+ except Exception:
1104
+ logger.warning("Plaintext-secret scan failed; skipping", exc_info=True)
1105
+ plaintext_secret_warnings = []
1106
+
1107
+ return {
1108
+ "modified": modified,
1109
+ "added": added,
1110
+ "deleted": deleted,
1111
+ "unchanged": unchanged,
1112
+ "total_tracked": len(manifest.configurations),
1113
+ "plaintext_secret_warnings": plaintext_secret_warnings,
1114
+ }
1115
+
1116
+ # ------------------------------------------------------------------
1117
+ # diff
1118
+ # ------------------------------------------------------------------
1119
+
1120
+ def diff(
1121
+ self,
1122
+ alias: str,
1123
+ project_root: Path,
1124
+ branch_override: int | None = None,
1125
+ ) -> dict[str, Any]:
1126
+ """Compare local configs against the remote API state.
1127
+
1128
+ Fetches current state from API, reads local _config.yml files,
1129
+ and runs the diff engine to produce a detailed changeset.
1130
+
1131
+ Args:
1132
+ alias: Project alias.
1133
+ project_root: Sync working-tree root.
1134
+ branch_override: If set, diff against this branch ID rather
1135
+ than the resolved active/manifest branch.
1136
+
1137
+ Returns:
1138
+ Dict with 'changes' list and summary counts.
1139
+ """
1140
+ projects = self.resolve_projects([alias])
1141
+ project = projects[alias]
1142
+ manifest = load_manifest(project_root)
1143
+
1144
+ branch_id = self._resolve_branch_id(
1145
+ project, manifest, project_root, branch_override=branch_override
1146
+ )
1147
+
1148
+ # Fetch remote state
1149
+ client = self._client_factory(project.stack_url, project.token)
1150
+ with client:
1151
+ components = client.list_components_with_configs(branch_id=branch_id)
1152
+ self._ensure_branch_registered(manifest, branch_id, client)
1153
+
1154
+ # Build remote lookups:
1155
+ # remote_configs: "{component_id}/{config_id}" -> parent config data
1156
+ # remote_rows: "{component_id}/{parent_config_id}/rows/{row_id}" -> row data
1157
+ remote_configs: dict[str, dict[str, Any]] = {}
1158
+ remote_rows: dict[str, dict[str, Any]] = {}
1159
+ for component in components:
1160
+ component_id = component.get("id", "")
1161
+ if component_id in ALWAYS_IGNORED_COMPONENTS:
1162
+ continue
1163
+ for cfg in component.get("configurations", []):
1164
+ config_id = str(cfg.get("id", ""))
1165
+ key = f"{component_id}/{config_id}"
1166
+ remote_configs[key] = api_config_to_local(component_id, cfg, config_id)
1167
+ for row in cfg.get("rows", []):
1168
+ row_id = str(row.get("id", ""))
1169
+ row_key = f"{component_id}/{config_id}/rows/{row_id}"
1170
+ remote_rows[row_key] = api_row_to_local(row, component_id)
1171
+
1172
+ # Build local configs list from manifest.
1173
+ # For files unchanged since pull, use the stored pull_config_hash
1174
+ # directly (avoids lossy code extraction roundtrip).
1175
+ # For locally modified files, merge code back for real comparison.
1176
+ local_configs: list[dict[str, Any]] = []
1177
+ file_unchanged: dict[str, bool] = {}
1178
+ local_override_hashes: dict[str, str] = {}
1179
+ for cfg in manifest.configurations:
1180
+ branch_path = self._find_branch_path(manifest, cfg.branch_id)
1181
+ config_dir = project_root / branch_path / cfg.path
1182
+ local_data = self._read_config_file(config_dir)
1183
+ if local_data is None:
1184
+ continue
1185
+
1186
+ key = f"{cfg.component_id}/{cfg.id}"
1187
+
1188
+ # Check if ANY file in this config dir changed since pull.
1189
+ # Manifest stores pull_extra_hashes for extracted files.
1190
+ pull_hash = cfg.metadata.get("pull_hash", "")
1191
+ config_file = config_dir / CONFIG_FILENAME
1192
+ current_file_hash = self._file_hash(config_file) if config_file.exists() else ""
1193
+ config_unchanged = bool(pull_hash and current_file_hash == pull_hash)
1194
+
1195
+ extras_unchanged = True
1196
+ stored_extra = cfg.metadata.get("pull_extra_hashes", {})
1197
+ for fname, stored_h in stored_extra.items():
1198
+ fpath = config_dir / fname
1199
+ if fpath.exists():
1200
+ if self._file_hash(fpath) != stored_h:
1201
+ extras_unchanged = False
1202
+ break
1203
+ else:
1204
+ extras_unchanged = False
1205
+ break
1206
+
1207
+ is_unchanged = config_unchanged and extras_unchanged
1208
+ file_unchanged[key] = is_unchanged
1209
+
1210
+ if is_unchanged:
1211
+ # All files match pull state -- use stored API hash
1212
+ stored_cfg_hash = cfg.metadata.get("pull_config_hash", "")
1213
+ if stored_cfg_hash:
1214
+ local_override_hashes[key] = stored_cfg_hash
1215
+ # Always merge for local_data (needed for deep_diff details)
1216
+ merge_code_files(cfg.component_id, local_data, config_dir)
1217
+
1218
+ local_configs.append(
1219
+ {
1220
+ "component_id": cfg.component_id,
1221
+ "config_id": cfg.id,
1222
+ "config_name": local_data.get("name", ""),
1223
+ "path": cfg.path,
1224
+ "data": local_data,
1225
+ }
1226
+ )
1227
+
1228
+ # Also add untracked local configs (new files)
1229
+ for added_cfg in self._find_untracked_configs(project_root, manifest, branch_id):
1230
+ branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id)
1231
+ config_dir = project_root / branch_path / added_cfg["path"]
1232
+ local_data = self._read_config_file(config_dir)
1233
+ if local_data is None:
1234
+ continue
1235
+ local_configs.append(
1236
+ {
1237
+ "component_id": added_cfg.get("component_id", "unknown"),
1238
+ "config_id": "", # new config, no ID yet
1239
+ "config_name": local_data.get("name", ""),
1240
+ "path": added_cfg["path"],
1241
+ "data": local_data,
1242
+ }
1243
+ )
1244
+
1245
+ # Build set of manifest-tracked keys so that compute_changeset only
1246
+ # flags configs that were previously pulled (not brand-new remote ones).
1247
+ tracked_keys = {f"{cfg.component_id}/{cfg.id}" for cfg in manifest.configurations}
1248
+
1249
+ # Build base hashes for 3-way diff.
1250
+ # Preferred: pull_config_hash (normalized hash stored at pull time).
1251
+ # Fallback: if file is unchanged since pull, use current config_hash
1252
+ # as the base (since local == base when file hasn't been modified).
1253
+ base_hashes: dict[str, str] = {}
1254
+ for cfg in manifest.configurations:
1255
+ key = f"{cfg.component_id}/{cfg.id}"
1256
+ pch = cfg.metadata.get("pull_config_hash")
1257
+ if pch:
1258
+ base_hashes[key] = pch
1259
+ elif file_unchanged.get(key):
1260
+ # File not modified locally → local data IS the base.
1261
+ # Find the matching local_configs entry and hash it.
1262
+ for lc in local_configs:
1263
+ if lc.get("config_id") == cfg.id and lc["component_id"] == cfg.component_id:
1264
+ base_hashes[key] = config_hash(lc["data"])
1265
+ break
1266
+
1267
+ changeset = compute_changeset(
1268
+ local_configs,
1269
+ remote_configs,
1270
+ tracked_keys,
1271
+ base_hashes or None,
1272
+ local_override_hashes or None,
1273
+ )
1274
+
1275
+ # Row-level diff: walk manifest rows, load local YAML, feed into
1276
+ # compute_row_changeset alongside remote_rows built above.
1277
+ local_rows: list[dict[str, Any]] = []
1278
+ tracked_row_keys: set[str] = set()
1279
+ row_base_hashes: dict[str, str] = {}
1280
+ for cfg in manifest.configurations:
1281
+ branch_path = self._find_branch_path(manifest, cfg.branch_id)
1282
+ parent_dir = project_root / branch_path / cfg.path
1283
+ for row in cfg.rows:
1284
+ row_key = f"{cfg.component_id}/{cfg.id}/rows/{row.id}"
1285
+ tracked_row_keys.add(row_key)
1286
+ row_dir = parent_dir / row.path
1287
+ row_local = self._read_config_file(row_dir)
1288
+ if row_local is None:
1289
+ # File missing -> "deleted" detected via tracked_row_keys.
1290
+ continue
1291
+ local_rows.append(
1292
+ {
1293
+ "component_id": cfg.component_id,
1294
+ "parent_config_id": cfg.id,
1295
+ "row_id": row.id,
1296
+ "row_name": row_local.get("name", ""),
1297
+ "path": row.path,
1298
+ "data": row_local,
1299
+ }
1300
+ )
1301
+ pch = row.metadata.get("pull_config_hash") if row.metadata else ""
1302
+ if pch:
1303
+ row_base_hashes[row_key] = pch
1304
+
1305
+ # Also add untracked local rows (new row dirs dropped under a tracked
1306
+ # config). They have no row_id yet so compute_row_changeset flags
1307
+ # them as "added" and push dispatches them via create_config_row.
1308
+ for untracked in self._find_untracked_rows(project_root, manifest):
1309
+ local_rows.append(
1310
+ {
1311
+ "component_id": untracked["component_id"],
1312
+ "parent_config_id": untracked["parent_config_id"],
1313
+ "row_id": "",
1314
+ "row_name": untracked["row_name"],
1315
+ "path": untracked["path"],
1316
+ "data": untracked["data"],
1317
+ }
1318
+ )
1319
+
1320
+ row_changeset = compute_row_changeset(
1321
+ local_rows,
1322
+ remote_rows,
1323
+ tracked_row_keys,
1324
+ row_base_hashes or None,
1325
+ )
1326
+ changeset.extend(row_changeset)
1327
+
1328
+ added = [c for c in changeset if c.change_type == "added"]
1329
+ modified = [c for c in changeset if c.change_type == "modified"]
1330
+ remote_modified = [c for c in changeset if c.change_type == "remote_modified"]
1331
+ conflicts = [c for c in changeset if c.change_type == "conflict"]
1332
+ deleted = [c for c in changeset if c.change_type == "deleted"]
1333
+
1334
+ # Detect remote-only configs (new on server, not yet pulled).
1335
+ local_keys = {
1336
+ f"{e['component_id']}/{e['config_id']}" for e in local_configs if e.get("config_id")
1337
+ } | tracked_keys
1338
+ remote_only: list[dict[str, str]] = []
1339
+ for remote_key, remote_data in remote_configs.items():
1340
+ if remote_key not in local_keys:
1341
+ parts = remote_key.split("/", 1)
1342
+ remote_only.append(
1343
+ {
1344
+ "component_id": parts[0] if parts else "",
1345
+ "config_id": parts[1] if len(parts) > 1 else "",
1346
+ "config_name": remote_data.get("name", ""),
1347
+ }
1348
+ )
1349
+
1350
+ return {
1351
+ "changes": [c.to_dict() for c in changeset],
1352
+ "remote_only": remote_only,
1353
+ "summary": {
1354
+ "added": len(added),
1355
+ "modified": len(modified),
1356
+ "remote_modified": len(remote_modified),
1357
+ "conflict": len(conflicts),
1358
+ "deleted": len(deleted),
1359
+ "unchanged": len(local_configs)
1360
+ - len(added)
1361
+ - len(modified)
1362
+ - len(remote_modified)
1363
+ - len(conflicts),
1364
+ "remote_only": len(remote_only),
1365
+ },
1366
+ }
1367
+
1368
+ # ------------------------------------------------------------------
1369
+ # push
1370
+ # ------------------------------------------------------------------
1371
+
1372
+ def push(
1373
+ self,
1374
+ alias: str,
1375
+ project_root: Path,
1376
+ dry_run: bool = False,
1377
+ force: bool = False,
1378
+ allow_plaintext_fallback: bool = False,
1379
+ branch_override: int | None = None,
1380
+ no_name_drift_warnings: bool = False,
1381
+ ) -> dict[str, Any]:
1382
+ """Push local changes to Keboola.
1383
+
1384
+ Computes diff, then creates/updates/deletes configs via API.
1385
+ New configs get IDs assigned by the API; the manifest is updated.
1386
+
1387
+ Args:
1388
+ alias: Project alias from config store.
1389
+ project_root: Root directory of the sync working tree.
1390
+ dry_run: If True, compute changes but don't execute them.
1391
+ force: If True, allow deletions without extra confirmation.
1392
+ allow_plaintext_fallback: If True, allow push when secret
1393
+ encryption fails (DANGEROUS).
1394
+ branch_override: If set, target this dev-branch ID for the push.
1395
+ Wins over ``active_branch_id`` / ``manifest.branches[0]`` /
1396
+ git-branching mapping. Used by the CLI ``--branch`` flag. When
1397
+ no ``<branch_name>/`` subtree exists on disk for the target,
1398
+ the default tree (``main/``) is read as the source and promoted
1399
+ to the target branch (KFR-07); API writes still target the
1400
+ branch id.
1401
+ no_name_drift_warnings: If True, omit the ``name_drift_warnings``
1402
+ array from the result envelope (the underlying detection
1403
+ still runs; only the report is suppressed). Used by the CLI
1404
+ ``--no-name-drift-warnings`` flag.
1405
+
1406
+ Returns:
1407
+ Dict with push results (created, updated, deleted, errors).
1408
+ """
1409
+ diff_result = self.diff(alias, project_root, branch_override=branch_override)
1410
+ all_changes = diff_result["changes"]
1411
+
1412
+ # Only push local-side changes (added, modified, deleted).
1413
+ # Skip remote_modified (need pull) and conflict (need resolution).
1414
+ pushable_types = {"added", "modified", "deleted"}
1415
+ changes = [c for c in all_changes if c["change_type"] in pushable_types]
1416
+
1417
+ # Warn about skipped changes
1418
+ skipped = [c for c in all_changes if c["change_type"] not in pushable_types]
1419
+
1420
+ if not changes:
1421
+ result: dict[str, Any] = {
1422
+ "status": "no_changes",
1423
+ "created": 0,
1424
+ "updated": 0,
1425
+ "deleted": 0,
1426
+ "errors": [],
1427
+ }
1428
+ if skipped:
1429
+ result["skipped"] = len(skipped)
1430
+ result["skipped_reason"] = "Remote changes detected. Run 'sync pull' first."
1431
+ return result
1432
+
1433
+ if dry_run:
1434
+ return {
1435
+ "status": "dry_run",
1436
+ "changes": changes,
1437
+ "summary": diff_result["summary"],
1438
+ }
1439
+
1440
+ projects = self.resolve_projects([alias])
1441
+ project = projects[alias]
1442
+ manifest = load_manifest(project_root)
1443
+
1444
+ branch_id = self._resolve_branch_id(
1445
+ project, manifest, project_root, branch_override=branch_override
1446
+ )
1447
+
1448
+ # Detect name drift: local dir name doesn't match config name
1449
+ name_drift_warnings = self._detect_name_drift(manifest, project_root)
1450
+
1451
+ client = self._client_factory(project.stack_url, project.token)
1452
+ created = 0
1453
+ updated = 0
1454
+ deleted = 0
1455
+ errors: list[dict[str, str]] = []
1456
+ pushed_details: list[dict[str, str]] = []
1457
+ manifest_dirty = False
1458
+
1459
+ with client:
1460
+ self._ensure_branch_registered(manifest, branch_id, client)
1461
+ branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id)
1462
+
1463
+ # Process configs before rows, and rebind variable links last.
1464
+ # A freshly-created parent config must carry its API-assigned ULID
1465
+ # before its rows POST (KFR-05), and a transformation's
1466
+ # variables_id / variables_values_id can only be resolved once both
1467
+ # the variables config and its values row exist (KFR-03). Partition
1468
+ # explicitly rather than relying on incidental diff ordering.
1469
+ config_changes = [c for c in changes if not bool(c.get("is_row"))]
1470
+ row_changes = [c for c in changes if bool(c.get("is_row"))]
1471
+
1472
+ # (component_id, placeholder_id) -> API-assigned ULID, captured
1473
+ # before the manifest writeback overwrites the placeholder in place.
1474
+ created_id_map: dict[tuple[str, str], str] = {}
1475
+ created_configs: list[CreatedConfig] = []
1476
+
1477
+ # ---- Phase A: config creates / updates / deletes -------------
1478
+ for change in config_changes:
1479
+ change_type = change["change_type"]
1480
+ component_id = change["component_id"]
1481
+ config_id = change["config_id"]
1482
+ config_path_str = change.get("path", "")
1483
+
1484
+ try:
1485
+ if change_type == "added":
1486
+ result = push_create(
1487
+ self,
1488
+ client,
1489
+ component_id,
1490
+ config_path_str,
1491
+ project_root,
1492
+ manifest,
1493
+ branch_id,
1494
+ allow_plaintext_fallback=allow_plaintext_fallback,
1495
+ )
1496
+ if result:
1497
+ new_id = str(result.get("id", ""))
1498
+ config_dir = project_root / branch_path / config_path_str
1499
+ hashes = self._compute_config_hashes(config_dir, component_id)
1500
+ writeback = writeback_create_config_in_manifest(
1501
+ manifest=manifest,
1502
+ component_id=component_id,
1503
+ branch_id=branch_id,
1504
+ config_path_str=config_path_str,
1505
+ new_id=new_id,
1506
+ file_hash=hashes.file_hash,
1507
+ cfg_hash=hashes.cfg_hash,
1508
+ )
1509
+ # Record placeholder -> ULID so child rows and
1510
+ # transformation variable links can be remapped.
1511
+ if writeback.previous_id:
1512
+ created_id_map[(component_id, writeback.previous_id)] = new_id
1513
+ created_configs.append(
1514
+ CreatedConfig(
1515
+ component_id=component_id,
1516
+ config_id=new_id,
1517
+ config_dir=config_dir,
1518
+ )
1519
+ )
1520
+ metadata_error = propagate_kbc_metadata(
1521
+ client, writeback.entry, branch_id
1522
+ )
1523
+ if metadata_error is not None:
1524
+ # The config IS on the remote; only the
1525
+ # follow-up metadata POST failed. Accumulate
1526
+ # like any other per-change error so the rest
1527
+ # of the push continues, and surface the
1528
+ # original cause in the envelope.
1529
+ errors.append(
1530
+ {
1531
+ "change_type": "metadata_propagation",
1532
+ "component_id": component_id,
1533
+ "config_id": new_id,
1534
+ "message": metadata_error,
1535
+ }
1536
+ )
1537
+ manifest_dirty = True
1538
+ created += 1
1539
+ pushed_details.append(change)
1540
+
1541
+ elif change_type == "modified":
1542
+ push_update(
1543
+ self,
1544
+ client,
1545
+ component_id,
1546
+ config_id,
1547
+ config_path_str,
1548
+ project_root,
1549
+ manifest,
1550
+ branch_id,
1551
+ allow_plaintext_fallback=allow_plaintext_fallback,
1552
+ )
1553
+ # Update hashes so pull knows local == remote
1554
+ config_dir = project_root / branch_path / config_path_str
1555
+ config_file = config_dir / CONFIG_FILENAME
1556
+ if config_file.exists():
1557
+ hashes = self._compute_config_hashes(config_dir, component_id)
1558
+ for cfg in manifest.configurations:
1559
+ if cfg.component_id == component_id and cfg.id == config_id:
1560
+ cfg.metadata["pull_hash"] = hashes.file_hash
1561
+ cfg.metadata["pull_config_hash"] = hashes.cfg_hash
1562
+ cfg.metadata["pull_extra_hashes"] = hashes.extra_hashes
1563
+ break
1564
+ manifest_dirty = True
1565
+ updated += 1
1566
+ pushed_details.append(change)
1567
+
1568
+ elif change_type == "deleted":
1569
+ client.delete_config(
1570
+ component_id=component_id,
1571
+ config_id=config_id,
1572
+ branch_id=branch_id,
1573
+ )
1574
+ # Remove from manifest
1575
+ manifest.configurations = [
1576
+ c
1577
+ for c in manifest.configurations
1578
+ if not (c.component_id == component_id and c.id == config_id)
1579
+ ]
1580
+ manifest_dirty = True
1581
+ deleted += 1
1582
+ pushed_details.append(change)
1583
+
1584
+ except Exception as exc:
1585
+ # Fail-closed on encryption failures: a partial push that
1586
+ # omits the failed change would leave a plaintext secret
1587
+ # elsewhere or a caller believing the push "mostly succeeded".
1588
+ # Surface to the CLI (exit non-zero) rather than burying in
1589
+ # result["errors"].
1590
+ if (
1591
+ isinstance(exc, KeboolaApiError)
1592
+ and exc.error_code == ErrorCode.ENCRYPTION_FAILED
1593
+ ):
1594
+ raise
1595
+ self._record_push_error(errors, change_type, component_id, config_id, exc)
1596
+
1597
+ # ---- Phase B: row creates / updates / deletes ----------------
1598
+ # row placeholder id -> ULID; ULID parent -> rows created under it.
1599
+ created_row_id_map: dict[str, str] = {}
1600
+ created_rows_by_parent: dict[str, list[str]] = {}
1601
+ for change in row_changes:
1602
+ change_type = change["change_type"]
1603
+ component_id = change["component_id"]
1604
+ config_id = change["config_id"]
1605
+ config_path_str = change.get("path", "")
1606
+ parent_config_id = change.get("parent_config_id", "")
1607
+ # Remap the diff-time parent placeholder to the ULID assigned in
1608
+ # Phase A so the manifest lookup and create_config_row both hit
1609
+ # the real config (KFR-05). UPDATE/DELETE parents already carry
1610
+ # a ULID and pass through unchanged.
1611
+ effective_parent_id = created_id_map.get(
1612
+ (component_id, parent_config_id), parent_config_id
1613
+ )
1614
+
1615
+ try:
1616
+ new_row_id = push_row_change(
1617
+ self,
1618
+ client,
1619
+ change_type=change_type,
1620
+ component_id=component_id,
1621
+ parent_config_id=effective_parent_id,
1622
+ row_id=config_id,
1623
+ row_path_str=config_path_str,
1624
+ project_root=project_root,
1625
+ manifest=manifest,
1626
+ branch_id=branch_id,
1627
+ allow_plaintext_fallback=allow_plaintext_fallback,
1628
+ )
1629
+ manifest_dirty = True
1630
+ if change_type == "added":
1631
+ created += 1
1632
+ if new_row_id:
1633
+ if config_id:
1634
+ created_row_id_map[config_id] = new_row_id
1635
+ created_rows_by_parent.setdefault(effective_parent_id, []).append(
1636
+ new_row_id
1637
+ )
1638
+ elif change_type == "modified":
1639
+ updated += 1
1640
+ elif change_type == "deleted":
1641
+ deleted += 1
1642
+ pushed_details.append(change)
1643
+
1644
+ except Exception as exc:
1645
+ if (
1646
+ isinstance(exc, KeboolaApiError)
1647
+ and exc.error_code == ErrorCode.ENCRYPTION_FAILED
1648
+ ):
1649
+ raise
1650
+ self._record_push_error(errors, change_type, component_id, config_id, exc)
1651
+
1652
+ # ---- Phase C: variable-link backfill (KFR-03) ----------------
1653
+ binding = resolve_variable_bindings(
1654
+ self,
1655
+ client,
1656
+ created_configs=created_configs,
1657
+ created_id_map=created_id_map,
1658
+ created_row_id_map=created_row_id_map,
1659
+ created_rows_by_parent=created_rows_by_parent,
1660
+ manifest=manifest,
1661
+ branch_id=branch_id,
1662
+ )
1663
+ errors.extend(binding.errors)
1664
+ if binding.configs_rewritten:
1665
+ manifest_dirty = True
1666
+
1667
+ # ---- Phase D: flow task configId backfill (#426) -------------
1668
+ # After variable links, remap keboola.flow task configIds that point
1669
+ # at configs created this push (golden/placeholder -> ULID). Reuses
1670
+ # created_id_map; a no-op when no flow was created.
1671
+ flow_binding = resolve_flow_task_bindings(
1672
+ self,
1673
+ client,
1674
+ created_configs=created_configs,
1675
+ created_id_map=created_id_map,
1676
+ manifest=manifest,
1677
+ branch_id=branch_id,
1678
+ )
1679
+ errors.extend(flow_binding.errors)
1680
+ if flow_binding.configs_rewritten:
1681
+ manifest_dirty = True
1682
+
1683
+ # Save manifest with updated hashes / new IDs / removed entries
1684
+ if manifest_dirty:
1685
+ save_manifest(project_root, manifest)
1686
+
1687
+ result_data: dict[str, Any] = {
1688
+ "status": "pushed",
1689
+ "created": created,
1690
+ "updated": updated,
1691
+ "deleted": deleted,
1692
+ "errors": errors,
1693
+ "pushed_details": pushed_details,
1694
+ }
1695
+ if flow_binding.tasks_remapped:
1696
+ result_data["flow_task_remaps"] = flow_binding.tasks_remapped
1697
+ if name_drift_warnings and not no_name_drift_warnings:
1698
+ result_data["name_drift_warnings"] = name_drift_warnings
1699
+ return result_data
1700
+
1701
+ def clone_project(
1702
+ self,
1703
+ source: str | Path,
1704
+ target_alias: str,
1705
+ target_dir: str | Path,
1706
+ *,
1707
+ overrides: dict[str, Any] | None = None,
1708
+ dry_run: bool = False,
1709
+ branch_override: int | None = None,
1710
+ ) -> dict[str, Any]:
1711
+ """Clone a reference synced project into a fresh target project (#426).
1712
+
1713
+ Thin delegator to :func:`._sync_clone.clone_project`; see that function
1714
+ (and ``sync-workflow.md``) for the full behavior -- copy + parameterize
1715
+ (bucket_map / variable_values / instance_rename) + push, with Phase-C/D
1716
+ link remap, a fresh-target guard, and idempotent re-runs.
1717
+ """
1718
+ return _clone_project_impl(
1719
+ self,
1720
+ source,
1721
+ target_alias,
1722
+ target_dir,
1723
+ overrides=overrides,
1724
+ dry_run=dry_run,
1725
+ branch_override=branch_override,
1726
+ )
1727
+
1728
+ # Kept for test compatibility: tests exercise fail-closed encryption via
1729
+ # ``SyncService._encrypt_secrets_in_config(...)``. Production code uses
1730
+ # :func:`encrypt_secrets_in_config` directly.
1731
+ _encrypt_secrets_in_config = staticmethod(encrypt_secrets_in_config)
1732
+
1733
+ def _compute_config_hashes(self, config_dir: Path, component_id: str) -> LocalConfigHashes:
1734
+ """Recompute the manifest bookkeeping hashes from a config dir on disk.
1735
+
1736
+ Reads the (post-writeback) ``_config.yml``, merges code files for the
1737
+ normalized config hash, and hashes each tracked companion file. Used
1738
+ after create / update / variable-link backfill so ``sync diff`` sees
1739
+ local == remote on the next run.
1740
+ """
1741
+ config_file = config_dir / CONFIG_FILENAME
1742
+ file_hash = self._file_hash(config_file) if config_file.exists() else ""
1743
+ local_data = self._read_config_file(config_dir)
1744
+ if local_data is not None:
1745
+ merge_code_files(component_id, local_data, config_dir)
1746
+ cfg_hash = config_hash(local_data)
1747
+ else:
1748
+ cfg_hash = ""
1749
+ extra_hashes: dict[str, str] = {}
1750
+ for fname in _EXTRA_HASH_FILENAMES:
1751
+ fpath = config_dir / fname
1752
+ if fpath.exists():
1753
+ extra_hashes[fname] = self._file_hash(fpath)
1754
+ return LocalConfigHashes(file_hash=file_hash, cfg_hash=cfg_hash, extra_hashes=extra_hashes)
1755
+
1756
+ def _record_push_error(
1757
+ self,
1758
+ errors: list[dict[str, str]],
1759
+ change_type: str,
1760
+ component_id: str,
1761
+ config_id: str,
1762
+ exc: Exception,
1763
+ ) -> None:
1764
+ """Log a non-fatal per-change push failure and accumulate it.
1765
+
1766
+ Callers must re-raise fail-closed encryption errors
1767
+ (:data:`ErrorCode.ENCRYPTION_FAILED`) *before* delegating here so a
1768
+ partial push never silently drops a secret-bearing change.
1769
+ """
1770
+ logger.warning(
1771
+ "Failed to push %s %s/%s: %s",
1772
+ change_type,
1773
+ component_id,
1774
+ config_id,
1775
+ exc,
1776
+ )
1777
+ errors.append(
1778
+ {
1779
+ "change_type": change_type,
1780
+ "component_id": component_id,
1781
+ "config_id": config_id,
1782
+ "message": str(exc),
1783
+ }
1784
+ )
1785
+
1786
+ # ------------------------------------------------------------------
1787
+ # bulk operations (all projects)
1788
+ # ------------------------------------------------------------------
1789
+
1790
+ def pull_all(
1791
+ self,
1792
+ base_dir: Path,
1793
+ force: bool = False,
1794
+ dry_run: bool = False,
1795
+ job_limit: int = DEFAULT_JOBS_PER_CONFIG,
1796
+ no_storage: bool = False,
1797
+ no_jobs: bool = False,
1798
+ with_samples: bool = False,
1799
+ sample_limit: int = DEFAULT_SAMPLE_LIMIT,
1800
+ max_samples: int = DEFAULT_MAX_SAMPLES,
1801
+ ) -> dict[str, Any]:
1802
+ """Pull all registered projects in parallel (see ``_sync_bulk.pull_all``)."""
1803
+ return _bulk_pull_all(
1804
+ self,
1805
+ base_dir,
1806
+ force=force,
1807
+ dry_run=dry_run,
1808
+ job_limit=job_limit,
1809
+ no_storage=no_storage,
1810
+ no_jobs=no_jobs,
1811
+ with_samples=with_samples,
1812
+ sample_limit=sample_limit,
1813
+ max_samples=max_samples,
1814
+ )
1815
+
1816
+ def diff_all(self, base_dir: Path) -> dict[str, Any]:
1817
+ """Diff all registered projects with a local manifest (see ``_sync_bulk.diff_all``)."""
1818
+ return _bulk_diff_all(self, base_dir)
1819
+
1820
+ def push_all(
1821
+ self,
1822
+ base_dir: Path,
1823
+ dry_run: bool = False,
1824
+ force: bool = False,
1825
+ allow_plaintext_fallback: bool = False,
1826
+ ) -> dict[str, Any]:
1827
+ """Push all registered projects with a local manifest (see ``_sync_bulk.push_all``)."""
1828
+ return _bulk_push_all(
1829
+ self,
1830
+ base_dir,
1831
+ dry_run=dry_run,
1832
+ force=force,
1833
+ allow_plaintext_fallback=allow_plaintext_fallback,
1834
+ )
1835
+
1836
+ # ------------------------------------------------------------------
1837
+ # branch mapping
1838
+ # ------------------------------------------------------------------
1839
+
1840
+ def branch_link(
1841
+ self,
1842
+ alias: str,
1843
+ project_root: Path,
1844
+ branch_id: int | None = None,
1845
+ branch_name: str | None = None,
1846
+ ) -> dict[str, Any]:
1847
+ """Link the current git branch to a Keboola dev branch (see ``_sync_branch``)."""
1848
+ return _branch_link(self, alias, project_root, branch_id, branch_name)
1849
+
1850
+ def branch_unlink(self, project_root: Path) -> dict[str, Any]:
1851
+ """Remove the branch mapping for the current git branch (see ``_sync_branch``)."""
1852
+ return _branch_unlink(project_root)
1853
+
1854
+ def branch_status(self, project_root: Path) -> dict[str, Any]:
1855
+ """Show the branch mapping status for the current git branch (see ``_sync_branch``)."""
1856
+ return _branch_status(project_root)
1857
+
1858
+ # ------------------------------------------------------------------
1859
+ # Private helpers
1860
+ # ------------------------------------------------------------------
1861
+
1862
+ @staticmethod
1863
+ def _resolve_branch_id(
1864
+ project: Any,
1865
+ manifest: "Manifest",
1866
+ project_root: Path,
1867
+ branch_override: int | None = None,
1868
+ ) -> int | None:
1869
+ """Resolve the Keboola branch ID for sync operations.
1870
+
1871
+ Priority:
1872
+ 0. ``branch_override`` (CLI ``--branch <id>``) -- wins over everything
1873
+ 1. Git-branching mode: read branch-mapping.json for current git branch
1874
+ 2. ``active_branch_id`` from project config (``kbagent branch use``)
1875
+ 3. First branch in manifest (production fallback)
1876
+
1877
+ Raises ``ConfigError`` if git-branching is enabled but the current
1878
+ branch is not the default and is not linked.
1879
+
1880
+ The default git branch always resolves to ``None`` (production), even
1881
+ if ``branch-mapping.json`` is missing or has no entry for it. This
1882
+ guarantees there is always a recovery path when the mapping file is
1883
+ lost (issue #267, Bug E).
1884
+ """
1885
+ from ..sync.branch_mapping import load_branch_mapping
1886
+ from ..sync.git_utils import get_current_branch
1887
+
1888
+ # CLI override beats every persisted source so a user can target a
1889
+ # dev branch from a clean git workspace without first running
1890
+ # `branch use` or `branch-link`.
1891
+ if branch_override is not None:
1892
+ return branch_override
1893
+
1894
+ if manifest.git_branching.enabled:
1895
+ git_branch = get_current_branch(project_root)
1896
+ if git_branch:
1897
+ default_branch = manifest.git_branching.default_branch
1898
+ is_default = git_branch == default_branch
1899
+ try:
1900
+ mapping = load_branch_mapping(project_root)
1901
+ except FileNotFoundError:
1902
+ # Mapping missing -- auto-recover for the default branch
1903
+ # so the user is never locked out of production.
1904
+ if is_default:
1905
+ return None
1906
+ raise ConfigError(
1907
+ f"Git branch '{git_branch}' is not linked to a Keboola "
1908
+ f"branch (branch-mapping.json missing). "
1909
+ f"Run 'kbagent sync branch-link --project ALIAS' first."
1910
+ ) from None
1911
+ entry = mapping.get(git_branch)
1912
+ if entry is not None:
1913
+ # entry.keboola_id is None for production (default branch)
1914
+ return entry.keboola_id
1915
+ # No entry for current branch -- default branch is always production
1916
+ if is_default:
1917
+ return None
1918
+ raise ConfigError(
1919
+ f"Git branch '{git_branch}' is not linked to a Keboola branch. "
1920
+ f"Run 'kbagent sync branch-link --project ALIAS' first."
1921
+ )
1922
+
1923
+ # Non git-branching: use active_branch_id or manifest fallback
1924
+ branch_id = project.active_branch_id if project is not None else None
1925
+ if not branch_id and manifest.branches:
1926
+ branch_id = manifest.branches[0].id
1927
+ return branch_id
1928
+
1929
+ # ------------------------------------------------------------------
1930
+ # Storage metadata / jobs / samples helpers
1931
+ # ------------------------------------------------------------------
1932
+
1933
+ def _write_config_file(self, config_dir: Path, config_data: dict[str, Any]) -> str:
1934
+ """Write a ``_config.yml`` file and return its SHA256 hash.
1935
+
1936
+ Uses ``newline=""`` so Windows does NOT translate ``\\n`` into ``\\r\\n``
1937
+ on write -- without that, the in-memory ``content`` hash diverges from
1938
+ the on-disk byte hash (:meth:`_file_hash`) and every post-pull
1939
+ ``status`` would report the file as modified.
1940
+ """
1941
+ config_dir.mkdir(parents=True, exist_ok=True)
1942
+ config_file = config_dir / CONFIG_FILENAME
1943
+ content = yaml.dump(
1944
+ config_data,
1945
+ default_flow_style=False,
1946
+ allow_unicode=True,
1947
+ sort_keys=False,
1948
+ width=120,
1949
+ )
1950
+ config_file.write_text(content, encoding="utf-8", newline="")
1951
+ return hashlib.sha256(content.encode("utf-8")).hexdigest()
1952
+
1953
+ def _file_hash(self, file_path: Path) -> str:
1954
+ """Return the SHA256 hex digest of a file's contents."""
1955
+ content = file_path.read_bytes()
1956
+ return hashlib.sha256(content).hexdigest()
1957
+
1958
+ def _detect_name_drift(self, manifest: Manifest, project_root: Path) -> list[dict[str, str]]:
1959
+ """Detect configs where local dir name doesn't match the config name.
1960
+
1961
+ Reads each tracked config's _config.yml to get the current name,
1962
+ then compares sanitize_name(name) against the directory basename.
1963
+
1964
+ Returns a list of warning dicts with component_id, config_id,
1965
+ local_dirname, and expected_dirname.
1966
+ """
1967
+ warnings: list[dict[str, str]] = []
1968
+ for cfg in manifest.configurations:
1969
+ path = cfg.path
1970
+ dirname = path.rsplit("/", 1)[-1] if "/" in path else path
1971
+
1972
+ # Find branch dir and read _config.yml
1973
+ branch_path = self._find_branch_path(manifest, cfg.branch_id)
1974
+ config_dir = project_root / branch_path / path
1975
+ local_data = self._read_config_file(config_dir)
1976
+ if local_data is None:
1977
+ continue
1978
+
1979
+ config_name = local_data.get("name", "")
1980
+ if not config_name:
1981
+ continue
1982
+
1983
+ expected_dirname = sanitize_name(config_name)
1984
+ if dirname != expected_dirname:
1985
+ warnings.append(
1986
+ {
1987
+ "component_id": cfg.component_id,
1988
+ "config_id": cfg.id,
1989
+ "local_dirname": dirname,
1990
+ "expected_dirname": expected_dirname,
1991
+ "config_name": config_name,
1992
+ }
1993
+ )
1994
+ return warnings
1995
+
1996
+ def _rename_directory(self, source: Path, target: Path) -> str:
1997
+ """Rename a directory, using git mv if in a git repo, else shutil.move.
1998
+
1999
+ Returns 'git_mv' or 'shutil_move' indicating which method was used.
2000
+ """
2001
+ target.parent.mkdir(parents=True, exist_ok=True)
2002
+ try:
2003
+ result = subprocess.run(
2004
+ ["git", "mv", str(source), str(target)],
2005
+ cwd=source.parent,
2006
+ capture_output=True,
2007
+ text=True,
2008
+ check=False,
2009
+ )
2010
+ if result.returncode == 0:
2011
+ return "git_mv"
2012
+ except FileNotFoundError:
2013
+ pass # git not installed
2014
+ shutil.move(str(source), str(target))
2015
+ return "shutil_move"
2016
+
2017
+ def _read_config_file(self, config_dir: Path) -> dict[str, Any] | None:
2018
+ """Read and parse a ``_config.yml`` file, returning None if missing."""
2019
+ config_file = config_dir / CONFIG_FILENAME
2020
+ if not config_file.exists():
2021
+ return None
2022
+ try:
2023
+ return yaml.safe_load(config_file.read_text(encoding="utf-8"))
2024
+ except yaml.YAMLError:
2025
+ logger.warning("Failed to parse %s", config_file)
2026
+ return None
2027
+
2028
+ def _ensure_branch_registered(
2029
+ self,
2030
+ manifest: Manifest,
2031
+ branch_id: int | None,
2032
+ client: Any,
2033
+ ) -> str | None:
2034
+ """Ensure *branch_id* has an entry in ``manifest.branches``.
2035
+
2036
+ If *branch_id* is ``None`` (production) or already present, this is
2037
+ a no-op. Otherwise the branch name is fetched from the API and a
2038
+ new :class:`ManifestBranch` is appended.
2039
+
2040
+ Returns:
2041
+ The new branch path if one was added, ``None`` otherwise.
2042
+ """
2043
+ if branch_id is None:
2044
+ return None
2045
+
2046
+ # Already registered?
2047
+ for branch in manifest.branches:
2048
+ if branch.id == branch_id:
2049
+ return None
2050
+
2051
+ # Fetch branch info from API to get a human-readable name
2052
+ all_branches = client.list_dev_branches()
2053
+ branch_name = ""
2054
+ for b in all_branches:
2055
+ if b.get("id") == branch_id:
2056
+ branch_name = b.get("name", "")
2057
+ break
2058
+
2059
+ # Generate filesystem-safe path
2060
+ path = sanitize_name(branch_name) if branch_name else ""
2061
+ if not path:
2062
+ path = f"branch-{branch_id}"
2063
+
2064
+ # Handle path uniqueness -- avoid collisions with existing entries
2065
+ existing_paths = {br.path for br in manifest.branches}
2066
+ if path in existing_paths:
2067
+ path = f"{path}-{branch_id}"
2068
+
2069
+ manifest.branches.append(ManifestBranch(id=branch_id, path=path))
2070
+ logger.info("Registered dev branch %d as '%s' in manifest", branch_id, path)
2071
+ return path
2072
+
2073
+ def _find_branch_path(self, manifest: Manifest, branch_id: int | None) -> str:
2074
+ """Find the branch directory name for a given branch ID.
2075
+
2076
+ When ``branch_id`` is ``None`` (production / default branch),
2077
+ return the first branch path from the manifest.
2078
+ """
2079
+ if branch_id is None:
2080
+ # Production -- use default branch path
2081
+ return manifest.branches[0].path if manifest.branches else "main"
2082
+ for branch in manifest.branches:
2083
+ if branch.id == branch_id:
2084
+ return branch.path
2085
+ # Fallback to default branch -- should not happen after _ensure_branch_registered
2086
+ logger.warning(
2087
+ "Branch ID %s not found in manifest, falling back to default path",
2088
+ branch_id,
2089
+ )
2090
+ return manifest.branches[0].path if manifest.branches else "main"
2091
+
2092
+ def _branch_path_has_configs(self, branch_dir: Path) -> bool:
2093
+ """Return True if *branch_dir* exists and holds at least one config.
2094
+
2095
+ A config is any ``_config.yml`` below the branch root other than the
2096
+ optional branch-level ``_config.yml`` itself. Used to decide whether a
2097
+ target-branch subtree is materialized on disk.
2098
+ """
2099
+ if not branch_dir.is_dir():
2100
+ return False
2101
+ for config_file in branch_dir.rglob(CONFIG_FILENAME):
2102
+ if config_file.parent != branch_dir:
2103
+ return True
2104
+ return False
2105
+
2106
+ def _resolve_source_branch_path(
2107
+ self,
2108
+ manifest: Manifest,
2109
+ project_root: Path,
2110
+ target_branch_id: int | None,
2111
+ ) -> str:
2112
+ """Resolve the on-disk branch subtree to read local configs from.
2113
+
2114
+ Source (where files live) and target (where the API writes) are
2115
+ decoupled. When the target branch has its own materialized
2116
+ ``<branch_name>/`` subtree on disk, that subtree is the source
2117
+ (unchanged multi-branch-directory behaviour). Otherwise the default
2118
+ branch tree (``manifest.branches[0]``, i.e. ``main/``) is the source --
2119
+ the "promote the default tree to a target dev branch" path used by
2120
+ ``sync push --branch <id>`` when no per-branch subtree exists
2121
+ (KFR-07 option-B).
2122
+
2123
+ API calls still target ``target_branch_id``; only the *read* path is
2124
+ affected.
2125
+ """
2126
+ target_path = self._find_branch_path(manifest, target_branch_id)
2127
+ if self._branch_path_has_configs(project_root / target_path):
2128
+ return target_path
2129
+ default_path = manifest.branches[0].path if manifest.branches else target_path
2130
+ if default_path != target_path:
2131
+ logger.info(
2132
+ "No config files under target branch path '%s'; promoting default "
2133
+ "tree '%s' to branch %s",
2134
+ target_path,
2135
+ default_path,
2136
+ target_branch_id,
2137
+ )
2138
+ return default_path
2139
+
2140
+ def _find_untracked_configs(
2141
+ self,
2142
+ project_root: Path,
2143
+ manifest: Manifest,
2144
+ resolved_branch_id: int | None = None,
2145
+ ) -> list[dict[str, str]]:
2146
+ """Scan for _config.yml files that are not tracked in the manifest.
2147
+
2148
+ Scans branch directories that the user is actively working with:
2149
+ branches that already have tracked configs, the default branch
2150
+ (production), and the branch the caller resolved for the current
2151
+ operation (when provided). This supports the documented
2152
+ "scaffold locally then push" workflow on git-branching workspaces
2153
+ with empty ``manifest.configurations`` (issue #267, Bug B).
2154
+
2155
+ Branches outside this scope are skipped to avoid phantom "added"
2156
+ configs from orphaned dev-branch directories left over from
2157
+ previous work.
2158
+ """
2159
+ tracked_paths: set[str] = set()
2160
+ in_scope_branch_ids: set[int] = set()
2161
+ for cfg in manifest.configurations:
2162
+ branch_path = self._find_branch_path(manifest, cfg.branch_id)
2163
+ tracked_paths.add(str(project_root / branch_path / cfg.path))
2164
+ in_scope_branch_ids.add(cfg.branch_id)
2165
+
2166
+ # Default branch is always in scope -- pushing a brand-new config
2167
+ # against production with empty configurations[] is a legitimate flow.
2168
+ if manifest.branches:
2169
+ in_scope_branch_ids.add(manifest.branches[0].id)
2170
+
2171
+ # The branch the caller resolved is in scope (linked feature branch
2172
+ # the user explicitly switched to via git checkout + branch-link).
2173
+ if resolved_branch_id is not None:
2174
+ in_scope_branch_ids.add(resolved_branch_id)
2175
+
2176
+ added: list[dict[str, str]] = []
2177
+ for branch in manifest.branches:
2178
+ if branch.id not in in_scope_branch_ids:
2179
+ continue
2180
+ branch_dir = project_root / branch.path
2181
+ if not branch_dir.exists():
2182
+ continue
2183
+ for config_file in branch_dir.rglob(CONFIG_FILENAME):
2184
+ config_dir = config_file.parent
2185
+ # Skip row-level configs (they're under rows/ subdirectory)
2186
+ if "rows" in config_dir.parts:
2187
+ continue
2188
+ # Skip branch-level _config.yml
2189
+ if config_dir == branch_dir:
2190
+ continue
2191
+ if str(config_dir) not in tracked_paths:
2192
+ local_data = self._read_config_file(config_dir)
2193
+ keboola_meta = local_data.get("_keboola", {}) if local_data else {}
2194
+ added.append(
2195
+ {
2196
+ "component_id": keboola_meta.get("component_id", "unknown"),
2197
+ "config_id": keboola_meta.get("config_id", ""),
2198
+ "path": str(config_dir.relative_to(project_root / branch.path)),
2199
+ }
2200
+ )
2201
+
2202
+ return added
2203
+
2204
+ def _find_untracked_rows(self, project_root: Path, manifest: Manifest) -> list[dict[str, Any]]:
2205
+ """Scan tracked config dirs for ``rows/*/_config.yml`` not in manifest.
2206
+
2207
+ Paralleling :meth:`_find_untracked_configs` at the row level. A user
2208
+ can drop a hand-crafted row directory under a tracked config's
2209
+ ``rows/`` folder; this surfaces it so :meth:`diff` can flag it as
2210
+ ``"added"`` and :meth:`push` can POST it via ``create_config_row``.
2211
+
2212
+ Each entry contains ``component_id``, ``parent_config_id``,
2213
+ ``row_name`` (from the loaded YAML), ``path`` (relative to the parent
2214
+ config dir, e.g. ``rows/new-row``), and ``data`` (the loaded dict).
2215
+ """
2216
+ added: list[dict[str, Any]] = []
2217
+ for cfg in manifest.configurations:
2218
+ branch_path = self._find_branch_path(manifest, cfg.branch_id)
2219
+ parent_dir = project_root / branch_path / cfg.path
2220
+ rows_dir = parent_dir / "rows"
2221
+ if not rows_dir.is_dir():
2222
+ continue
2223
+ tracked_row_paths = {row.path for row in cfg.rows}
2224
+ for row_subdir in rows_dir.iterdir():
2225
+ if not row_subdir.is_dir():
2226
+ continue
2227
+ row_rel_path = f"rows/{row_subdir.name}"
2228
+ if row_rel_path in tracked_row_paths:
2229
+ continue
2230
+ if not (row_subdir / CONFIG_FILENAME).exists():
2231
+ continue
2232
+ local_data = self._read_config_file(row_subdir)
2233
+ if local_data is None:
2234
+ continue
2235
+ added.append(
2236
+ {
2237
+ "component_id": cfg.component_id,
2238
+ "parent_config_id": cfg.id,
2239
+ "row_name": local_data.get("name", ""),
2240
+ "path": row_rel_path,
2241
+ "data": local_data,
2242
+ }
2243
+ )
2244
+ return added