keboola-cli 0.63.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keboola_agent_cli/__init__.py +34 -0
- keboola_agent_cli/__main__.py +5 -0
- keboola_agent_cli/_ui_dist/assets/arc-DhFYIddx.js +2 -0
- keboola_agent_cli/_ui_dist/assets/arc-DhFYIddx.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/architecture-7EHR7CIX-hNCijx_H.js +1 -0
- keboola_agent_cli/_ui_dist/assets/architectureDiagram-3BPJPVTR-C6hUlprM.js +37 -0
- keboola_agent_cli/_ui_dist/assets/architectureDiagram-3BPJPVTR-C6hUlprM.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/array-BifhSqXX.js +2 -0
- keboola_agent_cli/_ui_dist/assets/array-BifhSqXX.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/blockDiagram-GPEHLZMM-DC7qY9i4.js +133 -0
- keboola_agent_cli/_ui_dist/assets/blockDiagram-GPEHLZMM-DC7qY9i4.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/c4Diagram-AAUBKEIU-5Lh44evt.js +11 -0
- keboola_agent_cli/_ui_dist/assets/c4Diagram-AAUBKEIU-5Lh44evt.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/channel-DBMrXlxx.js +2 -0
- keboola_agent_cli/_ui_dist/assets/channel-DBMrXlxx.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-2J33WTMH-Coy82EBh.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-2J33WTMH-Coy82EBh.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-3OPIFGDE-BQC5CRHI.js +63 -0
- keboola_agent_cli/_ui_dist/assets/chunk-3OPIFGDE-BQC5CRHI.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-4BX2VUAB-DUuEt70o.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-4BX2VUAB-DUuEt70o.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-55IACEB6-BvR-6chF.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-55IACEB6-BvR-6chF.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-5ZQYHXKU-BjcTN7ul.js +3 -0
- keboola_agent_cli/_ui_dist/assets/chunk-5ZQYHXKU-BjcTN7ul.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-727SXJPM-C0zxqqRN.js +207 -0
- keboola_agent_cli/_ui_dist/assets/chunk-727SXJPM-C0zxqqRN.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-AQP2D5EJ-CXf7rIlZ.js +232 -0
- keboola_agent_cli/_ui_dist/assets/chunk-AQP2D5EJ-CXf7rIlZ.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-BSJP7CBP-Oj_FO9Q7.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-BSJP7CBP-Oj_FO9Q7.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-CSCIHK7Q-CcTsLrFc.js +124 -0
- keboola_agent_cli/_ui_dist/assets/chunk-CSCIHK7Q-CcTsLrFc.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-FMBD7UC4-FH-zLkkW.js +16 -0
- keboola_agent_cli/_ui_dist/assets/chunk-FMBD7UC4-FH-zLkkW.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-L5ZTLDWV-B1Ky_e7O.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-L5ZTLDWV-B1Ky_e7O.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-ND2GUHAM-BHz1rpbm.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-ND2GUHAM-BHz1rpbm.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-NNHCCRGN-DlpIbxXb.js +160 -0
- keboola_agent_cli/_ui_dist/assets/chunk-NNHCCRGN-DlpIbxXb.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-NZK2D7GU-tnrSoegS.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-NZK2D7GU-tnrSoegS.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-O5CBEL6O-DxxqDH0l.js +71 -0
- keboola_agent_cli/_ui_dist/assets/chunk-O5CBEL6O-DxxqDH0l.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/chunk-QZHKN3VN-CSjc2gjj.js +2 -0
- keboola_agent_cli/_ui_dist/assets/chunk-QZHKN3VN-CSjc2gjj.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/classDiagram-4FO5ZUOK-BuZcZu85.js +2 -0
- keboola_agent_cli/_ui_dist/assets/classDiagram-4FO5ZUOK-BuZcZu85.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/classDiagram-v2-Q7XG4LA2-BuZcZu85.js +2 -0
- keboola_agent_cli/_ui_dist/assets/classDiagram-v2-Q7XG4LA2-BuZcZu85.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/cose-bilkent-S5V4N54A-Y0L8LDMa.js +2 -0
- keboola_agent_cli/_ui_dist/assets/cose-bilkent-S5V4N54A-Y0L8LDMa.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/cytoscape.esm-C8YCVR3_.js +322 -0
- keboola_agent_cli/_ui_dist/assets/cytoscape.esm-C8YCVR3_.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/dagre-BM42HDAG-UZ-9BTqF.js +5 -0
- keboola_agent_cli/_ui_dist/assets/dagre-BM42HDAG-UZ-9BTqF.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/dagre-Bx709z4p.js +2 -0
- keboola_agent_cli/_ui_dist/assets/dagre-Bx709z4p.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/defaultLocale-C8Fc0cco.js +2 -0
- keboola_agent_cli/_ui_dist/assets/defaultLocale-C8Fc0cco.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/diagram-2AECGRRQ-DoDQ60wi.js +44 -0
- keboola_agent_cli/_ui_dist/assets/diagram-2AECGRRQ-DoDQ60wi.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/diagram-5GNKFQAL-CMGFxpUs.js +11 -0
- keboola_agent_cli/_ui_dist/assets/diagram-5GNKFQAL-CMGFxpUs.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/diagram-KO2AKTUF-1uGDa-Iu.js +4 -0
- keboola_agent_cli/_ui_dist/assets/diagram-KO2AKTUF-1uGDa-Iu.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/diagram-LMA3HP47-XtFH7B51.js +25 -0
- keboola_agent_cli/_ui_dist/assets/diagram-LMA3HP47-XtFH7B51.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/diagram-OG6HWLK6-B4_Te1T5.js +25 -0
- keboola_agent_cli/_ui_dist/assets/diagram-OG6HWLK6-B4_Te1T5.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/dist-Di6zmlv0.js +2 -0
- keboola_agent_cli/_ui_dist/assets/dist-Di6zmlv0.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/erDiagram-TEJ5UH35-NjQkrdFt.js +86 -0
- keboola_agent_cli/_ui_dist/assets/erDiagram-TEJ5UH35-NjQkrdFt.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/eventmodeling-FCH6USID-BrJMIks8.js +1 -0
- keboola_agent_cli/_ui_dist/assets/flowDiagram-I6XJVG4X-CIr8DWl7.js +163 -0
- keboola_agent_cli/_ui_dist/assets/flowDiagram-I6XJVG4X-CIr8DWl7.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/ganttDiagram-6RSMTGT7-C1VY_xbQ.js +293 -0
- keboola_agent_cli/_ui_dist/assets/ganttDiagram-6RSMTGT7-C1VY_xbQ.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/gitGraph-WXDBUCRP-COacYjo-.js +1 -0
- keboola_agent_cli/_ui_dist/assets/gitGraphDiagram-PVQCEYII-DQT8-kg2.js +107 -0
- keboola_agent_cli/_ui_dist/assets/gitGraphDiagram-PVQCEYII-DQT8-kg2.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/graphlib-B8gBHxth.js +2 -0
- keboola_agent_cli/_ui_dist/assets/graphlib-B8gBHxth.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/index-CMq50kkV.css +1 -0
- keboola_agent_cli/_ui_dist/assets/index-D8W97DAz.js +118 -0
- keboola_agent_cli/_ui_dist/assets/index-D8W97DAz.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/info-J43DQDTF-DdCTRIzU.js +1 -0
- keboola_agent_cli/_ui_dist/assets/infoDiagram-5YYISTIA-C77rsoTp.js +3 -0
- keboola_agent_cli/_ui_dist/assets/infoDiagram-5YYISTIA-C77rsoTp.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/init-D6jRqBbL.js +2 -0
- keboola_agent_cli/_ui_dist/assets/init-D6jRqBbL.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/ishikawaDiagram-YF4QCWOH-BcTbXaLy.js +71 -0
- keboola_agent_cli/_ui_dist/assets/ishikawaDiagram-YF4QCWOH-BcTbXaLy.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/journeyDiagram-JHISSGLW-BejeAJQ_.js +140 -0
- keboola_agent_cli/_ui_dist/assets/journeyDiagram-JHISSGLW-BejeAJQ_.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/kanban-definition-UN3LZRKU-BRNz_UrH.js +90 -0
- keboola_agent_cli/_ui_dist/assets/kanban-definition-UN3LZRKU-BRNz_UrH.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/katex-C4eR7coU.js +258 -0
- keboola_agent_cli/_ui_dist/assets/katex-C4eR7coU.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/line-CzAQKFbJ.js +2 -0
- keboola_agent_cli/_ui_dist/assets/line-CzAQKFbJ.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/linear-DUNFFdck.js +2 -0
- keboola_agent_cli/_ui_dist/assets/linear-DUNFFdck.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/mermaid-parser.core-CpuBOkFa.js +5 -0
- keboola_agent_cli/_ui_dist/assets/mermaid-parser.core-CpuBOkFa.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/mindmap-definition-RKZ34NQL-9EJQNjH0.js +97 -0
- keboola_agent_cli/_ui_dist/assets/mindmap-definition-RKZ34NQL-9EJQNjH0.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/ordinal-hYBb2elL.js +2 -0
- keboola_agent_cli/_ui_dist/assets/ordinal-hYBb2elL.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/packet-YPE3B663-DLiiw_B2.js +1 -0
- keboola_agent_cli/_ui_dist/assets/path-BWPyau1x.js +2 -0
- keboola_agent_cli/_ui_dist/assets/path-BWPyau1x.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/pie-LRSECV5Y-CRoO8G1g.js +1 -0
- keboola_agent_cli/_ui_dist/assets/pieDiagram-4H26LBE5-XH4cy6Cb.js +31 -0
- keboola_agent_cli/_ui_dist/assets/pieDiagram-4H26LBE5-XH4cy6Cb.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/quadrantDiagram-W4KKPZXB-fdhc93U8.js +8 -0
- keboola_agent_cli/_ui_dist/assets/quadrantDiagram-W4KKPZXB-fdhc93U8.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/radar-GUYGQ44K-DAlLVJHm.js +1 -0
- keboola_agent_cli/_ui_dist/assets/requirementDiagram-4Y6WPE33-a94eP3R9.js +85 -0
- keboola_agent_cli/_ui_dist/assets/requirementDiagram-4Y6WPE33-a94eP3R9.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/rough.esm-CSKSodPl.js +2 -0
- keboola_agent_cli/_ui_dist/assets/rough.esm-CSKSodPl.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/sankeyDiagram-5OEKKPKP-jcBa02sp.js +41 -0
- keboola_agent_cli/_ui_dist/assets/sankeyDiagram-5OEKKPKP-jcBa02sp.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/sequenceDiagram-3UESZ5HK-A5-GGM-e.js +163 -0
- keboola_agent_cli/_ui_dist/assets/sequenceDiagram-3UESZ5HK-A5-GGM-e.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/src-ZI-V_AF0.js +2 -0
- keboola_agent_cli/_ui_dist/assets/src-ZI-V_AF0.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/stateDiagram-AJRCARHV-BKAA5rqE.js +2 -0
- keboola_agent_cli/_ui_dist/assets/stateDiagram-AJRCARHV-BKAA5rqE.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/stateDiagram-v2-BHNVJYJU-DnJwJBsE.js +2 -0
- keboola_agent_cli/_ui_dist/assets/stateDiagram-v2-BHNVJYJU-DnJwJBsE.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/timeline-definition-PNZ67QCA-Cy39jp8b.js +121 -0
- keboola_agent_cli/_ui_dist/assets/timeline-definition-PNZ67QCA-Cy39jp8b.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/treeView-BLDUP644-DbLYl23-.js +1 -0
- keboola_agent_cli/_ui_dist/assets/treemap-LRROVOQU-Bp0eGlOt.js +1 -0
- keboola_agent_cli/_ui_dist/assets/vennDiagram-CIIHVFJN-BGECKubd.js +35 -0
- keboola_agent_cli/_ui_dist/assets/vennDiagram-CIIHVFJN-BGECKubd.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/wardley-L42UT6IY-D4yH4jqS.js +1 -0
- keboola_agent_cli/_ui_dist/assets/wardleyDiagram-YWT4CUSO-D6XRG3cZ.js +79 -0
- keboola_agent_cli/_ui_dist/assets/wardleyDiagram-YWT4CUSO-D6XRG3cZ.js.map +1 -0
- keboola_agent_cli/_ui_dist/assets/xychartDiagram-2RQKCTM6-DRre-pfZ.js +8 -0
- keboola_agent_cli/_ui_dist/assets/xychartDiagram-2RQKCTM6-DRre-pfZ.js.map +1 -0
- keboola_agent_cli/_ui_dist/index.html +50 -0
- keboola_agent_cli/ai_client.py +83 -0
- keboola_agent_cli/auto_update.py +550 -0
- keboola_agent_cli/changelog.py +1198 -0
- keboola_agent_cli/cli.py +448 -0
- keboola_agent_cli/client.py +3422 -0
- keboola_agent_cli/commands/__init__.py +0 -0
- keboola_agent_cli/commands/_data_app_git.py +343 -0
- keboola_agent_cli/commands/_helpers.py +377 -0
- keboola_agent_cli/commands/_metadata_input.py +49 -0
- keboola_agent_cli/commands/_semantic_layer_crud.py +632 -0
- keboola_agent_cli/commands/_semantic_layer_helpers.py +44 -0
- keboola_agent_cli/commands/_semantic_layer_reference_data.py +247 -0
- keboola_agent_cli/commands/agent.py +968 -0
- keboola_agent_cli/commands/branch.py +423 -0
- keboola_agent_cli/commands/changelog.py +168 -0
- keboola_agent_cli/commands/component.py +216 -0
- keboola_agent_cli/commands/config.py +2442 -0
- keboola_agent_cli/commands/context.py +1481 -0
- keboola_agent_cli/commands/data_app.py +1279 -0
- keboola_agent_cli/commands/dev_portal.py +584 -0
- keboola_agent_cli/commands/doctor.py +37 -0
- keboola_agent_cli/commands/encrypt.py +145 -0
- keboola_agent_cli/commands/feature.py +311 -0
- keboola_agent_cli/commands/flow.py +948 -0
- keboola_agent_cli/commands/http_client.py +157 -0
- keboola_agent_cli/commands/init.py +279 -0
- keboola_agent_cli/commands/job.py +661 -0
- keboola_agent_cli/commands/kai.py +301 -0
- keboola_agent_cli/commands/lineage.py +1464 -0
- keboola_agent_cli/commands/org.py +292 -0
- keboola_agent_cli/commands/permissions.py +360 -0
- keboola_agent_cli/commands/project.py +1192 -0
- keboola_agent_cli/commands/repl.py +243 -0
- keboola_agent_cli/commands/schedule.py +340 -0
- keboola_agent_cli/commands/search.py +178 -0
- keboola_agent_cli/commands/semantic_layer.py +939 -0
- keboola_agent_cli/commands/serve.py +272 -0
- keboola_agent_cli/commands/sharing.py +340 -0
- keboola_agent_cli/commands/storage.py +2630 -0
- keboola_agent_cli/commands/stream.py +266 -0
- keboola_agent_cli/commands/sync.py +1277 -0
- keboola_agent_cli/commands/tool.py +206 -0
- keboola_agent_cli/commands/version.py +186 -0
- keboola_agent_cli/commands/workspace.py +635 -0
- keboola_agent_cli/config_store.py +582 -0
- keboola_agent_cli/constants.py +528 -0
- keboola_agent_cli/data_science_client.py +342 -0
- keboola_agent_cli/dev_portal_client.py +323 -0
- keboola_agent_cli/errors.py +248 -0
- keboola_agent_cli/http_base.py +315 -0
- keboola_agent_cli/json_utils.py +126 -0
- keboola_agent_cli/lib.py +536 -0
- keboola_agent_cli/manage_client.py +324 -0
- keboola_agent_cli/metastore_client.py +214 -0
- keboola_agent_cli/models.py +427 -0
- keboola_agent_cli/output.py +1084 -0
- keboola_agent_cli/permissions.py +469 -0
- keboola_agent_cli/py.typed +3 -0
- keboola_agent_cli/result_models.py +271 -0
- keboola_agent_cli/server/__init__.py +34 -0
- keboola_agent_cli/server/agent_runner.py +1289 -0
- keboola_agent_cli/server/agents_store.py +325 -0
- keboola_agent_cli/server/app.py +764 -0
- keboola_agent_cli/server/auth.py +117 -0
- keboola_agent_cli/server/dependencies.py +149 -0
- keboola_agent_cli/server/pricing.py +303 -0
- keboola_agent_cli/server/routers/__init__.py +1 -0
- keboola_agent_cli/server/routers/agents.py +616 -0
- keboola_agent_cli/server/routers/ai_chat.py +129 -0
- keboola_agent_cli/server/routers/branches.py +133 -0
- keboola_agent_cli/server/routers/components.py +48 -0
- keboola_agent_cli/server/routers/configs.py +507 -0
- keboola_agent_cli/server/routers/data_apps.py +384 -0
- keboola_agent_cli/server/routers/dev_portal.py +67 -0
- keboola_agent_cli/server/routers/encrypt.py +35 -0
- keboola_agent_cli/server/routers/feature.py +179 -0
- keboola_agent_cli/server/routers/flows.py +204 -0
- keboola_agent_cli/server/routers/health.py +53 -0
- keboola_agent_cli/server/routers/jobs.py +175 -0
- keboola_agent_cli/server/routers/kai.py +80 -0
- keboola_agent_cli/server/routers/lineage.py +226 -0
- keboola_agent_cli/server/routers/mcp.py +70 -0
- keboola_agent_cli/server/routers/members.py +170 -0
- keboola_agent_cli/server/routers/org.py +96 -0
- keboola_agent_cli/server/routers/projects.py +106 -0
- keboola_agent_cli/server/routers/schedules.py +54 -0
- keboola_agent_cli/server/routers/search.py +30 -0
- keboola_agent_cli/server/routers/semantic_layer.py +650 -0
- keboola_agent_cli/server/routers/sharing.py +86 -0
- keboola_agent_cli/server/routers/storage.py +574 -0
- keboola_agent_cli/server/routers/stream.py +100 -0
- keboola_agent_cli/server/routers/workspaces.py +302 -0
- keboola_agent_cli/server/run_broadcaster.py +329 -0
- keboola_agent_cli/server/sse.py +25 -0
- keboola_agent_cli/services/__init__.py +0 -0
- keboola_agent_cli/services/_encryption.py +217 -0
- keboola_agent_cli/services/_semantic_layer_cascade.py +147 -0
- keboola_agent_cli/services/_semantic_layer_crud.py +382 -0
- keboola_agent_cli/services/_semantic_layer_internals.py +1078 -0
- keboola_agent_cli/services/_semantic_layer_lookup.py +181 -0
- keboola_agent_cli/services/_semantic_layer_reference_data.py +217 -0
- keboola_agent_cli/services/_sync_bindings.py +456 -0
- keboola_agent_cli/services/_sync_branch.py +191 -0
- keboola_agent_cli/services/_sync_bulk.py +228 -0
- keboola_agent_cli/services/_sync_clone.py +163 -0
- keboola_agent_cli/services/_sync_models.py +97 -0
- keboola_agent_cli/services/_sync_push_ops.py +369 -0
- keboola_agent_cli/services/_sync_storage.py +376 -0
- keboola_agent_cli/services/_sync_writeback.py +167 -0
- keboola_agent_cli/services/agent_service.py +458 -0
- keboola_agent_cli/services/base.py +175 -0
- keboola_agent_cli/services/branch_service.py +588 -0
- keboola_agent_cli/services/component_service.py +694 -0
- keboola_agent_cli/services/config_service.py +2099 -0
- keboola_agent_cli/services/data_app_git_service.py +224 -0
- keboola_agent_cli/services/data_app_service.py +2082 -0
- keboola_agent_cli/services/deep_lineage_service.py +1322 -0
- keboola_agent_cli/services/dev_portal_service.py +345 -0
- keboola_agent_cli/services/doctor_service.py +445 -0
- keboola_agent_cli/services/encrypt_service.py +87 -0
- keboola_agent_cli/services/feature_service.py +268 -0
- keboola_agent_cli/services/flow_service.py +769 -0
- keboola_agent_cli/services/flow_validation.py +188 -0
- keboola_agent_cli/services/http_forwarder_service.py +236 -0
- keboola_agent_cli/services/job_idempotency_store.py +285 -0
- keboola_agent_cli/services/job_service.py +797 -0
- keboola_agent_cli/services/kai_service.py +367 -0
- keboola_agent_cli/services/lineage_service.py +274 -0
- keboola_agent_cli/services/mcp_service.py +1498 -0
- keboola_agent_cli/services/mcp_transport.py +259 -0
- keboola_agent_cli/services/member_service.py +593 -0
- keboola_agent_cli/services/org_service.py +619 -0
- keboola_agent_cli/services/project_service.py +947 -0
- keboola_agent_cli/services/repo_validate_service.py +767 -0
- keboola_agent_cli/services/schedule_service.py +731 -0
- keboola_agent_cli/services/search_service.py +331 -0
- keboola_agent_cli/services/semantic_layer_service.py +1497 -0
- keboola_agent_cli/services/sharing_service.py +307 -0
- keboola_agent_cli/services/storage_service.py +2524 -0
- keboola_agent_cli/services/stream_service.py +395 -0
- keboola_agent_cli/services/sync_service.py +2244 -0
- keboola_agent_cli/services/variables_service.py +447 -0
- keboola_agent_cli/services/version_service.py +1038 -0
- keboola_agent_cli/services/workspace_service.py +1103 -0
- keboola_agent_cli/stream_client.py +217 -0
- keboola_agent_cli/sync/__init__.py +1 -0
- keboola_agent_cli/sync/branch_mapping.py +174 -0
- keboola_agent_cli/sync/clone.py +211 -0
- keboola_agent_cli/sync/code_extraction.py +655 -0
- keboola_agent_cli/sync/config_format.py +290 -0
- keboola_agent_cli/sync/diff_engine.py +566 -0
- keboola_agent_cli/sync/git_utils.py +93 -0
- keboola_agent_cli/sync/manifest.py +162 -0
- keboola_agent_cli/sync/naming.py +90 -0
- keboola_agent_cli/sync/secrets.py +62 -0
- keboola_agent_cli/sync/sql_split.py +134 -0
- keboola_cli-0.63.4.dist-info/METADATA +308 -0
- keboola_cli-0.63.4.dist-info/RECORD +306 -0
- keboola_cli-0.63.4.dist-info/WHEEL +4 -0
- keboola_cli-0.63.4.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,2244 @@
|
|
|
1
|
+
"""Sync service - business logic for project pull/push/status operations.
|
|
2
|
+
|
|
3
|
+
Handles downloading Keboola project configurations to the local filesystem
|
|
4
|
+
in a dev-friendly format (YAML configs), and tracking local changes.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import shutil
|
|
11
|
+
import subprocess
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
from ..constants import (
|
|
18
|
+
ALWAYS_IGNORED_COMPONENTS,
|
|
19
|
+
BRANCH_MAPPING_FILENAME,
|
|
20
|
+
CONFIG_FILENAME,
|
|
21
|
+
DEFAULT_JOBS_PER_CONFIG,
|
|
22
|
+
DEFAULT_MAX_SAMPLES,
|
|
23
|
+
DEFAULT_SAMPLE_LIMIT,
|
|
24
|
+
KEBOOLA_DIR_NAME,
|
|
25
|
+
MANIFEST_VERSION,
|
|
26
|
+
)
|
|
27
|
+
from ..errors import ConfigError, ErrorCode, KeboolaApiError, SyncConflictError
|
|
28
|
+
from ..sync.code_extraction import extract_code_files, merge_code_files
|
|
29
|
+
from ..sync.config_format import (
|
|
30
|
+
api_config_to_local,
|
|
31
|
+
api_row_to_local,
|
|
32
|
+
classify_component_type,
|
|
33
|
+
local_config_to_api,
|
|
34
|
+
local_row_to_api,
|
|
35
|
+
)
|
|
36
|
+
from ..sync.diff_engine import compute_changeset, compute_row_changeset, config_hash
|
|
37
|
+
from ..sync.git_utils import get_default_branch, is_git_repo
|
|
38
|
+
from ..sync.manifest import (
|
|
39
|
+
Manifest,
|
|
40
|
+
ManifestBranch,
|
|
41
|
+
ManifestConfigRow,
|
|
42
|
+
ManifestConfiguration,
|
|
43
|
+
ManifestGitBranching,
|
|
44
|
+
ManifestNaming,
|
|
45
|
+
ManifestProject,
|
|
46
|
+
load_manifest,
|
|
47
|
+
save_manifest,
|
|
48
|
+
)
|
|
49
|
+
from ..sync.naming import config_path, config_row_path, sanitize_name
|
|
50
|
+
from ._encryption import (
|
|
51
|
+
encrypt_secrets_in_config,
|
|
52
|
+
find_plaintext_secret_keys,
|
|
53
|
+
)
|
|
54
|
+
from ._sync_bindings import resolve_flow_task_bindings, resolve_variable_bindings
|
|
55
|
+
from ._sync_branch import (
|
|
56
|
+
branch_link as _branch_link,
|
|
57
|
+
)
|
|
58
|
+
from ._sync_branch import (
|
|
59
|
+
branch_status as _branch_status,
|
|
60
|
+
)
|
|
61
|
+
from ._sync_branch import (
|
|
62
|
+
branch_unlink as _branch_unlink,
|
|
63
|
+
)
|
|
64
|
+
from ._sync_bulk import (
|
|
65
|
+
diff_all as _bulk_diff_all,
|
|
66
|
+
)
|
|
67
|
+
from ._sync_bulk import (
|
|
68
|
+
pull_all as _bulk_pull_all,
|
|
69
|
+
)
|
|
70
|
+
from ._sync_bulk import (
|
|
71
|
+
push_all as _bulk_push_all,
|
|
72
|
+
)
|
|
73
|
+
from ._sync_clone import clone_project as _clone_project_impl
|
|
74
|
+
from ._sync_models import CreatedConfig, LocalConfigHashes
|
|
75
|
+
from ._sync_push_ops import push_create, push_row_change, push_update
|
|
76
|
+
from ._sync_storage import (
|
|
77
|
+
fetch_jobs_per_config,
|
|
78
|
+
fetch_samples,
|
|
79
|
+
write_per_config_jobs,
|
|
80
|
+
write_storage_metadata,
|
|
81
|
+
)
|
|
82
|
+
from ._sync_writeback import (
|
|
83
|
+
propagate_kbc_metadata,
|
|
84
|
+
writeback_create_config_in_manifest,
|
|
85
|
+
)
|
|
86
|
+
from .base import BaseService
|
|
87
|
+
|
|
88
|
+
logger = logging.getLogger(__name__)
|
|
89
|
+
|
|
90
|
+
# Companion files extracted alongside a config's ``_config.yml`` whose hashes
|
|
91
|
+
# are tracked so ``sync diff`` notices edits to code/description files.
|
|
92
|
+
_EXTRA_HASH_FILENAMES: tuple[str, ...] = (
|
|
93
|
+
"_description.md",
|
|
94
|
+
"transform.sql",
|
|
95
|
+
"transform.py",
|
|
96
|
+
"code.py",
|
|
97
|
+
"pyproject.toml",
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _ensure_within_branch(
|
|
102
|
+
branch_dir: Path,
|
|
103
|
+
config_dir: Path,
|
|
104
|
+
component_id: str,
|
|
105
|
+
config_id: str,
|
|
106
|
+
) -> None:
|
|
107
|
+
"""Reject paths that escape the branch directory (issue #269 sec-01).
|
|
108
|
+
|
|
109
|
+
Resolves both paths and checks that *config_dir* is contained in
|
|
110
|
+
*branch_dir*. Raises ConfigError if not. Defense-in-depth on top of
|
|
111
|
+
``naming.sanitize_path_segment()`` so a regression in either layer
|
|
112
|
+
cannot turn into a path-traversal write.
|
|
113
|
+
"""
|
|
114
|
+
try:
|
|
115
|
+
branch_resolved = branch_dir.resolve()
|
|
116
|
+
config_resolved = config_dir.resolve()
|
|
117
|
+
except OSError as exc:
|
|
118
|
+
raise ConfigError(f"Cannot resolve sync path: {exc}") from exc
|
|
119
|
+
if not config_resolved.is_relative_to(branch_resolved):
|
|
120
|
+
raise ConfigError(
|
|
121
|
+
f"Config path escapes sync workspace (component='{component_id}', "
|
|
122
|
+
f"config_id='{config_id}'). Refusing to write outside "
|
|
123
|
+
f"'{branch_resolved}'. This indicates a malformed API response "
|
|
124
|
+
f"or a regression in path sanitization."
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def scan_synced_plaintext_secrets(
|
|
129
|
+
project_root: Path, manifest: Manifest | None = None
|
|
130
|
+
) -> list[dict[str, Any]]:
|
|
131
|
+
"""Find in-sync configs/rows whose local files still hold plaintext #-secrets.
|
|
132
|
+
|
|
133
|
+
Regression guard for issue #378. A config that is *in sync* with the remote
|
|
134
|
+
(local file hash == manifest ``pull_hash``) but whose ``#``-prefixed values
|
|
135
|
+
are NOT ``KBC::``-encrypted means the remote is holding that secret in
|
|
136
|
+
plaintext -- the value passed through the sync baseline unencrypted (written
|
|
137
|
+
by a pre-0.54.0 ``config``/``sync`` path, or pulled from an already-leaked
|
|
138
|
+
remote). Pending local edits (hash != ``pull_hash``) are deliberately
|
|
139
|
+
skipped: there a ``sync push`` on >=0.54.0 encrypts on write, so flagging
|
|
140
|
+
them would be noise.
|
|
141
|
+
|
|
142
|
+
Read-only -- filesystem + manifest only, no API client. Returns one entry
|
|
143
|
+
per affected config/row with the secret *key paths* (never the values).
|
|
144
|
+
|
|
145
|
+
Args:
|
|
146
|
+
project_root: Root of the sync working tree.
|
|
147
|
+
manifest: Already-loaded manifest to reuse (callers like
|
|
148
|
+
:meth:`SyncService.status` load it themselves). Loaded from
|
|
149
|
+
*project_root* when ``None``.
|
|
150
|
+
|
|
151
|
+
Raises:
|
|
152
|
+
FileNotFoundError: if *manifest* is ``None`` and *project_root* has no
|
|
153
|
+
``.keboola/manifest.json``.
|
|
154
|
+
"""
|
|
155
|
+
if manifest is None:
|
|
156
|
+
manifest = load_manifest(project_root)
|
|
157
|
+
warnings: list[dict[str, Any]] = []
|
|
158
|
+
|
|
159
|
+
def _branch_path(branch_id: int | None) -> str:
|
|
160
|
+
if branch_id is not None:
|
|
161
|
+
for b in manifest.branches:
|
|
162
|
+
if b.id == branch_id:
|
|
163
|
+
return b.path
|
|
164
|
+
return manifest.branches[0].path if manifest.branches else "main"
|
|
165
|
+
|
|
166
|
+
def _read_yaml(path: Path) -> dict[str, Any] | None:
|
|
167
|
+
if not path.exists():
|
|
168
|
+
return None
|
|
169
|
+
try:
|
|
170
|
+
parsed = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
171
|
+
except yaml.YAMLError:
|
|
172
|
+
return None
|
|
173
|
+
return parsed if isinstance(parsed, dict) else None
|
|
174
|
+
|
|
175
|
+
def _in_sync(path: Path, pull_hash: str) -> bool:
|
|
176
|
+
if not pull_hash or not path.exists():
|
|
177
|
+
return False
|
|
178
|
+
return hashlib.sha256(path.read_bytes()).hexdigest() == pull_hash
|
|
179
|
+
|
|
180
|
+
for cfg in manifest.configurations:
|
|
181
|
+
config_dir = project_root / _branch_path(cfg.branch_id) / cfg.path
|
|
182
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
183
|
+
data = _read_yaml(config_file)
|
|
184
|
+
if data is not None and _in_sync(config_file, cfg.metadata.get("pull_hash", "")):
|
|
185
|
+
_name, _desc, configuration = local_config_to_api(data)
|
|
186
|
+
keys = find_plaintext_secret_keys(configuration)
|
|
187
|
+
if keys:
|
|
188
|
+
warnings.append(
|
|
189
|
+
{
|
|
190
|
+
"component_id": cfg.component_id,
|
|
191
|
+
"config_id": cfg.id,
|
|
192
|
+
"path": str(cfg.path),
|
|
193
|
+
"scope": "config",
|
|
194
|
+
"secret_keys": keys,
|
|
195
|
+
}
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
for row in cfg.rows:
|
|
199
|
+
row_file = config_dir / row.path / CONFIG_FILENAME
|
|
200
|
+
row_data = _read_yaml(row_file)
|
|
201
|
+
if row_data is not None and _in_sync(row_file, row.metadata.get("pull_hash", "")):
|
|
202
|
+
_name, _desc, row_cfg = local_row_to_api(row_data, cfg.component_id)
|
|
203
|
+
row_keys = find_plaintext_secret_keys(row_cfg)
|
|
204
|
+
if row_keys:
|
|
205
|
+
warnings.append(
|
|
206
|
+
{
|
|
207
|
+
"component_id": cfg.component_id,
|
|
208
|
+
"config_id": cfg.id,
|
|
209
|
+
"row_id": row.id,
|
|
210
|
+
"path": f"{cfg.path}/{row.path}",
|
|
211
|
+
"scope": "row",
|
|
212
|
+
"secret_keys": row_keys,
|
|
213
|
+
}
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
return warnings
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
class SyncService(BaseService):
|
|
220
|
+
"""Business logic for project sync operations (init, pull, status).
|
|
221
|
+
|
|
222
|
+
Single-project operations only. Uses dependency injection for
|
|
223
|
+
config_store and client_factory following the BaseService pattern.
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
# ------------------------------------------------------------------
|
|
227
|
+
# init
|
|
228
|
+
# ------------------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
def init_sync(
|
|
231
|
+
self,
|
|
232
|
+
alias: str,
|
|
233
|
+
project_root: Path,
|
|
234
|
+
git_branching: bool = False,
|
|
235
|
+
adopt_existing: bool = False,
|
|
236
|
+
) -> dict[str, Any]:
|
|
237
|
+
"""Initialize a sync working directory for a project.
|
|
238
|
+
|
|
239
|
+
Creates the ``.keboola/`` directory with ``manifest.json``.
|
|
240
|
+
Fetches project metadata from the API to populate the manifest.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
alias: Project alias from config store.
|
|
244
|
+
project_root: Root directory for the sync working tree.
|
|
245
|
+
git_branching: Enable git-branching mode.
|
|
246
|
+
adopt_existing: If True and a manifest already exists, validate it
|
|
247
|
+
against the alias's project_id and normalise it (idempotent
|
|
248
|
+
upgrade of a ``kbc``-written manifest) instead of refusing.
|
|
249
|
+
|
|
250
|
+
Returns:
|
|
251
|
+
Dict with initialization stats and created file paths.
|
|
252
|
+
|
|
253
|
+
Raises:
|
|
254
|
+
ConfigError: If the project alias is not found, or if
|
|
255
|
+
``adopt_existing`` is True but the manifest's project_id does
|
|
256
|
+
not match the alias's project.
|
|
257
|
+
FileExistsError: If manifest already exists and adopt_existing is
|
|
258
|
+
False (use ``sync pull`` to update).
|
|
259
|
+
"""
|
|
260
|
+
projects = self.resolve_projects([alias])
|
|
261
|
+
project = projects[alias]
|
|
262
|
+
|
|
263
|
+
keboola_dir = project_root / KEBOOLA_DIR_NAME
|
|
264
|
+
manifest_path = keboola_dir / "manifest.json"
|
|
265
|
+
if manifest_path.exists():
|
|
266
|
+
if adopt_existing:
|
|
267
|
+
return self._adopt_existing_manifest(alias, project_root, project)
|
|
268
|
+
raise FileExistsError(
|
|
269
|
+
f"Manifest already exists at {manifest_path}. "
|
|
270
|
+
"Use 'sync pull' to update, 'sync init --adopt-existing' to adopt a "
|
|
271
|
+
"kbc-written manifest, or delete .keboola/ to reinitialize."
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
# Fetch project info from API
|
|
275
|
+
client = self._client_factory(project.stack_url, project.token)
|
|
276
|
+
with client:
|
|
277
|
+
token_info = client.verify_token()
|
|
278
|
+
branches = client.list_dev_branches()
|
|
279
|
+
|
|
280
|
+
project_id = token_info.project_id
|
|
281
|
+
if project_id is None:
|
|
282
|
+
raise ConfigError("Token verification returned no project ID; cannot build manifest.")
|
|
283
|
+
api_host = project.stack_url.replace("https://", "").rstrip("/")
|
|
284
|
+
default_branch_info = next(
|
|
285
|
+
(b for b in branches if b.get("isDefault")),
|
|
286
|
+
None,
|
|
287
|
+
)
|
|
288
|
+
default_branch_id = default_branch_info["id"] if default_branch_info else None
|
|
289
|
+
default_branch_name = "main"
|
|
290
|
+
|
|
291
|
+
# Git branching setup
|
|
292
|
+
git_branching_config = ManifestGitBranching(enabled=False)
|
|
293
|
+
if git_branching:
|
|
294
|
+
if not is_git_repo(project_root):
|
|
295
|
+
raise ConfigError("Git repository not found. Initialize git first: git init")
|
|
296
|
+
default_branch_name = get_default_branch(project_root)
|
|
297
|
+
git_branching_config = ManifestGitBranching(
|
|
298
|
+
enabled=True,
|
|
299
|
+
defaultBranch=default_branch_name,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
# Build manifest
|
|
303
|
+
manifest = Manifest(
|
|
304
|
+
version=MANIFEST_VERSION,
|
|
305
|
+
project=ManifestProject(id=project_id, apiHost=api_host),
|
|
306
|
+
allowTargetEnv=True,
|
|
307
|
+
gitBranching=git_branching_config,
|
|
308
|
+
naming=ManifestNaming(),
|
|
309
|
+
branches=[
|
|
310
|
+
ManifestBranch(
|
|
311
|
+
id=default_branch_id,
|
|
312
|
+
path=default_branch_name,
|
|
313
|
+
)
|
|
314
|
+
]
|
|
315
|
+
if default_branch_id
|
|
316
|
+
else [],
|
|
317
|
+
configurations=[],
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
# Save manifest
|
|
321
|
+
save_manifest(project_root, manifest)
|
|
322
|
+
|
|
323
|
+
created_files = [str(manifest_path)]
|
|
324
|
+
|
|
325
|
+
# Create branch mapping if git-branching mode
|
|
326
|
+
if git_branching:
|
|
327
|
+
mapping = {
|
|
328
|
+
"version": 1,
|
|
329
|
+
"mappings": {
|
|
330
|
+
default_branch_name: {
|
|
331
|
+
"id": None,
|
|
332
|
+
"name": "Main",
|
|
333
|
+
}
|
|
334
|
+
},
|
|
335
|
+
}
|
|
336
|
+
mapping_path = keboola_dir / BRANCH_MAPPING_FILENAME
|
|
337
|
+
mapping_path.write_text(
|
|
338
|
+
json.dumps(mapping, indent=4, ensure_ascii=False) + "\n",
|
|
339
|
+
encoding="utf-8",
|
|
340
|
+
)
|
|
341
|
+
created_files.append(str(mapping_path))
|
|
342
|
+
|
|
343
|
+
return {
|
|
344
|
+
"status": "initialized",
|
|
345
|
+
"project_id": project_id,
|
|
346
|
+
"project_alias": alias,
|
|
347
|
+
"api_host": api_host,
|
|
348
|
+
"git_branching": git_branching,
|
|
349
|
+
"default_branch": default_branch_name,
|
|
350
|
+
"files_created": created_files,
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
def _adopt_existing_manifest(
|
|
354
|
+
self,
|
|
355
|
+
alias: str,
|
|
356
|
+
project_root: Path,
|
|
357
|
+
project: Any,
|
|
358
|
+
) -> dict[str, Any]:
|
|
359
|
+
"""Validate and normalise an existing manifest written by kbc or kbagent.
|
|
360
|
+
|
|
361
|
+
Idempotent: loads the manifest, confirms project_id matches the alias,
|
|
362
|
+
then saves it back through kbagent's serialiser (fills missing optional
|
|
363
|
+
fields with defaults, normalises camelCase keys, preserves all
|
|
364
|
+
existing content).
|
|
365
|
+
|
|
366
|
+
Raises:
|
|
367
|
+
ConfigError: If the manifest's project_id doesn't match the alias.
|
|
368
|
+
"""
|
|
369
|
+
existing = load_manifest(project_root)
|
|
370
|
+
|
|
371
|
+
client = self._client_factory(project.stack_url, project.token)
|
|
372
|
+
with client:
|
|
373
|
+
token_info = client.verify_token()
|
|
374
|
+
|
|
375
|
+
if existing.project.id != token_info.project_id:
|
|
376
|
+
raise ConfigError(
|
|
377
|
+
f"Manifest project_id={existing.project.id} does not match alias "
|
|
378
|
+
f"'{alias}' project_id={token_info.project_id}; refusing to overwrite. "
|
|
379
|
+
"Check that --project points to the correct alias."
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
api_host = project.stack_url.replace("https://", "").rstrip("/")
|
|
383
|
+
save_manifest(project_root, existing)
|
|
384
|
+
|
|
385
|
+
return {
|
|
386
|
+
"status": "adopted",
|
|
387
|
+
"project_id": token_info.project_id,
|
|
388
|
+
"project_alias": alias,
|
|
389
|
+
"api_host": api_host,
|
|
390
|
+
"git_branching": existing.git_branching.enabled,
|
|
391
|
+
"default_branch": existing.git_branching.default_branch,
|
|
392
|
+
"files_created": [],
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
# ------------------------------------------------------------------
|
|
396
|
+
# pull
|
|
397
|
+
# ------------------------------------------------------------------
|
|
398
|
+
|
|
399
|
+
def _is_conflict(
|
|
400
|
+
self,
|
|
401
|
+
config_file: Path,
|
|
402
|
+
old_pull_hash: str,
|
|
403
|
+
old_cfg_hash: str,
|
|
404
|
+
api_cfg_hash: str,
|
|
405
|
+
) -> bool:
|
|
406
|
+
"""True iff the file is locally modified AND the remote also changed.
|
|
407
|
+
|
|
408
|
+
A 3-way conflict needs both a stored ``pull_hash`` (the synced file
|
|
409
|
+
state) and a stored ``pull_config_hash`` (the synced remote state);
|
|
410
|
+
without either we cannot prove a conflict, so return False -- be
|
|
411
|
+
conservative, ``--force`` must not abort on incomplete bookkeeping.
|
|
412
|
+
A missing local file is not a content conflict (nothing to lose).
|
|
413
|
+
"""
|
|
414
|
+
if not old_pull_hash or not old_cfg_hash:
|
|
415
|
+
return False
|
|
416
|
+
if not config_file.exists():
|
|
417
|
+
return False
|
|
418
|
+
locally_modified = self._file_hash(config_file) != old_pull_hash
|
|
419
|
+
remote_changed = api_cfg_hash != old_cfg_hash
|
|
420
|
+
return locally_modified and remote_changed
|
|
421
|
+
|
|
422
|
+
def _detect_force_pull_conflicts(
|
|
423
|
+
self,
|
|
424
|
+
components: list[dict[str, Any]],
|
|
425
|
+
branch_dir: Path,
|
|
426
|
+
*,
|
|
427
|
+
existing_keys: set[str],
|
|
428
|
+
existing_paths: dict[str, str],
|
|
429
|
+
existing_file_hashes: dict[str, str],
|
|
430
|
+
existing_config_hashes: dict[str, str],
|
|
431
|
+
existing_rows: dict[str, dict[str, str]],
|
|
432
|
+
) -> list[dict[str, str]]:
|
|
433
|
+
"""Return configs/rows a ``--force`` pull would clobber as conflicts.
|
|
434
|
+
|
|
435
|
+
A *conflict* is a config (or row) that is BOTH locally modified (its
|
|
436
|
+
on-disk ``_config.yml`` hash differs from the manifest ``pull_hash``)
|
|
437
|
+
AND changed on the remote since the last pull (the freshly fetched
|
|
438
|
+
config hash differs from ``pull_config_hash``). That is the only case
|
|
439
|
+
where ``--force`` must stop: local and remote have diverged, so neither
|
|
440
|
+
"take remote" nor "keep local" is safe without the user deciding.
|
|
441
|
+
|
|
442
|
+
Configs only locally modified (remote unchanged) are NOT conflicts --
|
|
443
|
+
``--force`` preserves them so their pending delta stays pushable.
|
|
444
|
+
Brand-new remote configs and configs whose local file is missing are
|
|
445
|
+
skipped (nothing local to lose). Read-only: hashes but writes nothing.
|
|
446
|
+
"""
|
|
447
|
+
conflicts: list[dict[str, str]] = []
|
|
448
|
+
for component in components:
|
|
449
|
+
component_id = component.get("id", "")
|
|
450
|
+
if component_id in ALWAYS_IGNORED_COMPONENTS:
|
|
451
|
+
continue
|
|
452
|
+
for cfg in component.get("configurations", []):
|
|
453
|
+
config_id = str(cfg.get("id", ""))
|
|
454
|
+
lookup_key = f"{component_id}/{config_id}"
|
|
455
|
+
if lookup_key not in existing_keys:
|
|
456
|
+
continue # brand-new remote config -- nothing local to lose
|
|
457
|
+
|
|
458
|
+
rel_path = existing_paths.get(lookup_key, "")
|
|
459
|
+
api_cfg_hash = config_hash(api_config_to_local(component_id, cfg, config_id))
|
|
460
|
+
if self._is_conflict(
|
|
461
|
+
branch_dir / rel_path / CONFIG_FILENAME,
|
|
462
|
+
existing_file_hashes.get(lookup_key, ""),
|
|
463
|
+
existing_config_hashes.get(lookup_key, ""),
|
|
464
|
+
api_cfg_hash,
|
|
465
|
+
):
|
|
466
|
+
conflicts.append(
|
|
467
|
+
{
|
|
468
|
+
"scope": "config",
|
|
469
|
+
"component_id": component_id,
|
|
470
|
+
"config_id": config_id,
|
|
471
|
+
"config_name": str(cfg.get("name", "untitled")),
|
|
472
|
+
"path": rel_path,
|
|
473
|
+
}
|
|
474
|
+
)
|
|
475
|
+
|
|
476
|
+
# Row-level conflicts (same 3-way rule, per row).
|
|
477
|
+
config_dir = branch_dir / rel_path
|
|
478
|
+
for row in cfg.get("rows", []):
|
|
479
|
+
row_id = str(row.get("id", ""))
|
|
480
|
+
existing_row = existing_rows.get(f"{component_id}/{config_id}/{row_id}")
|
|
481
|
+
if not existing_row:
|
|
482
|
+
continue
|
|
483
|
+
row_rel_path = existing_row.get("path", "")
|
|
484
|
+
if self._is_conflict(
|
|
485
|
+
config_dir / row_rel_path / CONFIG_FILENAME,
|
|
486
|
+
existing_row.get("pull_hash", ""),
|
|
487
|
+
existing_row.get("pull_config_hash", ""),
|
|
488
|
+
config_hash(api_row_to_local(row, component_id)),
|
|
489
|
+
):
|
|
490
|
+
conflicts.append(
|
|
491
|
+
{
|
|
492
|
+
"scope": "row",
|
|
493
|
+
"component_id": component_id,
|
|
494
|
+
"config_id": config_id,
|
|
495
|
+
"config_name": (
|
|
496
|
+
f"{cfg.get('name', 'untitled')}/{row.get('name', 'untitled')}"
|
|
497
|
+
),
|
|
498
|
+
"path": f"{rel_path}/{row_rel_path}",
|
|
499
|
+
"row_id": row_id,
|
|
500
|
+
}
|
|
501
|
+
)
|
|
502
|
+
return conflicts
|
|
503
|
+
|
|
504
|
+
def pull(
|
|
505
|
+
self,
|
|
506
|
+
alias: str,
|
|
507
|
+
project_root: Path,
|
|
508
|
+
force: bool = False,
|
|
509
|
+
dry_run: bool = False,
|
|
510
|
+
job_limit: int = DEFAULT_JOBS_PER_CONFIG,
|
|
511
|
+
no_storage: bool = False,
|
|
512
|
+
no_jobs: bool = False,
|
|
513
|
+
with_samples: bool = False,
|
|
514
|
+
sample_limit: int = DEFAULT_SAMPLE_LIMIT,
|
|
515
|
+
max_samples: int = DEFAULT_MAX_SAMPLES,
|
|
516
|
+
branch_override: int | None = None,
|
|
517
|
+
) -> dict[str, Any]:
|
|
518
|
+
"""Download all configurations from Keboola to local filesystem.
|
|
519
|
+
|
|
520
|
+
Args:
|
|
521
|
+
alias: Project alias from config store.
|
|
522
|
+
project_root: Root directory of the sync working tree.
|
|
523
|
+
force: If True, overwrite existing local files without checking.
|
|
524
|
+
dry_run: If True, compute what would be pulled but don't write.
|
|
525
|
+
job_limit: Max jobs per config to pull (default 5).
|
|
526
|
+
no_storage: Skip storage metadata download.
|
|
527
|
+
no_jobs: Skip per-config jobs download.
|
|
528
|
+
with_samples: Download table data samples (opt-in).
|
|
529
|
+
sample_limit: Max rows per sample (default 100).
|
|
530
|
+
max_samples: Max number of tables to sample (default 50).
|
|
531
|
+
branch_override: If set, pull from this branch ID rather than
|
|
532
|
+
the resolved active/manifest branch (CLI ``--branch``).
|
|
533
|
+
|
|
534
|
+
Returns:
|
|
535
|
+
Dict with pull statistics (configs, rows, files written).
|
|
536
|
+
"""
|
|
537
|
+
projects = self.resolve_projects([alias])
|
|
538
|
+
project = projects[alias]
|
|
539
|
+
|
|
540
|
+
# Load or verify manifest exists
|
|
541
|
+
manifest = load_manifest(project_root)
|
|
542
|
+
|
|
543
|
+
# Determine branch to pull from (git-branching aware)
|
|
544
|
+
branch_id = self._resolve_branch_id(
|
|
545
|
+
project, manifest, project_root, branch_override=branch_override
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
# Fetch all components with configs from API (+ storage metadata + jobs)
|
|
549
|
+
client = self._client_factory(project.stack_url, project.token)
|
|
550
|
+
buckets_data: list[dict[str, Any]] = []
|
|
551
|
+
tables_data: list[dict[str, Any]] = []
|
|
552
|
+
jobs_grouped: list[dict[str, Any]] = []
|
|
553
|
+
samples_data: dict[str, str] = {} # table_id -> CSV string
|
|
554
|
+
with client:
|
|
555
|
+
components = client.list_components_with_configs(branch_id=branch_id)
|
|
556
|
+
self._ensure_branch_registered(manifest, branch_id, client)
|
|
557
|
+
|
|
558
|
+
if not no_storage:
|
|
559
|
+
try:
|
|
560
|
+
buckets_data = client.list_buckets_with_metadata()
|
|
561
|
+
tables_data = client.list_tables_with_metadata()
|
|
562
|
+
except Exception:
|
|
563
|
+
logger.warning("Failed to fetch storage metadata", exc_info=True)
|
|
564
|
+
|
|
565
|
+
if not no_jobs:
|
|
566
|
+
try:
|
|
567
|
+
# API constraint: jobsPerGroup * limit <= 500
|
|
568
|
+
group_limit = min(500 // max(job_limit, 1), 500)
|
|
569
|
+
total_configs = sum(len(comp.get("configurations", [])) for comp in components)
|
|
570
|
+
|
|
571
|
+
if group_limit >= total_configs:
|
|
572
|
+
# Fast path: one grouped API call covers all configs
|
|
573
|
+
jobs_grouped = client.list_jobs_grouped(
|
|
574
|
+
jobs_per_group=job_limit,
|
|
575
|
+
limit=group_limit,
|
|
576
|
+
)
|
|
577
|
+
else:
|
|
578
|
+
# Slow path: too many configs for grouped-jobs limit,
|
|
579
|
+
# fetch per-config via /search/jobs in parallel
|
|
580
|
+
logger.info(
|
|
581
|
+
"Project has %d configs but grouped-jobs limit is %d "
|
|
582
|
+
"(job_limit=%d); falling back to per-config fetching",
|
|
583
|
+
total_configs,
|
|
584
|
+
group_limit,
|
|
585
|
+
job_limit,
|
|
586
|
+
)
|
|
587
|
+
jobs_grouped = fetch_jobs_per_config(self, client, components, job_limit)
|
|
588
|
+
except Exception:
|
|
589
|
+
logger.warning("Failed to fetch jobs", exc_info=True)
|
|
590
|
+
|
|
591
|
+
if with_samples and tables_data:
|
|
592
|
+
samples_data = fetch_samples(client, tables_data, sample_limit, max_samples)
|
|
593
|
+
|
|
594
|
+
# Determine branch directory name
|
|
595
|
+
branch_dir_name = self._find_branch_path(manifest, branch_id)
|
|
596
|
+
|
|
597
|
+
branch_dir = project_root / branch_dir_name
|
|
598
|
+
|
|
599
|
+
# Track stats and change details
|
|
600
|
+
configs_pulled = 0
|
|
601
|
+
rows_pulled = 0
|
|
602
|
+
files_written = 0
|
|
603
|
+
new_configurations: list[ManifestConfiguration] = []
|
|
604
|
+
used_paths: set[str] = set() # detect naming collisions
|
|
605
|
+
pull_details: list[dict[str, str]] = [] # per-config change info
|
|
606
|
+
|
|
607
|
+
# Build lookups for existing manifest state
|
|
608
|
+
existing_paths: dict[str, str] = {
|
|
609
|
+
f"{c.component_id}/{c.id}": c.path for c in manifest.configurations
|
|
610
|
+
}
|
|
611
|
+
existing_keys: set[str] = set(existing_paths.keys())
|
|
612
|
+
existing_config_hashes: dict[str, str] = {
|
|
613
|
+
f"{c.component_id}/{c.id}": c.metadata.get("pull_config_hash", "")
|
|
614
|
+
for c in manifest.configurations
|
|
615
|
+
}
|
|
616
|
+
# Build lookup for file hashes at pull time (to detect local edits)
|
|
617
|
+
existing_file_hashes: dict[str, str] = {
|
|
618
|
+
f"{c.component_id}/{c.id}": c.metadata.get("pull_hash", "")
|
|
619
|
+
for c in manifest.configurations
|
|
620
|
+
}
|
|
621
|
+
# Track branch_id per config to detect branch switches
|
|
622
|
+
existing_branch_ids: dict[str, int] = {
|
|
623
|
+
f"{c.component_id}/{c.id}": c.branch_id for c in manifest.configurations
|
|
624
|
+
}
|
|
625
|
+
# Row-level state lookup, keyed by "{component_id}/{config_id}/{row_id}".
|
|
626
|
+
# Values: dict with path, pull_hash, pull_config_hash (all optional).
|
|
627
|
+
existing_rows: dict[str, dict[str, str]] = {}
|
|
628
|
+
for c in manifest.configurations:
|
|
629
|
+
for r in c.rows:
|
|
630
|
+
row_key = f"{c.component_id}/{c.id}/{r.id}"
|
|
631
|
+
existing_rows[row_key] = {
|
|
632
|
+
"path": r.path,
|
|
633
|
+
"pull_hash": r.metadata.get("pull_hash", ""),
|
|
634
|
+
"pull_config_hash": r.metadata.get("pull_config_hash", ""),
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
# Force-pull conflict guard (force-pull baseline corruption fix).
|
|
638
|
+
# ``--force`` bypasses the "preserve locally-modified files" guard
|
|
639
|
+
# below, so a config edited locally AND changed on the remote since the
|
|
640
|
+
# last pull (a true 3-way conflict) would be silently overwritten -- and
|
|
641
|
+
# when the remote is unchanged, its baseline would instead be re-stamped
|
|
642
|
+
# from the edited file, stranding the un-pushed edits. Detect such
|
|
643
|
+
# conflicts up front and abort BEFORE writing anything (the read-only
|
|
644
|
+
# API fetch above has already happened; nothing is on disk yet), so the
|
|
645
|
+
# user can resolve them. Non-force pull preserves locally-modified
|
|
646
|
+
# files and surfaces conflicts via ``sync diff``, so it needs no abort.
|
|
647
|
+
if force:
|
|
648
|
+
conflicts = self._detect_force_pull_conflicts(
|
|
649
|
+
components,
|
|
650
|
+
branch_dir,
|
|
651
|
+
existing_keys=existing_keys,
|
|
652
|
+
existing_paths=existing_paths,
|
|
653
|
+
existing_file_hashes=existing_file_hashes,
|
|
654
|
+
existing_config_hashes=existing_config_hashes,
|
|
655
|
+
existing_rows=existing_rows,
|
|
656
|
+
)
|
|
657
|
+
if conflicts:
|
|
658
|
+
raise SyncConflictError(conflicts)
|
|
659
|
+
|
|
660
|
+
for component in components:
|
|
661
|
+
component_id = component.get("id", "")
|
|
662
|
+
if component_id in ALWAYS_IGNORED_COMPONENTS:
|
|
663
|
+
continue
|
|
664
|
+
component_type = classify_component_type(component.get("type", "other"))
|
|
665
|
+
configs = component.get("configurations", [])
|
|
666
|
+
|
|
667
|
+
for cfg in configs:
|
|
668
|
+
config_id = str(cfg.get("id", ""))
|
|
669
|
+
config_name = cfg.get("name", "untitled")
|
|
670
|
+
|
|
671
|
+
# Reuse existing path if config is already tracked (stable paths)
|
|
672
|
+
lookup_key = f"{component_id}/{config_id}"
|
|
673
|
+
is_new = lookup_key not in existing_keys
|
|
674
|
+
if lookup_key in existing_paths:
|
|
675
|
+
rel_path = existing_paths[lookup_key]
|
|
676
|
+
|
|
677
|
+
# Auto-rename: detect when remote name changed
|
|
678
|
+
expected_path = config_path(
|
|
679
|
+
manifest.naming.config,
|
|
680
|
+
component_type,
|
|
681
|
+
component_id,
|
|
682
|
+
config_name,
|
|
683
|
+
)
|
|
684
|
+
if rel_path != expected_path and not dry_run:
|
|
685
|
+
rename_target = expected_path
|
|
686
|
+
# Collision: if target already used, add suffix
|
|
687
|
+
if rename_target in used_paths:
|
|
688
|
+
suffix = config_id[:8] if len(config_id) > 8 else config_id
|
|
689
|
+
rename_target = f"{rename_target}-{suffix}"
|
|
690
|
+
|
|
691
|
+
old_dir = branch_dir / rel_path
|
|
692
|
+
new_dir = branch_dir / rename_target
|
|
693
|
+
if old_dir.exists() and not new_dir.exists():
|
|
694
|
+
self._rename_directory(old_dir, new_dir)
|
|
695
|
+
pull_details.append(
|
|
696
|
+
{
|
|
697
|
+
"action": "renamed",
|
|
698
|
+
"component_id": component_id,
|
|
699
|
+
"config_name": config_name,
|
|
700
|
+
"path": rename_target,
|
|
701
|
+
"old_path": rel_path,
|
|
702
|
+
}
|
|
703
|
+
)
|
|
704
|
+
rel_path = rename_target
|
|
705
|
+
logger.info(
|
|
706
|
+
"Renamed config dir: %s -> %s",
|
|
707
|
+
rel_path,
|
|
708
|
+
rename_target,
|
|
709
|
+
)
|
|
710
|
+
else:
|
|
711
|
+
# Generate new filesystem path with collision detection
|
|
712
|
+
rel_path = config_path(
|
|
713
|
+
manifest.naming.config,
|
|
714
|
+
component_type,
|
|
715
|
+
component_id,
|
|
716
|
+
config_name,
|
|
717
|
+
)
|
|
718
|
+
if rel_path in used_paths:
|
|
719
|
+
# Append short config ID suffix to resolve collision
|
|
720
|
+
suffix = config_id[:8] if len(config_id) > 8 else config_id
|
|
721
|
+
rel_path = f"{rel_path}-{suffix}"
|
|
722
|
+
used_paths.add(rel_path)
|
|
723
|
+
config_dir = branch_dir / rel_path
|
|
724
|
+
# Defense-in-depth: refuse to write outside the branch dir
|
|
725
|
+
# (issue #269 sec-01). naming.sanitize_path_segment() should
|
|
726
|
+
# already neutralize traversal; this check guards against any
|
|
727
|
+
# future regression in the sanitizer or template parsing.
|
|
728
|
+
_ensure_within_branch(branch_dir, config_dir, component_id, config_id)
|
|
729
|
+
|
|
730
|
+
# Convert API format to local _config.yml
|
|
731
|
+
local_data = api_config_to_local(component_id, cfg, config_id)
|
|
732
|
+
|
|
733
|
+
# Hash of API-converted data. Stored as pull_config_hash so
|
|
734
|
+
# diff can compare it directly with fresh remote data without
|
|
735
|
+
# a lossy file roundtrip.
|
|
736
|
+
api_cfg_hash = config_hash(local_data)
|
|
737
|
+
pull_cfg_hash = api_cfg_hash
|
|
738
|
+
|
|
739
|
+
# Detect local modifications: if the file hash differs from the
|
|
740
|
+
# pull_hash stored in manifest, the user edited the file -- so
|
|
741
|
+
# preserve it instead of overwriting. This now runs even under
|
|
742
|
+
# ``--force``: a force-pull that reaches here for a modified file
|
|
743
|
+
# has already passed the conflict guard above (so the remote is
|
|
744
|
+
# unchanged), and re-stamping the baseline from the edited file
|
|
745
|
+
# would silently strand the un-pushed edits. Preserving keeps
|
|
746
|
+
# the pending delta visible to ``sync push``.
|
|
747
|
+
locally_modified = False
|
|
748
|
+
if not is_new:
|
|
749
|
+
old_file_hash = existing_file_hashes.get(lookup_key, "")
|
|
750
|
+
if old_file_hash:
|
|
751
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
752
|
+
if config_file.exists():
|
|
753
|
+
current_file_hash = self._file_hash(config_file)
|
|
754
|
+
locally_modified = current_file_hash != old_file_hash
|
|
755
|
+
|
|
756
|
+
remote_unchanged = False # set in else branch; default for locally_modified path
|
|
757
|
+
if locally_modified and not dry_run:
|
|
758
|
+
# Preserve the existing local file -- don't overwrite.
|
|
759
|
+
# Still update manifest entry (remote hash changes, but
|
|
760
|
+
# local file stays as-is).
|
|
761
|
+
file_hash = self._file_hash(config_dir / CONFIG_FILENAME)
|
|
762
|
+
pull_details.append(
|
|
763
|
+
{
|
|
764
|
+
"action": "skipped",
|
|
765
|
+
"component_id": component_id,
|
|
766
|
+
"config_name": config_name,
|
|
767
|
+
"path": rel_path,
|
|
768
|
+
"reason": "locally modified",
|
|
769
|
+
}
|
|
770
|
+
)
|
|
771
|
+
elif locally_modified and dry_run:
|
|
772
|
+
file_hash = ""
|
|
773
|
+
pull_details.append(
|
|
774
|
+
{
|
|
775
|
+
"action": "skipped",
|
|
776
|
+
"component_id": component_id,
|
|
777
|
+
"config_name": config_name,
|
|
778
|
+
"path": rel_path,
|
|
779
|
+
"reason": "locally modified",
|
|
780
|
+
}
|
|
781
|
+
)
|
|
782
|
+
else:
|
|
783
|
+
# Check if remote actually changed since last pull.
|
|
784
|
+
# If pull_config_hash matches AND branch hasn't changed,
|
|
785
|
+
# skip write (idempotent). A branch switch means files
|
|
786
|
+
# live in a different directory, so we must re-write.
|
|
787
|
+
old_cfg_hash = existing_config_hashes.get(lookup_key, "")
|
|
788
|
+
branch_switched = existing_branch_ids.get(lookup_key, branch_id or 0) != (
|
|
789
|
+
branch_id or 0
|
|
790
|
+
)
|
|
791
|
+
remote_unchanged = (
|
|
792
|
+
not is_new
|
|
793
|
+
and not branch_switched
|
|
794
|
+
and old_cfg_hash
|
|
795
|
+
and old_cfg_hash == api_cfg_hash
|
|
796
|
+
)
|
|
797
|
+
|
|
798
|
+
if remote_unchanged:
|
|
799
|
+
# Nothing changed -- reuse existing file hash
|
|
800
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
801
|
+
file_hash = self._file_hash(config_file) if config_file.exists() else ""
|
|
802
|
+
else:
|
|
803
|
+
# Extract code files (SQL, Python) if applicable.
|
|
804
|
+
# This modifies local_data in place (removes
|
|
805
|
+
# blocks/code) and writes separate code files.
|
|
806
|
+
if not dry_run:
|
|
807
|
+
extract_code_files(component_id, local_data, config_dir)
|
|
808
|
+
file_hash = self._write_config_file(config_dir, local_data)
|
|
809
|
+
else:
|
|
810
|
+
file_hash = ""
|
|
811
|
+
|
|
812
|
+
configs_pulled += 1
|
|
813
|
+
files_written += 1
|
|
814
|
+
|
|
815
|
+
if is_new:
|
|
816
|
+
pull_details.append(
|
|
817
|
+
{
|
|
818
|
+
"action": "new",
|
|
819
|
+
"component_id": component_id,
|
|
820
|
+
"config_name": config_name,
|
|
821
|
+
"path": rel_path,
|
|
822
|
+
}
|
|
823
|
+
)
|
|
824
|
+
else:
|
|
825
|
+
pull_details.append(
|
|
826
|
+
{
|
|
827
|
+
"action": "updated",
|
|
828
|
+
"component_id": component_id,
|
|
829
|
+
"config_name": config_name,
|
|
830
|
+
"path": rel_path,
|
|
831
|
+
}
|
|
832
|
+
)
|
|
833
|
+
|
|
834
|
+
# Row-level pull: each row gets its own 3-way diff state
|
|
835
|
+
# (pull_hash, pull_config_hash) so ``sync push`` can detect
|
|
836
|
+
# per-row local modifications and push only changed rows.
|
|
837
|
+
row_manifests: list[ManifestConfigRow] = []
|
|
838
|
+
used_row_paths: set[str] = set()
|
|
839
|
+
for row in cfg.get("rows", []):
|
|
840
|
+
row_id = str(row.get("id", ""))
|
|
841
|
+
row_name = row.get("name", "untitled")
|
|
842
|
+
|
|
843
|
+
# Reuse existing row path (stable) if already tracked.
|
|
844
|
+
row_lookup_key = f"{component_id}/{config_id}/{row_id}"
|
|
845
|
+
existing_row = existing_rows.get(row_lookup_key)
|
|
846
|
+
if existing_row:
|
|
847
|
+
row_rel_path = existing_row["path"]
|
|
848
|
+
else:
|
|
849
|
+
row_rel_path = config_row_path(
|
|
850
|
+
manifest.naming.config_row,
|
|
851
|
+
row_name,
|
|
852
|
+
)
|
|
853
|
+
if row_rel_path in used_row_paths:
|
|
854
|
+
suffix = row_id[:8] if len(row_id) > 8 else row_id
|
|
855
|
+
row_rel_path = f"{row_rel_path}-{suffix}"
|
|
856
|
+
used_row_paths.add(row_rel_path)
|
|
857
|
+
row_dir = config_dir / row_rel_path
|
|
858
|
+
|
|
859
|
+
row_local = api_row_to_local(row, component_id)
|
|
860
|
+
row_api_cfg_hash = config_hash(row_local)
|
|
861
|
+
|
|
862
|
+
row_file = row_dir / CONFIG_FILENAME
|
|
863
|
+
old_row_file_hash = existing_row["pull_hash"] if existing_row else ""
|
|
864
|
+
old_row_cfg_hash = existing_row["pull_config_hash"] if existing_row else ""
|
|
865
|
+
# Runs even under ``--force``: a force-pull reaching a
|
|
866
|
+
# modified row has passed the conflict guard (remote
|
|
867
|
+
# unchanged), so preserve the row rather than re-stamp its
|
|
868
|
+
# baseline from the edited file and strand the edits.
|
|
869
|
+
row_locally_modified = False
|
|
870
|
+
if existing_row and old_row_file_hash and row_file.exists():
|
|
871
|
+
row_locally_modified = self._file_hash(row_file) != old_row_file_hash
|
|
872
|
+
|
|
873
|
+
if row_locally_modified:
|
|
874
|
+
# Preserve local edits; keep old hashes as the 3-way base.
|
|
875
|
+
row_file_hash = old_row_file_hash
|
|
876
|
+
row_pull_cfg_hash = old_row_cfg_hash
|
|
877
|
+
pull_details.append(
|
|
878
|
+
{
|
|
879
|
+
"action": "skipped",
|
|
880
|
+
"component_id": component_id,
|
|
881
|
+
"config_name": f"{config_name}/{row_name}",
|
|
882
|
+
"path": f"{rel_path}/{row_rel_path}",
|
|
883
|
+
"reason": "row locally modified",
|
|
884
|
+
}
|
|
885
|
+
)
|
|
886
|
+
elif (
|
|
887
|
+
existing_row
|
|
888
|
+
and old_row_cfg_hash
|
|
889
|
+
and old_row_cfg_hash == row_api_cfg_hash
|
|
890
|
+
and row_file.exists()
|
|
891
|
+
):
|
|
892
|
+
# Idempotent: remote unchanged since last pull, file untouched.
|
|
893
|
+
# Guard: row_file.exists() ensures we don't skip writing when
|
|
894
|
+
# the directory is new (e.g. first pull of a dev branch that
|
|
895
|
+
# clones main -- same hash but no file on disk yet).
|
|
896
|
+
row_file_hash = (
|
|
897
|
+
self._file_hash(row_file) if row_file.exists() else old_row_file_hash
|
|
898
|
+
)
|
|
899
|
+
row_pull_cfg_hash = row_api_cfg_hash
|
|
900
|
+
else:
|
|
901
|
+
# New or remote-changed row: write the file.
|
|
902
|
+
if not dry_run:
|
|
903
|
+
self._write_config_file(row_dir, row_local)
|
|
904
|
+
row_file_hash = self._file_hash(row_file) if row_file.exists() else ""
|
|
905
|
+
else:
|
|
906
|
+
row_file_hash = ""
|
|
907
|
+
row_pull_cfg_hash = row_api_cfg_hash
|
|
908
|
+
files_written += 1
|
|
909
|
+
rows_pulled += 1
|
|
910
|
+
|
|
911
|
+
row_manifests.append(
|
|
912
|
+
ManifestConfigRow(
|
|
913
|
+
id=row_id,
|
|
914
|
+
path=row_rel_path,
|
|
915
|
+
metadata={
|
|
916
|
+
"pull_hash": row_file_hash,
|
|
917
|
+
"pull_config_hash": row_pull_cfg_hash,
|
|
918
|
+
},
|
|
919
|
+
)
|
|
920
|
+
)
|
|
921
|
+
|
|
922
|
+
# Record in manifest (store file hash for change detection).
|
|
923
|
+
# For skipped configs: keep existing pull_hash (file untouched)
|
|
924
|
+
# but do NOT update pull_config_hash -- keep the old base so
|
|
925
|
+
# 3-way diff still correctly detects the local modification.
|
|
926
|
+
if locally_modified:
|
|
927
|
+
old_pull_hash = existing_file_hashes.get(lookup_key, file_hash)
|
|
928
|
+
old_cfg_hash = existing_config_hashes.get(lookup_key, pull_cfg_hash)
|
|
929
|
+
cfg_metadata = {
|
|
930
|
+
"pull_hash": old_pull_hash,
|
|
931
|
+
"pull_config_hash": old_cfg_hash,
|
|
932
|
+
}
|
|
933
|
+
else:
|
|
934
|
+
# Compute hashes for all extracted files
|
|
935
|
+
extra_hashes: dict[str, str] = {}
|
|
936
|
+
if not dry_run:
|
|
937
|
+
for fname in [
|
|
938
|
+
"_description.md",
|
|
939
|
+
"transform.sql",
|
|
940
|
+
"transform.py",
|
|
941
|
+
"code.py",
|
|
942
|
+
"pyproject.toml",
|
|
943
|
+
]:
|
|
944
|
+
fpath = config_dir / fname
|
|
945
|
+
if fpath.exists():
|
|
946
|
+
extra_hashes[fname] = self._file_hash(fpath)
|
|
947
|
+
cfg_metadata = {
|
|
948
|
+
"pull_hash": file_hash,
|
|
949
|
+
"pull_config_hash": pull_cfg_hash,
|
|
950
|
+
"pull_extra_hashes": extra_hashes,
|
|
951
|
+
}
|
|
952
|
+
new_configurations.append(
|
|
953
|
+
ManifestConfiguration(
|
|
954
|
+
branchId=branch_id or 0,
|
|
955
|
+
componentId=component_id,
|
|
956
|
+
id=config_id,
|
|
957
|
+
path=rel_path,
|
|
958
|
+
metadata=cfg_metadata,
|
|
959
|
+
rows=row_manifests,
|
|
960
|
+
)
|
|
961
|
+
)
|
|
962
|
+
|
|
963
|
+
# Detect configs removed from remote (in old manifest but not in new)
|
|
964
|
+
new_keys = {f"{c.component_id}/{c.id}" for c in new_configurations}
|
|
965
|
+
for old_cfg in manifest.configurations:
|
|
966
|
+
old_key = f"{old_cfg.component_id}/{old_cfg.id}"
|
|
967
|
+
if old_key not in new_keys:
|
|
968
|
+
pull_details.append(
|
|
969
|
+
{
|
|
970
|
+
"action": "removed",
|
|
971
|
+
"component_id": old_cfg.component_id,
|
|
972
|
+
"config_name": "",
|
|
973
|
+
"path": old_cfg.path,
|
|
974
|
+
}
|
|
975
|
+
)
|
|
976
|
+
|
|
977
|
+
# Delete orphaned directories for removed configurations
|
|
978
|
+
if not dry_run:
|
|
979
|
+
for detail in pull_details:
|
|
980
|
+
if detail["action"] == "removed" and detail.get("path"):
|
|
981
|
+
orphan_dir = branch_dir / detail["path"]
|
|
982
|
+
if orphan_dir.exists() and orphan_dir.is_dir():
|
|
983
|
+
shutil.rmtree(orphan_dir)
|
|
984
|
+
logger.info("Removed orphaned directory: %s", orphan_dir)
|
|
985
|
+
# Clean up empty parent dirs up to (but not including) branch_dir
|
|
986
|
+
parent = orphan_dir.parent
|
|
987
|
+
while parent != branch_dir and parent.exists():
|
|
988
|
+
if not any(parent.iterdir()):
|
|
989
|
+
parent.rmdir()
|
|
990
|
+
logger.info("Removed empty parent directory: %s", parent)
|
|
991
|
+
parent = parent.parent
|
|
992
|
+
else:
|
|
993
|
+
break
|
|
994
|
+
|
|
995
|
+
# -- Storage metadata (read-only, not tracked in manifest) --
|
|
996
|
+
storage_stats: dict[str, int] = {"buckets": 0, "tables": 0, "samples": 0}
|
|
997
|
+
if not dry_run and buckets_data:
|
|
998
|
+
storage_stats = write_storage_metadata(
|
|
999
|
+
project_root, buckets_data, tables_data, samples_data
|
|
1000
|
+
)
|
|
1001
|
+
|
|
1002
|
+
# -- Per-config jobs (JSONL files next to _config.yml) --
|
|
1003
|
+
jobs_written = 0
|
|
1004
|
+
if not dry_run and jobs_grouped:
|
|
1005
|
+
jobs_written = write_per_config_jobs(branch_dir, new_configurations, jobs_grouped)
|
|
1006
|
+
|
|
1007
|
+
if not dry_run:
|
|
1008
|
+
# Update manifest with pulled configurations
|
|
1009
|
+
manifest.configurations = new_configurations
|
|
1010
|
+
save_manifest(project_root, manifest)
|
|
1011
|
+
|
|
1012
|
+
return {
|
|
1013
|
+
"status": "dry_run" if dry_run else "pulled",
|
|
1014
|
+
"project_alias": alias,
|
|
1015
|
+
"branch_id": branch_id,
|
|
1016
|
+
"branch_dir": branch_dir_name,
|
|
1017
|
+
"configs_pulled": configs_pulled,
|
|
1018
|
+
"rows_pulled": rows_pulled,
|
|
1019
|
+
"files_written": files_written,
|
|
1020
|
+
"jobs_written": jobs_written,
|
|
1021
|
+
"storage": storage_stats,
|
|
1022
|
+
"details": pull_details,
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
# ------------------------------------------------------------------
|
|
1026
|
+
# status
|
|
1027
|
+
# ------------------------------------------------------------------
|
|
1028
|
+
|
|
1029
|
+
def status(self, project_root: Path) -> dict[str, Any]:
|
|
1030
|
+
"""Compare local state against the manifest to detect changes.
|
|
1031
|
+
|
|
1032
|
+
Walks the local filesystem and compares against manifest entries
|
|
1033
|
+
to classify configurations as modified, added, deleted, or unchanged.
|
|
1034
|
+
|
|
1035
|
+
Args:
|
|
1036
|
+
project_root: Root directory of the sync working tree.
|
|
1037
|
+
|
|
1038
|
+
Returns:
|
|
1039
|
+
Dict with lists of modified/added/deleted configs, count of
|
|
1040
|
+
unchanged, and ``plaintext_secret_warnings`` -- in-sync configs/rows
|
|
1041
|
+
whose ``#``-secrets are still plaintext on the remote (issue #378).
|
|
1042
|
+
"""
|
|
1043
|
+
manifest = load_manifest(project_root)
|
|
1044
|
+
|
|
1045
|
+
modified: list[dict[str, str]] = []
|
|
1046
|
+
deleted: list[dict[str, str]] = []
|
|
1047
|
+
unchanged = 0
|
|
1048
|
+
|
|
1049
|
+
# Check each manifest entry against local files
|
|
1050
|
+
for cfg in manifest.configurations:
|
|
1051
|
+
branch_path = self._find_branch_path(manifest, cfg.branch_id)
|
|
1052
|
+
config_dir = project_root / branch_path / cfg.path
|
|
1053
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
1054
|
+
|
|
1055
|
+
if not config_file.exists():
|
|
1056
|
+
deleted.append(
|
|
1057
|
+
{
|
|
1058
|
+
"component_id": cfg.component_id,
|
|
1059
|
+
"config_id": cfg.id,
|
|
1060
|
+
"path": str(cfg.path),
|
|
1061
|
+
}
|
|
1062
|
+
)
|
|
1063
|
+
continue
|
|
1064
|
+
|
|
1065
|
+
# Compare file hash against the hash stored at pull time.
|
|
1066
|
+
# Check both _config.yml AND extracted code files (transform.sql etc.)
|
|
1067
|
+
# to match the same logic used by diff().
|
|
1068
|
+
current_hash = self._file_hash(config_file)
|
|
1069
|
+
pull_hash = cfg.metadata.get("pull_hash", "")
|
|
1070
|
+
config_unchanged = bool(pull_hash and current_hash == pull_hash)
|
|
1071
|
+
|
|
1072
|
+
extras_unchanged = True
|
|
1073
|
+
stored_extra = cfg.metadata.get("pull_extra_hashes", {})
|
|
1074
|
+
for fname, stored_h in stored_extra.items():
|
|
1075
|
+
fpath = config_dir / fname
|
|
1076
|
+
if fpath.exists():
|
|
1077
|
+
if self._file_hash(fpath) != stored_h:
|
|
1078
|
+
extras_unchanged = False
|
|
1079
|
+
break
|
|
1080
|
+
else:
|
|
1081
|
+
extras_unchanged = False
|
|
1082
|
+
break
|
|
1083
|
+
|
|
1084
|
+
if config_unchanged and extras_unchanged:
|
|
1085
|
+
unchanged += 1
|
|
1086
|
+
else:
|
|
1087
|
+
modified.append(
|
|
1088
|
+
{
|
|
1089
|
+
"component_id": cfg.component_id,
|
|
1090
|
+
"config_id": cfg.id,
|
|
1091
|
+
"path": str(cfg.path),
|
|
1092
|
+
}
|
|
1093
|
+
)
|
|
1094
|
+
|
|
1095
|
+
# Scan for added configs (local files without manifest entry)
|
|
1096
|
+
added = self._find_untracked_configs(project_root, manifest)
|
|
1097
|
+
|
|
1098
|
+
# Flag in-sync configs whose secrets are still plaintext (issue #378).
|
|
1099
|
+
# Best-effort: a file race during the scan must not crash `sync status`
|
|
1100
|
+
# (mirrors the doctor sync_secrets check's defensive guard).
|
|
1101
|
+
try:
|
|
1102
|
+
plaintext_secret_warnings = scan_synced_plaintext_secrets(project_root, manifest)
|
|
1103
|
+
except Exception:
|
|
1104
|
+
logger.warning("Plaintext-secret scan failed; skipping", exc_info=True)
|
|
1105
|
+
plaintext_secret_warnings = []
|
|
1106
|
+
|
|
1107
|
+
return {
|
|
1108
|
+
"modified": modified,
|
|
1109
|
+
"added": added,
|
|
1110
|
+
"deleted": deleted,
|
|
1111
|
+
"unchanged": unchanged,
|
|
1112
|
+
"total_tracked": len(manifest.configurations),
|
|
1113
|
+
"plaintext_secret_warnings": plaintext_secret_warnings,
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
# ------------------------------------------------------------------
|
|
1117
|
+
# diff
|
|
1118
|
+
# ------------------------------------------------------------------
|
|
1119
|
+
|
|
1120
|
+
def diff(
|
|
1121
|
+
self,
|
|
1122
|
+
alias: str,
|
|
1123
|
+
project_root: Path,
|
|
1124
|
+
branch_override: int | None = None,
|
|
1125
|
+
) -> dict[str, Any]:
|
|
1126
|
+
"""Compare local configs against the remote API state.
|
|
1127
|
+
|
|
1128
|
+
Fetches current state from API, reads local _config.yml files,
|
|
1129
|
+
and runs the diff engine to produce a detailed changeset.
|
|
1130
|
+
|
|
1131
|
+
Args:
|
|
1132
|
+
alias: Project alias.
|
|
1133
|
+
project_root: Sync working-tree root.
|
|
1134
|
+
branch_override: If set, diff against this branch ID rather
|
|
1135
|
+
than the resolved active/manifest branch.
|
|
1136
|
+
|
|
1137
|
+
Returns:
|
|
1138
|
+
Dict with 'changes' list and summary counts.
|
|
1139
|
+
"""
|
|
1140
|
+
projects = self.resolve_projects([alias])
|
|
1141
|
+
project = projects[alias]
|
|
1142
|
+
manifest = load_manifest(project_root)
|
|
1143
|
+
|
|
1144
|
+
branch_id = self._resolve_branch_id(
|
|
1145
|
+
project, manifest, project_root, branch_override=branch_override
|
|
1146
|
+
)
|
|
1147
|
+
|
|
1148
|
+
# Fetch remote state
|
|
1149
|
+
client = self._client_factory(project.stack_url, project.token)
|
|
1150
|
+
with client:
|
|
1151
|
+
components = client.list_components_with_configs(branch_id=branch_id)
|
|
1152
|
+
self._ensure_branch_registered(manifest, branch_id, client)
|
|
1153
|
+
|
|
1154
|
+
# Build remote lookups:
|
|
1155
|
+
# remote_configs: "{component_id}/{config_id}" -> parent config data
|
|
1156
|
+
# remote_rows: "{component_id}/{parent_config_id}/rows/{row_id}" -> row data
|
|
1157
|
+
remote_configs: dict[str, dict[str, Any]] = {}
|
|
1158
|
+
remote_rows: dict[str, dict[str, Any]] = {}
|
|
1159
|
+
for component in components:
|
|
1160
|
+
component_id = component.get("id", "")
|
|
1161
|
+
if component_id in ALWAYS_IGNORED_COMPONENTS:
|
|
1162
|
+
continue
|
|
1163
|
+
for cfg in component.get("configurations", []):
|
|
1164
|
+
config_id = str(cfg.get("id", ""))
|
|
1165
|
+
key = f"{component_id}/{config_id}"
|
|
1166
|
+
remote_configs[key] = api_config_to_local(component_id, cfg, config_id)
|
|
1167
|
+
for row in cfg.get("rows", []):
|
|
1168
|
+
row_id = str(row.get("id", ""))
|
|
1169
|
+
row_key = f"{component_id}/{config_id}/rows/{row_id}"
|
|
1170
|
+
remote_rows[row_key] = api_row_to_local(row, component_id)
|
|
1171
|
+
|
|
1172
|
+
# Build local configs list from manifest.
|
|
1173
|
+
# For files unchanged since pull, use the stored pull_config_hash
|
|
1174
|
+
# directly (avoids lossy code extraction roundtrip).
|
|
1175
|
+
# For locally modified files, merge code back for real comparison.
|
|
1176
|
+
local_configs: list[dict[str, Any]] = []
|
|
1177
|
+
file_unchanged: dict[str, bool] = {}
|
|
1178
|
+
local_override_hashes: dict[str, str] = {}
|
|
1179
|
+
for cfg in manifest.configurations:
|
|
1180
|
+
branch_path = self._find_branch_path(manifest, cfg.branch_id)
|
|
1181
|
+
config_dir = project_root / branch_path / cfg.path
|
|
1182
|
+
local_data = self._read_config_file(config_dir)
|
|
1183
|
+
if local_data is None:
|
|
1184
|
+
continue
|
|
1185
|
+
|
|
1186
|
+
key = f"{cfg.component_id}/{cfg.id}"
|
|
1187
|
+
|
|
1188
|
+
# Check if ANY file in this config dir changed since pull.
|
|
1189
|
+
# Manifest stores pull_extra_hashes for extracted files.
|
|
1190
|
+
pull_hash = cfg.metadata.get("pull_hash", "")
|
|
1191
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
1192
|
+
current_file_hash = self._file_hash(config_file) if config_file.exists() else ""
|
|
1193
|
+
config_unchanged = bool(pull_hash and current_file_hash == pull_hash)
|
|
1194
|
+
|
|
1195
|
+
extras_unchanged = True
|
|
1196
|
+
stored_extra = cfg.metadata.get("pull_extra_hashes", {})
|
|
1197
|
+
for fname, stored_h in stored_extra.items():
|
|
1198
|
+
fpath = config_dir / fname
|
|
1199
|
+
if fpath.exists():
|
|
1200
|
+
if self._file_hash(fpath) != stored_h:
|
|
1201
|
+
extras_unchanged = False
|
|
1202
|
+
break
|
|
1203
|
+
else:
|
|
1204
|
+
extras_unchanged = False
|
|
1205
|
+
break
|
|
1206
|
+
|
|
1207
|
+
is_unchanged = config_unchanged and extras_unchanged
|
|
1208
|
+
file_unchanged[key] = is_unchanged
|
|
1209
|
+
|
|
1210
|
+
if is_unchanged:
|
|
1211
|
+
# All files match pull state -- use stored API hash
|
|
1212
|
+
stored_cfg_hash = cfg.metadata.get("pull_config_hash", "")
|
|
1213
|
+
if stored_cfg_hash:
|
|
1214
|
+
local_override_hashes[key] = stored_cfg_hash
|
|
1215
|
+
# Always merge for local_data (needed for deep_diff details)
|
|
1216
|
+
merge_code_files(cfg.component_id, local_data, config_dir)
|
|
1217
|
+
|
|
1218
|
+
local_configs.append(
|
|
1219
|
+
{
|
|
1220
|
+
"component_id": cfg.component_id,
|
|
1221
|
+
"config_id": cfg.id,
|
|
1222
|
+
"config_name": local_data.get("name", ""),
|
|
1223
|
+
"path": cfg.path,
|
|
1224
|
+
"data": local_data,
|
|
1225
|
+
}
|
|
1226
|
+
)
|
|
1227
|
+
|
|
1228
|
+
# Also add untracked local configs (new files)
|
|
1229
|
+
for added_cfg in self._find_untracked_configs(project_root, manifest, branch_id):
|
|
1230
|
+
branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id)
|
|
1231
|
+
config_dir = project_root / branch_path / added_cfg["path"]
|
|
1232
|
+
local_data = self._read_config_file(config_dir)
|
|
1233
|
+
if local_data is None:
|
|
1234
|
+
continue
|
|
1235
|
+
local_configs.append(
|
|
1236
|
+
{
|
|
1237
|
+
"component_id": added_cfg.get("component_id", "unknown"),
|
|
1238
|
+
"config_id": "", # new config, no ID yet
|
|
1239
|
+
"config_name": local_data.get("name", ""),
|
|
1240
|
+
"path": added_cfg["path"],
|
|
1241
|
+
"data": local_data,
|
|
1242
|
+
}
|
|
1243
|
+
)
|
|
1244
|
+
|
|
1245
|
+
# Build set of manifest-tracked keys so that compute_changeset only
|
|
1246
|
+
# flags configs that were previously pulled (not brand-new remote ones).
|
|
1247
|
+
tracked_keys = {f"{cfg.component_id}/{cfg.id}" for cfg in manifest.configurations}
|
|
1248
|
+
|
|
1249
|
+
# Build base hashes for 3-way diff.
|
|
1250
|
+
# Preferred: pull_config_hash (normalized hash stored at pull time).
|
|
1251
|
+
# Fallback: if file is unchanged since pull, use current config_hash
|
|
1252
|
+
# as the base (since local == base when file hasn't been modified).
|
|
1253
|
+
base_hashes: dict[str, str] = {}
|
|
1254
|
+
for cfg in manifest.configurations:
|
|
1255
|
+
key = f"{cfg.component_id}/{cfg.id}"
|
|
1256
|
+
pch = cfg.metadata.get("pull_config_hash")
|
|
1257
|
+
if pch:
|
|
1258
|
+
base_hashes[key] = pch
|
|
1259
|
+
elif file_unchanged.get(key):
|
|
1260
|
+
# File not modified locally → local data IS the base.
|
|
1261
|
+
# Find the matching local_configs entry and hash it.
|
|
1262
|
+
for lc in local_configs:
|
|
1263
|
+
if lc.get("config_id") == cfg.id and lc["component_id"] == cfg.component_id:
|
|
1264
|
+
base_hashes[key] = config_hash(lc["data"])
|
|
1265
|
+
break
|
|
1266
|
+
|
|
1267
|
+
changeset = compute_changeset(
|
|
1268
|
+
local_configs,
|
|
1269
|
+
remote_configs,
|
|
1270
|
+
tracked_keys,
|
|
1271
|
+
base_hashes or None,
|
|
1272
|
+
local_override_hashes or None,
|
|
1273
|
+
)
|
|
1274
|
+
|
|
1275
|
+
# Row-level diff: walk manifest rows, load local YAML, feed into
|
|
1276
|
+
# compute_row_changeset alongside remote_rows built above.
|
|
1277
|
+
local_rows: list[dict[str, Any]] = []
|
|
1278
|
+
tracked_row_keys: set[str] = set()
|
|
1279
|
+
row_base_hashes: dict[str, str] = {}
|
|
1280
|
+
for cfg in manifest.configurations:
|
|
1281
|
+
branch_path = self._find_branch_path(manifest, cfg.branch_id)
|
|
1282
|
+
parent_dir = project_root / branch_path / cfg.path
|
|
1283
|
+
for row in cfg.rows:
|
|
1284
|
+
row_key = f"{cfg.component_id}/{cfg.id}/rows/{row.id}"
|
|
1285
|
+
tracked_row_keys.add(row_key)
|
|
1286
|
+
row_dir = parent_dir / row.path
|
|
1287
|
+
row_local = self._read_config_file(row_dir)
|
|
1288
|
+
if row_local is None:
|
|
1289
|
+
# File missing -> "deleted" detected via tracked_row_keys.
|
|
1290
|
+
continue
|
|
1291
|
+
local_rows.append(
|
|
1292
|
+
{
|
|
1293
|
+
"component_id": cfg.component_id,
|
|
1294
|
+
"parent_config_id": cfg.id,
|
|
1295
|
+
"row_id": row.id,
|
|
1296
|
+
"row_name": row_local.get("name", ""),
|
|
1297
|
+
"path": row.path,
|
|
1298
|
+
"data": row_local,
|
|
1299
|
+
}
|
|
1300
|
+
)
|
|
1301
|
+
pch = row.metadata.get("pull_config_hash") if row.metadata else ""
|
|
1302
|
+
if pch:
|
|
1303
|
+
row_base_hashes[row_key] = pch
|
|
1304
|
+
|
|
1305
|
+
# Also add untracked local rows (new row dirs dropped under a tracked
|
|
1306
|
+
# config). They have no row_id yet so compute_row_changeset flags
|
|
1307
|
+
# them as "added" and push dispatches them via create_config_row.
|
|
1308
|
+
for untracked in self._find_untracked_rows(project_root, manifest):
|
|
1309
|
+
local_rows.append(
|
|
1310
|
+
{
|
|
1311
|
+
"component_id": untracked["component_id"],
|
|
1312
|
+
"parent_config_id": untracked["parent_config_id"],
|
|
1313
|
+
"row_id": "",
|
|
1314
|
+
"row_name": untracked["row_name"],
|
|
1315
|
+
"path": untracked["path"],
|
|
1316
|
+
"data": untracked["data"],
|
|
1317
|
+
}
|
|
1318
|
+
)
|
|
1319
|
+
|
|
1320
|
+
row_changeset = compute_row_changeset(
|
|
1321
|
+
local_rows,
|
|
1322
|
+
remote_rows,
|
|
1323
|
+
tracked_row_keys,
|
|
1324
|
+
row_base_hashes or None,
|
|
1325
|
+
)
|
|
1326
|
+
changeset.extend(row_changeset)
|
|
1327
|
+
|
|
1328
|
+
added = [c for c in changeset if c.change_type == "added"]
|
|
1329
|
+
modified = [c for c in changeset if c.change_type == "modified"]
|
|
1330
|
+
remote_modified = [c for c in changeset if c.change_type == "remote_modified"]
|
|
1331
|
+
conflicts = [c for c in changeset if c.change_type == "conflict"]
|
|
1332
|
+
deleted = [c for c in changeset if c.change_type == "deleted"]
|
|
1333
|
+
|
|
1334
|
+
# Detect remote-only configs (new on server, not yet pulled).
|
|
1335
|
+
local_keys = {
|
|
1336
|
+
f"{e['component_id']}/{e['config_id']}" for e in local_configs if e.get("config_id")
|
|
1337
|
+
} | tracked_keys
|
|
1338
|
+
remote_only: list[dict[str, str]] = []
|
|
1339
|
+
for remote_key, remote_data in remote_configs.items():
|
|
1340
|
+
if remote_key not in local_keys:
|
|
1341
|
+
parts = remote_key.split("/", 1)
|
|
1342
|
+
remote_only.append(
|
|
1343
|
+
{
|
|
1344
|
+
"component_id": parts[0] if parts else "",
|
|
1345
|
+
"config_id": parts[1] if len(parts) > 1 else "",
|
|
1346
|
+
"config_name": remote_data.get("name", ""),
|
|
1347
|
+
}
|
|
1348
|
+
)
|
|
1349
|
+
|
|
1350
|
+
return {
|
|
1351
|
+
"changes": [c.to_dict() for c in changeset],
|
|
1352
|
+
"remote_only": remote_only,
|
|
1353
|
+
"summary": {
|
|
1354
|
+
"added": len(added),
|
|
1355
|
+
"modified": len(modified),
|
|
1356
|
+
"remote_modified": len(remote_modified),
|
|
1357
|
+
"conflict": len(conflicts),
|
|
1358
|
+
"deleted": len(deleted),
|
|
1359
|
+
"unchanged": len(local_configs)
|
|
1360
|
+
- len(added)
|
|
1361
|
+
- len(modified)
|
|
1362
|
+
- len(remote_modified)
|
|
1363
|
+
- len(conflicts),
|
|
1364
|
+
"remote_only": len(remote_only),
|
|
1365
|
+
},
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
# ------------------------------------------------------------------
|
|
1369
|
+
# push
|
|
1370
|
+
# ------------------------------------------------------------------
|
|
1371
|
+
|
|
1372
|
+
def push(
|
|
1373
|
+
self,
|
|
1374
|
+
alias: str,
|
|
1375
|
+
project_root: Path,
|
|
1376
|
+
dry_run: bool = False,
|
|
1377
|
+
force: bool = False,
|
|
1378
|
+
allow_plaintext_fallback: bool = False,
|
|
1379
|
+
branch_override: int | None = None,
|
|
1380
|
+
no_name_drift_warnings: bool = False,
|
|
1381
|
+
) -> dict[str, Any]:
|
|
1382
|
+
"""Push local changes to Keboola.
|
|
1383
|
+
|
|
1384
|
+
Computes diff, then creates/updates/deletes configs via API.
|
|
1385
|
+
New configs get IDs assigned by the API; the manifest is updated.
|
|
1386
|
+
|
|
1387
|
+
Args:
|
|
1388
|
+
alias: Project alias from config store.
|
|
1389
|
+
project_root: Root directory of the sync working tree.
|
|
1390
|
+
dry_run: If True, compute changes but don't execute them.
|
|
1391
|
+
force: If True, allow deletions without extra confirmation.
|
|
1392
|
+
allow_plaintext_fallback: If True, allow push when secret
|
|
1393
|
+
encryption fails (DANGEROUS).
|
|
1394
|
+
branch_override: If set, target this dev-branch ID for the push.
|
|
1395
|
+
Wins over ``active_branch_id`` / ``manifest.branches[0]`` /
|
|
1396
|
+
git-branching mapping. Used by the CLI ``--branch`` flag. When
|
|
1397
|
+
no ``<branch_name>/`` subtree exists on disk for the target,
|
|
1398
|
+
the default tree (``main/``) is read as the source and promoted
|
|
1399
|
+
to the target branch (KFR-07); API writes still target the
|
|
1400
|
+
branch id.
|
|
1401
|
+
no_name_drift_warnings: If True, omit the ``name_drift_warnings``
|
|
1402
|
+
array from the result envelope (the underlying detection
|
|
1403
|
+
still runs; only the report is suppressed). Used by the CLI
|
|
1404
|
+
``--no-name-drift-warnings`` flag.
|
|
1405
|
+
|
|
1406
|
+
Returns:
|
|
1407
|
+
Dict with push results (created, updated, deleted, errors).
|
|
1408
|
+
"""
|
|
1409
|
+
diff_result = self.diff(alias, project_root, branch_override=branch_override)
|
|
1410
|
+
all_changes = diff_result["changes"]
|
|
1411
|
+
|
|
1412
|
+
# Only push local-side changes (added, modified, deleted).
|
|
1413
|
+
# Skip remote_modified (need pull) and conflict (need resolution).
|
|
1414
|
+
pushable_types = {"added", "modified", "deleted"}
|
|
1415
|
+
changes = [c for c in all_changes if c["change_type"] in pushable_types]
|
|
1416
|
+
|
|
1417
|
+
# Warn about skipped changes
|
|
1418
|
+
skipped = [c for c in all_changes if c["change_type"] not in pushable_types]
|
|
1419
|
+
|
|
1420
|
+
if not changes:
|
|
1421
|
+
result: dict[str, Any] = {
|
|
1422
|
+
"status": "no_changes",
|
|
1423
|
+
"created": 0,
|
|
1424
|
+
"updated": 0,
|
|
1425
|
+
"deleted": 0,
|
|
1426
|
+
"errors": [],
|
|
1427
|
+
}
|
|
1428
|
+
if skipped:
|
|
1429
|
+
result["skipped"] = len(skipped)
|
|
1430
|
+
result["skipped_reason"] = "Remote changes detected. Run 'sync pull' first."
|
|
1431
|
+
return result
|
|
1432
|
+
|
|
1433
|
+
if dry_run:
|
|
1434
|
+
return {
|
|
1435
|
+
"status": "dry_run",
|
|
1436
|
+
"changes": changes,
|
|
1437
|
+
"summary": diff_result["summary"],
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
projects = self.resolve_projects([alias])
|
|
1441
|
+
project = projects[alias]
|
|
1442
|
+
manifest = load_manifest(project_root)
|
|
1443
|
+
|
|
1444
|
+
branch_id = self._resolve_branch_id(
|
|
1445
|
+
project, manifest, project_root, branch_override=branch_override
|
|
1446
|
+
)
|
|
1447
|
+
|
|
1448
|
+
# Detect name drift: local dir name doesn't match config name
|
|
1449
|
+
name_drift_warnings = self._detect_name_drift(manifest, project_root)
|
|
1450
|
+
|
|
1451
|
+
client = self._client_factory(project.stack_url, project.token)
|
|
1452
|
+
created = 0
|
|
1453
|
+
updated = 0
|
|
1454
|
+
deleted = 0
|
|
1455
|
+
errors: list[dict[str, str]] = []
|
|
1456
|
+
pushed_details: list[dict[str, str]] = []
|
|
1457
|
+
manifest_dirty = False
|
|
1458
|
+
|
|
1459
|
+
with client:
|
|
1460
|
+
self._ensure_branch_registered(manifest, branch_id, client)
|
|
1461
|
+
branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id)
|
|
1462
|
+
|
|
1463
|
+
# Process configs before rows, and rebind variable links last.
|
|
1464
|
+
# A freshly-created parent config must carry its API-assigned ULID
|
|
1465
|
+
# before its rows POST (KFR-05), and a transformation's
|
|
1466
|
+
# variables_id / variables_values_id can only be resolved once both
|
|
1467
|
+
# the variables config and its values row exist (KFR-03). Partition
|
|
1468
|
+
# explicitly rather than relying on incidental diff ordering.
|
|
1469
|
+
config_changes = [c for c in changes if not bool(c.get("is_row"))]
|
|
1470
|
+
row_changes = [c for c in changes if bool(c.get("is_row"))]
|
|
1471
|
+
|
|
1472
|
+
# (component_id, placeholder_id) -> API-assigned ULID, captured
|
|
1473
|
+
# before the manifest writeback overwrites the placeholder in place.
|
|
1474
|
+
created_id_map: dict[tuple[str, str], str] = {}
|
|
1475
|
+
created_configs: list[CreatedConfig] = []
|
|
1476
|
+
|
|
1477
|
+
# ---- Phase A: config creates / updates / deletes -------------
|
|
1478
|
+
for change in config_changes:
|
|
1479
|
+
change_type = change["change_type"]
|
|
1480
|
+
component_id = change["component_id"]
|
|
1481
|
+
config_id = change["config_id"]
|
|
1482
|
+
config_path_str = change.get("path", "")
|
|
1483
|
+
|
|
1484
|
+
try:
|
|
1485
|
+
if change_type == "added":
|
|
1486
|
+
result = push_create(
|
|
1487
|
+
self,
|
|
1488
|
+
client,
|
|
1489
|
+
component_id,
|
|
1490
|
+
config_path_str,
|
|
1491
|
+
project_root,
|
|
1492
|
+
manifest,
|
|
1493
|
+
branch_id,
|
|
1494
|
+
allow_plaintext_fallback=allow_plaintext_fallback,
|
|
1495
|
+
)
|
|
1496
|
+
if result:
|
|
1497
|
+
new_id = str(result.get("id", ""))
|
|
1498
|
+
config_dir = project_root / branch_path / config_path_str
|
|
1499
|
+
hashes = self._compute_config_hashes(config_dir, component_id)
|
|
1500
|
+
writeback = writeback_create_config_in_manifest(
|
|
1501
|
+
manifest=manifest,
|
|
1502
|
+
component_id=component_id,
|
|
1503
|
+
branch_id=branch_id,
|
|
1504
|
+
config_path_str=config_path_str,
|
|
1505
|
+
new_id=new_id,
|
|
1506
|
+
file_hash=hashes.file_hash,
|
|
1507
|
+
cfg_hash=hashes.cfg_hash,
|
|
1508
|
+
)
|
|
1509
|
+
# Record placeholder -> ULID so child rows and
|
|
1510
|
+
# transformation variable links can be remapped.
|
|
1511
|
+
if writeback.previous_id:
|
|
1512
|
+
created_id_map[(component_id, writeback.previous_id)] = new_id
|
|
1513
|
+
created_configs.append(
|
|
1514
|
+
CreatedConfig(
|
|
1515
|
+
component_id=component_id,
|
|
1516
|
+
config_id=new_id,
|
|
1517
|
+
config_dir=config_dir,
|
|
1518
|
+
)
|
|
1519
|
+
)
|
|
1520
|
+
metadata_error = propagate_kbc_metadata(
|
|
1521
|
+
client, writeback.entry, branch_id
|
|
1522
|
+
)
|
|
1523
|
+
if metadata_error is not None:
|
|
1524
|
+
# The config IS on the remote; only the
|
|
1525
|
+
# follow-up metadata POST failed. Accumulate
|
|
1526
|
+
# like any other per-change error so the rest
|
|
1527
|
+
# of the push continues, and surface the
|
|
1528
|
+
# original cause in the envelope.
|
|
1529
|
+
errors.append(
|
|
1530
|
+
{
|
|
1531
|
+
"change_type": "metadata_propagation",
|
|
1532
|
+
"component_id": component_id,
|
|
1533
|
+
"config_id": new_id,
|
|
1534
|
+
"message": metadata_error,
|
|
1535
|
+
}
|
|
1536
|
+
)
|
|
1537
|
+
manifest_dirty = True
|
|
1538
|
+
created += 1
|
|
1539
|
+
pushed_details.append(change)
|
|
1540
|
+
|
|
1541
|
+
elif change_type == "modified":
|
|
1542
|
+
push_update(
|
|
1543
|
+
self,
|
|
1544
|
+
client,
|
|
1545
|
+
component_id,
|
|
1546
|
+
config_id,
|
|
1547
|
+
config_path_str,
|
|
1548
|
+
project_root,
|
|
1549
|
+
manifest,
|
|
1550
|
+
branch_id,
|
|
1551
|
+
allow_plaintext_fallback=allow_plaintext_fallback,
|
|
1552
|
+
)
|
|
1553
|
+
# Update hashes so pull knows local == remote
|
|
1554
|
+
config_dir = project_root / branch_path / config_path_str
|
|
1555
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
1556
|
+
if config_file.exists():
|
|
1557
|
+
hashes = self._compute_config_hashes(config_dir, component_id)
|
|
1558
|
+
for cfg in manifest.configurations:
|
|
1559
|
+
if cfg.component_id == component_id and cfg.id == config_id:
|
|
1560
|
+
cfg.metadata["pull_hash"] = hashes.file_hash
|
|
1561
|
+
cfg.metadata["pull_config_hash"] = hashes.cfg_hash
|
|
1562
|
+
cfg.metadata["pull_extra_hashes"] = hashes.extra_hashes
|
|
1563
|
+
break
|
|
1564
|
+
manifest_dirty = True
|
|
1565
|
+
updated += 1
|
|
1566
|
+
pushed_details.append(change)
|
|
1567
|
+
|
|
1568
|
+
elif change_type == "deleted":
|
|
1569
|
+
client.delete_config(
|
|
1570
|
+
component_id=component_id,
|
|
1571
|
+
config_id=config_id,
|
|
1572
|
+
branch_id=branch_id,
|
|
1573
|
+
)
|
|
1574
|
+
# Remove from manifest
|
|
1575
|
+
manifest.configurations = [
|
|
1576
|
+
c
|
|
1577
|
+
for c in manifest.configurations
|
|
1578
|
+
if not (c.component_id == component_id and c.id == config_id)
|
|
1579
|
+
]
|
|
1580
|
+
manifest_dirty = True
|
|
1581
|
+
deleted += 1
|
|
1582
|
+
pushed_details.append(change)
|
|
1583
|
+
|
|
1584
|
+
except Exception as exc:
|
|
1585
|
+
# Fail-closed on encryption failures: a partial push that
|
|
1586
|
+
# omits the failed change would leave a plaintext secret
|
|
1587
|
+
# elsewhere or a caller believing the push "mostly succeeded".
|
|
1588
|
+
# Surface to the CLI (exit non-zero) rather than burying in
|
|
1589
|
+
# result["errors"].
|
|
1590
|
+
if (
|
|
1591
|
+
isinstance(exc, KeboolaApiError)
|
|
1592
|
+
and exc.error_code == ErrorCode.ENCRYPTION_FAILED
|
|
1593
|
+
):
|
|
1594
|
+
raise
|
|
1595
|
+
self._record_push_error(errors, change_type, component_id, config_id, exc)
|
|
1596
|
+
|
|
1597
|
+
# ---- Phase B: row creates / updates / deletes ----------------
|
|
1598
|
+
# row placeholder id -> ULID; ULID parent -> rows created under it.
|
|
1599
|
+
created_row_id_map: dict[str, str] = {}
|
|
1600
|
+
created_rows_by_parent: dict[str, list[str]] = {}
|
|
1601
|
+
for change in row_changes:
|
|
1602
|
+
change_type = change["change_type"]
|
|
1603
|
+
component_id = change["component_id"]
|
|
1604
|
+
config_id = change["config_id"]
|
|
1605
|
+
config_path_str = change.get("path", "")
|
|
1606
|
+
parent_config_id = change.get("parent_config_id", "")
|
|
1607
|
+
# Remap the diff-time parent placeholder to the ULID assigned in
|
|
1608
|
+
# Phase A so the manifest lookup and create_config_row both hit
|
|
1609
|
+
# the real config (KFR-05). UPDATE/DELETE parents already carry
|
|
1610
|
+
# a ULID and pass through unchanged.
|
|
1611
|
+
effective_parent_id = created_id_map.get(
|
|
1612
|
+
(component_id, parent_config_id), parent_config_id
|
|
1613
|
+
)
|
|
1614
|
+
|
|
1615
|
+
try:
|
|
1616
|
+
new_row_id = push_row_change(
|
|
1617
|
+
self,
|
|
1618
|
+
client,
|
|
1619
|
+
change_type=change_type,
|
|
1620
|
+
component_id=component_id,
|
|
1621
|
+
parent_config_id=effective_parent_id,
|
|
1622
|
+
row_id=config_id,
|
|
1623
|
+
row_path_str=config_path_str,
|
|
1624
|
+
project_root=project_root,
|
|
1625
|
+
manifest=manifest,
|
|
1626
|
+
branch_id=branch_id,
|
|
1627
|
+
allow_plaintext_fallback=allow_plaintext_fallback,
|
|
1628
|
+
)
|
|
1629
|
+
manifest_dirty = True
|
|
1630
|
+
if change_type == "added":
|
|
1631
|
+
created += 1
|
|
1632
|
+
if new_row_id:
|
|
1633
|
+
if config_id:
|
|
1634
|
+
created_row_id_map[config_id] = new_row_id
|
|
1635
|
+
created_rows_by_parent.setdefault(effective_parent_id, []).append(
|
|
1636
|
+
new_row_id
|
|
1637
|
+
)
|
|
1638
|
+
elif change_type == "modified":
|
|
1639
|
+
updated += 1
|
|
1640
|
+
elif change_type == "deleted":
|
|
1641
|
+
deleted += 1
|
|
1642
|
+
pushed_details.append(change)
|
|
1643
|
+
|
|
1644
|
+
except Exception as exc:
|
|
1645
|
+
if (
|
|
1646
|
+
isinstance(exc, KeboolaApiError)
|
|
1647
|
+
and exc.error_code == ErrorCode.ENCRYPTION_FAILED
|
|
1648
|
+
):
|
|
1649
|
+
raise
|
|
1650
|
+
self._record_push_error(errors, change_type, component_id, config_id, exc)
|
|
1651
|
+
|
|
1652
|
+
# ---- Phase C: variable-link backfill (KFR-03) ----------------
|
|
1653
|
+
binding = resolve_variable_bindings(
|
|
1654
|
+
self,
|
|
1655
|
+
client,
|
|
1656
|
+
created_configs=created_configs,
|
|
1657
|
+
created_id_map=created_id_map,
|
|
1658
|
+
created_row_id_map=created_row_id_map,
|
|
1659
|
+
created_rows_by_parent=created_rows_by_parent,
|
|
1660
|
+
manifest=manifest,
|
|
1661
|
+
branch_id=branch_id,
|
|
1662
|
+
)
|
|
1663
|
+
errors.extend(binding.errors)
|
|
1664
|
+
if binding.configs_rewritten:
|
|
1665
|
+
manifest_dirty = True
|
|
1666
|
+
|
|
1667
|
+
# ---- Phase D: flow task configId backfill (#426) -------------
|
|
1668
|
+
# After variable links, remap keboola.flow task configIds that point
|
|
1669
|
+
# at configs created this push (golden/placeholder -> ULID). Reuses
|
|
1670
|
+
# created_id_map; a no-op when no flow was created.
|
|
1671
|
+
flow_binding = resolve_flow_task_bindings(
|
|
1672
|
+
self,
|
|
1673
|
+
client,
|
|
1674
|
+
created_configs=created_configs,
|
|
1675
|
+
created_id_map=created_id_map,
|
|
1676
|
+
manifest=manifest,
|
|
1677
|
+
branch_id=branch_id,
|
|
1678
|
+
)
|
|
1679
|
+
errors.extend(flow_binding.errors)
|
|
1680
|
+
if flow_binding.configs_rewritten:
|
|
1681
|
+
manifest_dirty = True
|
|
1682
|
+
|
|
1683
|
+
# Save manifest with updated hashes / new IDs / removed entries
|
|
1684
|
+
if manifest_dirty:
|
|
1685
|
+
save_manifest(project_root, manifest)
|
|
1686
|
+
|
|
1687
|
+
result_data: dict[str, Any] = {
|
|
1688
|
+
"status": "pushed",
|
|
1689
|
+
"created": created,
|
|
1690
|
+
"updated": updated,
|
|
1691
|
+
"deleted": deleted,
|
|
1692
|
+
"errors": errors,
|
|
1693
|
+
"pushed_details": pushed_details,
|
|
1694
|
+
}
|
|
1695
|
+
if flow_binding.tasks_remapped:
|
|
1696
|
+
result_data["flow_task_remaps"] = flow_binding.tasks_remapped
|
|
1697
|
+
if name_drift_warnings and not no_name_drift_warnings:
|
|
1698
|
+
result_data["name_drift_warnings"] = name_drift_warnings
|
|
1699
|
+
return result_data
|
|
1700
|
+
|
|
1701
|
+
def clone_project(
|
|
1702
|
+
self,
|
|
1703
|
+
source: str | Path,
|
|
1704
|
+
target_alias: str,
|
|
1705
|
+
target_dir: str | Path,
|
|
1706
|
+
*,
|
|
1707
|
+
overrides: dict[str, Any] | None = None,
|
|
1708
|
+
dry_run: bool = False,
|
|
1709
|
+
branch_override: int | None = None,
|
|
1710
|
+
) -> dict[str, Any]:
|
|
1711
|
+
"""Clone a reference synced project into a fresh target project (#426).
|
|
1712
|
+
|
|
1713
|
+
Thin delegator to :func:`._sync_clone.clone_project`; see that function
|
|
1714
|
+
(and ``sync-workflow.md``) for the full behavior -- copy + parameterize
|
|
1715
|
+
(bucket_map / variable_values / instance_rename) + push, with Phase-C/D
|
|
1716
|
+
link remap, a fresh-target guard, and idempotent re-runs.
|
|
1717
|
+
"""
|
|
1718
|
+
return _clone_project_impl(
|
|
1719
|
+
self,
|
|
1720
|
+
source,
|
|
1721
|
+
target_alias,
|
|
1722
|
+
target_dir,
|
|
1723
|
+
overrides=overrides,
|
|
1724
|
+
dry_run=dry_run,
|
|
1725
|
+
branch_override=branch_override,
|
|
1726
|
+
)
|
|
1727
|
+
|
|
1728
|
+
# Kept for test compatibility: tests exercise fail-closed encryption via
|
|
1729
|
+
# ``SyncService._encrypt_secrets_in_config(...)``. Production code uses
|
|
1730
|
+
# :func:`encrypt_secrets_in_config` directly.
|
|
1731
|
+
_encrypt_secrets_in_config = staticmethod(encrypt_secrets_in_config)
|
|
1732
|
+
|
|
1733
|
+
def _compute_config_hashes(self, config_dir: Path, component_id: str) -> LocalConfigHashes:
|
|
1734
|
+
"""Recompute the manifest bookkeeping hashes from a config dir on disk.
|
|
1735
|
+
|
|
1736
|
+
Reads the (post-writeback) ``_config.yml``, merges code files for the
|
|
1737
|
+
normalized config hash, and hashes each tracked companion file. Used
|
|
1738
|
+
after create / update / variable-link backfill so ``sync diff`` sees
|
|
1739
|
+
local == remote on the next run.
|
|
1740
|
+
"""
|
|
1741
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
1742
|
+
file_hash = self._file_hash(config_file) if config_file.exists() else ""
|
|
1743
|
+
local_data = self._read_config_file(config_dir)
|
|
1744
|
+
if local_data is not None:
|
|
1745
|
+
merge_code_files(component_id, local_data, config_dir)
|
|
1746
|
+
cfg_hash = config_hash(local_data)
|
|
1747
|
+
else:
|
|
1748
|
+
cfg_hash = ""
|
|
1749
|
+
extra_hashes: dict[str, str] = {}
|
|
1750
|
+
for fname in _EXTRA_HASH_FILENAMES:
|
|
1751
|
+
fpath = config_dir / fname
|
|
1752
|
+
if fpath.exists():
|
|
1753
|
+
extra_hashes[fname] = self._file_hash(fpath)
|
|
1754
|
+
return LocalConfigHashes(file_hash=file_hash, cfg_hash=cfg_hash, extra_hashes=extra_hashes)
|
|
1755
|
+
|
|
1756
|
+
def _record_push_error(
|
|
1757
|
+
self,
|
|
1758
|
+
errors: list[dict[str, str]],
|
|
1759
|
+
change_type: str,
|
|
1760
|
+
component_id: str,
|
|
1761
|
+
config_id: str,
|
|
1762
|
+
exc: Exception,
|
|
1763
|
+
) -> None:
|
|
1764
|
+
"""Log a non-fatal per-change push failure and accumulate it.
|
|
1765
|
+
|
|
1766
|
+
Callers must re-raise fail-closed encryption errors
|
|
1767
|
+
(:data:`ErrorCode.ENCRYPTION_FAILED`) *before* delegating here so a
|
|
1768
|
+
partial push never silently drops a secret-bearing change.
|
|
1769
|
+
"""
|
|
1770
|
+
logger.warning(
|
|
1771
|
+
"Failed to push %s %s/%s: %s",
|
|
1772
|
+
change_type,
|
|
1773
|
+
component_id,
|
|
1774
|
+
config_id,
|
|
1775
|
+
exc,
|
|
1776
|
+
)
|
|
1777
|
+
errors.append(
|
|
1778
|
+
{
|
|
1779
|
+
"change_type": change_type,
|
|
1780
|
+
"component_id": component_id,
|
|
1781
|
+
"config_id": config_id,
|
|
1782
|
+
"message": str(exc),
|
|
1783
|
+
}
|
|
1784
|
+
)
|
|
1785
|
+
|
|
1786
|
+
# ------------------------------------------------------------------
|
|
1787
|
+
# bulk operations (all projects)
|
|
1788
|
+
# ------------------------------------------------------------------
|
|
1789
|
+
|
|
1790
|
+
def pull_all(
|
|
1791
|
+
self,
|
|
1792
|
+
base_dir: Path,
|
|
1793
|
+
force: bool = False,
|
|
1794
|
+
dry_run: bool = False,
|
|
1795
|
+
job_limit: int = DEFAULT_JOBS_PER_CONFIG,
|
|
1796
|
+
no_storage: bool = False,
|
|
1797
|
+
no_jobs: bool = False,
|
|
1798
|
+
with_samples: bool = False,
|
|
1799
|
+
sample_limit: int = DEFAULT_SAMPLE_LIMIT,
|
|
1800
|
+
max_samples: int = DEFAULT_MAX_SAMPLES,
|
|
1801
|
+
) -> dict[str, Any]:
|
|
1802
|
+
"""Pull all registered projects in parallel (see ``_sync_bulk.pull_all``)."""
|
|
1803
|
+
return _bulk_pull_all(
|
|
1804
|
+
self,
|
|
1805
|
+
base_dir,
|
|
1806
|
+
force=force,
|
|
1807
|
+
dry_run=dry_run,
|
|
1808
|
+
job_limit=job_limit,
|
|
1809
|
+
no_storage=no_storage,
|
|
1810
|
+
no_jobs=no_jobs,
|
|
1811
|
+
with_samples=with_samples,
|
|
1812
|
+
sample_limit=sample_limit,
|
|
1813
|
+
max_samples=max_samples,
|
|
1814
|
+
)
|
|
1815
|
+
|
|
1816
|
+
def diff_all(self, base_dir: Path) -> dict[str, Any]:
|
|
1817
|
+
"""Diff all registered projects with a local manifest (see ``_sync_bulk.diff_all``)."""
|
|
1818
|
+
return _bulk_diff_all(self, base_dir)
|
|
1819
|
+
|
|
1820
|
+
def push_all(
|
|
1821
|
+
self,
|
|
1822
|
+
base_dir: Path,
|
|
1823
|
+
dry_run: bool = False,
|
|
1824
|
+
force: bool = False,
|
|
1825
|
+
allow_plaintext_fallback: bool = False,
|
|
1826
|
+
) -> dict[str, Any]:
|
|
1827
|
+
"""Push all registered projects with a local manifest (see ``_sync_bulk.push_all``)."""
|
|
1828
|
+
return _bulk_push_all(
|
|
1829
|
+
self,
|
|
1830
|
+
base_dir,
|
|
1831
|
+
dry_run=dry_run,
|
|
1832
|
+
force=force,
|
|
1833
|
+
allow_plaintext_fallback=allow_plaintext_fallback,
|
|
1834
|
+
)
|
|
1835
|
+
|
|
1836
|
+
# ------------------------------------------------------------------
|
|
1837
|
+
# branch mapping
|
|
1838
|
+
# ------------------------------------------------------------------
|
|
1839
|
+
|
|
1840
|
+
def branch_link(
|
|
1841
|
+
self,
|
|
1842
|
+
alias: str,
|
|
1843
|
+
project_root: Path,
|
|
1844
|
+
branch_id: int | None = None,
|
|
1845
|
+
branch_name: str | None = None,
|
|
1846
|
+
) -> dict[str, Any]:
|
|
1847
|
+
"""Link the current git branch to a Keboola dev branch (see ``_sync_branch``)."""
|
|
1848
|
+
return _branch_link(self, alias, project_root, branch_id, branch_name)
|
|
1849
|
+
|
|
1850
|
+
def branch_unlink(self, project_root: Path) -> dict[str, Any]:
|
|
1851
|
+
"""Remove the branch mapping for the current git branch (see ``_sync_branch``)."""
|
|
1852
|
+
return _branch_unlink(project_root)
|
|
1853
|
+
|
|
1854
|
+
def branch_status(self, project_root: Path) -> dict[str, Any]:
|
|
1855
|
+
"""Show the branch mapping status for the current git branch (see ``_sync_branch``)."""
|
|
1856
|
+
return _branch_status(project_root)
|
|
1857
|
+
|
|
1858
|
+
# ------------------------------------------------------------------
|
|
1859
|
+
# Private helpers
|
|
1860
|
+
# ------------------------------------------------------------------
|
|
1861
|
+
|
|
1862
|
+
@staticmethod
|
|
1863
|
+
def _resolve_branch_id(
|
|
1864
|
+
project: Any,
|
|
1865
|
+
manifest: "Manifest",
|
|
1866
|
+
project_root: Path,
|
|
1867
|
+
branch_override: int | None = None,
|
|
1868
|
+
) -> int | None:
|
|
1869
|
+
"""Resolve the Keboola branch ID for sync operations.
|
|
1870
|
+
|
|
1871
|
+
Priority:
|
|
1872
|
+
0. ``branch_override`` (CLI ``--branch <id>``) -- wins over everything
|
|
1873
|
+
1. Git-branching mode: read branch-mapping.json for current git branch
|
|
1874
|
+
2. ``active_branch_id`` from project config (``kbagent branch use``)
|
|
1875
|
+
3. First branch in manifest (production fallback)
|
|
1876
|
+
|
|
1877
|
+
Raises ``ConfigError`` if git-branching is enabled but the current
|
|
1878
|
+
branch is not the default and is not linked.
|
|
1879
|
+
|
|
1880
|
+
The default git branch always resolves to ``None`` (production), even
|
|
1881
|
+
if ``branch-mapping.json`` is missing or has no entry for it. This
|
|
1882
|
+
guarantees there is always a recovery path when the mapping file is
|
|
1883
|
+
lost (issue #267, Bug E).
|
|
1884
|
+
"""
|
|
1885
|
+
from ..sync.branch_mapping import load_branch_mapping
|
|
1886
|
+
from ..sync.git_utils import get_current_branch
|
|
1887
|
+
|
|
1888
|
+
# CLI override beats every persisted source so a user can target a
|
|
1889
|
+
# dev branch from a clean git workspace without first running
|
|
1890
|
+
# `branch use` or `branch-link`.
|
|
1891
|
+
if branch_override is not None:
|
|
1892
|
+
return branch_override
|
|
1893
|
+
|
|
1894
|
+
if manifest.git_branching.enabled:
|
|
1895
|
+
git_branch = get_current_branch(project_root)
|
|
1896
|
+
if git_branch:
|
|
1897
|
+
default_branch = manifest.git_branching.default_branch
|
|
1898
|
+
is_default = git_branch == default_branch
|
|
1899
|
+
try:
|
|
1900
|
+
mapping = load_branch_mapping(project_root)
|
|
1901
|
+
except FileNotFoundError:
|
|
1902
|
+
# Mapping missing -- auto-recover for the default branch
|
|
1903
|
+
# so the user is never locked out of production.
|
|
1904
|
+
if is_default:
|
|
1905
|
+
return None
|
|
1906
|
+
raise ConfigError(
|
|
1907
|
+
f"Git branch '{git_branch}' is not linked to a Keboola "
|
|
1908
|
+
f"branch (branch-mapping.json missing). "
|
|
1909
|
+
f"Run 'kbagent sync branch-link --project ALIAS' first."
|
|
1910
|
+
) from None
|
|
1911
|
+
entry = mapping.get(git_branch)
|
|
1912
|
+
if entry is not None:
|
|
1913
|
+
# entry.keboola_id is None for production (default branch)
|
|
1914
|
+
return entry.keboola_id
|
|
1915
|
+
# No entry for current branch -- default branch is always production
|
|
1916
|
+
if is_default:
|
|
1917
|
+
return None
|
|
1918
|
+
raise ConfigError(
|
|
1919
|
+
f"Git branch '{git_branch}' is not linked to a Keboola branch. "
|
|
1920
|
+
f"Run 'kbagent sync branch-link --project ALIAS' first."
|
|
1921
|
+
)
|
|
1922
|
+
|
|
1923
|
+
# Non git-branching: use active_branch_id or manifest fallback
|
|
1924
|
+
branch_id = project.active_branch_id if project is not None else None
|
|
1925
|
+
if not branch_id and manifest.branches:
|
|
1926
|
+
branch_id = manifest.branches[0].id
|
|
1927
|
+
return branch_id
|
|
1928
|
+
|
|
1929
|
+
# ------------------------------------------------------------------
|
|
1930
|
+
# Storage metadata / jobs / samples helpers
|
|
1931
|
+
# ------------------------------------------------------------------
|
|
1932
|
+
|
|
1933
|
+
def _write_config_file(self, config_dir: Path, config_data: dict[str, Any]) -> str:
|
|
1934
|
+
"""Write a ``_config.yml`` file and return its SHA256 hash.
|
|
1935
|
+
|
|
1936
|
+
Uses ``newline=""`` so Windows does NOT translate ``\\n`` into ``\\r\\n``
|
|
1937
|
+
on write -- without that, the in-memory ``content`` hash diverges from
|
|
1938
|
+
the on-disk byte hash (:meth:`_file_hash`) and every post-pull
|
|
1939
|
+
``status`` would report the file as modified.
|
|
1940
|
+
"""
|
|
1941
|
+
config_dir.mkdir(parents=True, exist_ok=True)
|
|
1942
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
1943
|
+
content = yaml.dump(
|
|
1944
|
+
config_data,
|
|
1945
|
+
default_flow_style=False,
|
|
1946
|
+
allow_unicode=True,
|
|
1947
|
+
sort_keys=False,
|
|
1948
|
+
width=120,
|
|
1949
|
+
)
|
|
1950
|
+
config_file.write_text(content, encoding="utf-8", newline="")
|
|
1951
|
+
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
1952
|
+
|
|
1953
|
+
def _file_hash(self, file_path: Path) -> str:
|
|
1954
|
+
"""Return the SHA256 hex digest of a file's contents."""
|
|
1955
|
+
content = file_path.read_bytes()
|
|
1956
|
+
return hashlib.sha256(content).hexdigest()
|
|
1957
|
+
|
|
1958
|
+
def _detect_name_drift(self, manifest: Manifest, project_root: Path) -> list[dict[str, str]]:
|
|
1959
|
+
"""Detect configs where local dir name doesn't match the config name.
|
|
1960
|
+
|
|
1961
|
+
Reads each tracked config's _config.yml to get the current name,
|
|
1962
|
+
then compares sanitize_name(name) against the directory basename.
|
|
1963
|
+
|
|
1964
|
+
Returns a list of warning dicts with component_id, config_id,
|
|
1965
|
+
local_dirname, and expected_dirname.
|
|
1966
|
+
"""
|
|
1967
|
+
warnings: list[dict[str, str]] = []
|
|
1968
|
+
for cfg in manifest.configurations:
|
|
1969
|
+
path = cfg.path
|
|
1970
|
+
dirname = path.rsplit("/", 1)[-1] if "/" in path else path
|
|
1971
|
+
|
|
1972
|
+
# Find branch dir and read _config.yml
|
|
1973
|
+
branch_path = self._find_branch_path(manifest, cfg.branch_id)
|
|
1974
|
+
config_dir = project_root / branch_path / path
|
|
1975
|
+
local_data = self._read_config_file(config_dir)
|
|
1976
|
+
if local_data is None:
|
|
1977
|
+
continue
|
|
1978
|
+
|
|
1979
|
+
config_name = local_data.get("name", "")
|
|
1980
|
+
if not config_name:
|
|
1981
|
+
continue
|
|
1982
|
+
|
|
1983
|
+
expected_dirname = sanitize_name(config_name)
|
|
1984
|
+
if dirname != expected_dirname:
|
|
1985
|
+
warnings.append(
|
|
1986
|
+
{
|
|
1987
|
+
"component_id": cfg.component_id,
|
|
1988
|
+
"config_id": cfg.id,
|
|
1989
|
+
"local_dirname": dirname,
|
|
1990
|
+
"expected_dirname": expected_dirname,
|
|
1991
|
+
"config_name": config_name,
|
|
1992
|
+
}
|
|
1993
|
+
)
|
|
1994
|
+
return warnings
|
|
1995
|
+
|
|
1996
|
+
def _rename_directory(self, source: Path, target: Path) -> str:
|
|
1997
|
+
"""Rename a directory, using git mv if in a git repo, else shutil.move.
|
|
1998
|
+
|
|
1999
|
+
Returns 'git_mv' or 'shutil_move' indicating which method was used.
|
|
2000
|
+
"""
|
|
2001
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
2002
|
+
try:
|
|
2003
|
+
result = subprocess.run(
|
|
2004
|
+
["git", "mv", str(source), str(target)],
|
|
2005
|
+
cwd=source.parent,
|
|
2006
|
+
capture_output=True,
|
|
2007
|
+
text=True,
|
|
2008
|
+
check=False,
|
|
2009
|
+
)
|
|
2010
|
+
if result.returncode == 0:
|
|
2011
|
+
return "git_mv"
|
|
2012
|
+
except FileNotFoundError:
|
|
2013
|
+
pass # git not installed
|
|
2014
|
+
shutil.move(str(source), str(target))
|
|
2015
|
+
return "shutil_move"
|
|
2016
|
+
|
|
2017
|
+
def _read_config_file(self, config_dir: Path) -> dict[str, Any] | None:
|
|
2018
|
+
"""Read and parse a ``_config.yml`` file, returning None if missing."""
|
|
2019
|
+
config_file = config_dir / CONFIG_FILENAME
|
|
2020
|
+
if not config_file.exists():
|
|
2021
|
+
return None
|
|
2022
|
+
try:
|
|
2023
|
+
return yaml.safe_load(config_file.read_text(encoding="utf-8"))
|
|
2024
|
+
except yaml.YAMLError:
|
|
2025
|
+
logger.warning("Failed to parse %s", config_file)
|
|
2026
|
+
return None
|
|
2027
|
+
|
|
2028
|
+
def _ensure_branch_registered(
|
|
2029
|
+
self,
|
|
2030
|
+
manifest: Manifest,
|
|
2031
|
+
branch_id: int | None,
|
|
2032
|
+
client: Any,
|
|
2033
|
+
) -> str | None:
|
|
2034
|
+
"""Ensure *branch_id* has an entry in ``manifest.branches``.
|
|
2035
|
+
|
|
2036
|
+
If *branch_id* is ``None`` (production) or already present, this is
|
|
2037
|
+
a no-op. Otherwise the branch name is fetched from the API and a
|
|
2038
|
+
new :class:`ManifestBranch` is appended.
|
|
2039
|
+
|
|
2040
|
+
Returns:
|
|
2041
|
+
The new branch path if one was added, ``None`` otherwise.
|
|
2042
|
+
"""
|
|
2043
|
+
if branch_id is None:
|
|
2044
|
+
return None
|
|
2045
|
+
|
|
2046
|
+
# Already registered?
|
|
2047
|
+
for branch in manifest.branches:
|
|
2048
|
+
if branch.id == branch_id:
|
|
2049
|
+
return None
|
|
2050
|
+
|
|
2051
|
+
# Fetch branch info from API to get a human-readable name
|
|
2052
|
+
all_branches = client.list_dev_branches()
|
|
2053
|
+
branch_name = ""
|
|
2054
|
+
for b in all_branches:
|
|
2055
|
+
if b.get("id") == branch_id:
|
|
2056
|
+
branch_name = b.get("name", "")
|
|
2057
|
+
break
|
|
2058
|
+
|
|
2059
|
+
# Generate filesystem-safe path
|
|
2060
|
+
path = sanitize_name(branch_name) if branch_name else ""
|
|
2061
|
+
if not path:
|
|
2062
|
+
path = f"branch-{branch_id}"
|
|
2063
|
+
|
|
2064
|
+
# Handle path uniqueness -- avoid collisions with existing entries
|
|
2065
|
+
existing_paths = {br.path for br in manifest.branches}
|
|
2066
|
+
if path in existing_paths:
|
|
2067
|
+
path = f"{path}-{branch_id}"
|
|
2068
|
+
|
|
2069
|
+
manifest.branches.append(ManifestBranch(id=branch_id, path=path))
|
|
2070
|
+
logger.info("Registered dev branch %d as '%s' in manifest", branch_id, path)
|
|
2071
|
+
return path
|
|
2072
|
+
|
|
2073
|
+
def _find_branch_path(self, manifest: Manifest, branch_id: int | None) -> str:
|
|
2074
|
+
"""Find the branch directory name for a given branch ID.
|
|
2075
|
+
|
|
2076
|
+
When ``branch_id`` is ``None`` (production / default branch),
|
|
2077
|
+
return the first branch path from the manifest.
|
|
2078
|
+
"""
|
|
2079
|
+
if branch_id is None:
|
|
2080
|
+
# Production -- use default branch path
|
|
2081
|
+
return manifest.branches[0].path if manifest.branches else "main"
|
|
2082
|
+
for branch in manifest.branches:
|
|
2083
|
+
if branch.id == branch_id:
|
|
2084
|
+
return branch.path
|
|
2085
|
+
# Fallback to default branch -- should not happen after _ensure_branch_registered
|
|
2086
|
+
logger.warning(
|
|
2087
|
+
"Branch ID %s not found in manifest, falling back to default path",
|
|
2088
|
+
branch_id,
|
|
2089
|
+
)
|
|
2090
|
+
return manifest.branches[0].path if manifest.branches else "main"
|
|
2091
|
+
|
|
2092
|
+
def _branch_path_has_configs(self, branch_dir: Path) -> bool:
|
|
2093
|
+
"""Return True if *branch_dir* exists and holds at least one config.
|
|
2094
|
+
|
|
2095
|
+
A config is any ``_config.yml`` below the branch root other than the
|
|
2096
|
+
optional branch-level ``_config.yml`` itself. Used to decide whether a
|
|
2097
|
+
target-branch subtree is materialized on disk.
|
|
2098
|
+
"""
|
|
2099
|
+
if not branch_dir.is_dir():
|
|
2100
|
+
return False
|
|
2101
|
+
for config_file in branch_dir.rglob(CONFIG_FILENAME):
|
|
2102
|
+
if config_file.parent != branch_dir:
|
|
2103
|
+
return True
|
|
2104
|
+
return False
|
|
2105
|
+
|
|
2106
|
+
def _resolve_source_branch_path(
|
|
2107
|
+
self,
|
|
2108
|
+
manifest: Manifest,
|
|
2109
|
+
project_root: Path,
|
|
2110
|
+
target_branch_id: int | None,
|
|
2111
|
+
) -> str:
|
|
2112
|
+
"""Resolve the on-disk branch subtree to read local configs from.
|
|
2113
|
+
|
|
2114
|
+
Source (where files live) and target (where the API writes) are
|
|
2115
|
+
decoupled. When the target branch has its own materialized
|
|
2116
|
+
``<branch_name>/`` subtree on disk, that subtree is the source
|
|
2117
|
+
(unchanged multi-branch-directory behaviour). Otherwise the default
|
|
2118
|
+
branch tree (``manifest.branches[0]``, i.e. ``main/``) is the source --
|
|
2119
|
+
the "promote the default tree to a target dev branch" path used by
|
|
2120
|
+
``sync push --branch <id>`` when no per-branch subtree exists
|
|
2121
|
+
(KFR-07 option-B).
|
|
2122
|
+
|
|
2123
|
+
API calls still target ``target_branch_id``; only the *read* path is
|
|
2124
|
+
affected.
|
|
2125
|
+
"""
|
|
2126
|
+
target_path = self._find_branch_path(manifest, target_branch_id)
|
|
2127
|
+
if self._branch_path_has_configs(project_root / target_path):
|
|
2128
|
+
return target_path
|
|
2129
|
+
default_path = manifest.branches[0].path if manifest.branches else target_path
|
|
2130
|
+
if default_path != target_path:
|
|
2131
|
+
logger.info(
|
|
2132
|
+
"No config files under target branch path '%s'; promoting default "
|
|
2133
|
+
"tree '%s' to branch %s",
|
|
2134
|
+
target_path,
|
|
2135
|
+
default_path,
|
|
2136
|
+
target_branch_id,
|
|
2137
|
+
)
|
|
2138
|
+
return default_path
|
|
2139
|
+
|
|
2140
|
+
def _find_untracked_configs(
|
|
2141
|
+
self,
|
|
2142
|
+
project_root: Path,
|
|
2143
|
+
manifest: Manifest,
|
|
2144
|
+
resolved_branch_id: int | None = None,
|
|
2145
|
+
) -> list[dict[str, str]]:
|
|
2146
|
+
"""Scan for _config.yml files that are not tracked in the manifest.
|
|
2147
|
+
|
|
2148
|
+
Scans branch directories that the user is actively working with:
|
|
2149
|
+
branches that already have tracked configs, the default branch
|
|
2150
|
+
(production), and the branch the caller resolved for the current
|
|
2151
|
+
operation (when provided). This supports the documented
|
|
2152
|
+
"scaffold locally then push" workflow on git-branching workspaces
|
|
2153
|
+
with empty ``manifest.configurations`` (issue #267, Bug B).
|
|
2154
|
+
|
|
2155
|
+
Branches outside this scope are skipped to avoid phantom "added"
|
|
2156
|
+
configs from orphaned dev-branch directories left over from
|
|
2157
|
+
previous work.
|
|
2158
|
+
"""
|
|
2159
|
+
tracked_paths: set[str] = set()
|
|
2160
|
+
in_scope_branch_ids: set[int] = set()
|
|
2161
|
+
for cfg in manifest.configurations:
|
|
2162
|
+
branch_path = self._find_branch_path(manifest, cfg.branch_id)
|
|
2163
|
+
tracked_paths.add(str(project_root / branch_path / cfg.path))
|
|
2164
|
+
in_scope_branch_ids.add(cfg.branch_id)
|
|
2165
|
+
|
|
2166
|
+
# Default branch is always in scope -- pushing a brand-new config
|
|
2167
|
+
# against production with empty configurations[] is a legitimate flow.
|
|
2168
|
+
if manifest.branches:
|
|
2169
|
+
in_scope_branch_ids.add(manifest.branches[0].id)
|
|
2170
|
+
|
|
2171
|
+
# The branch the caller resolved is in scope (linked feature branch
|
|
2172
|
+
# the user explicitly switched to via git checkout + branch-link).
|
|
2173
|
+
if resolved_branch_id is not None:
|
|
2174
|
+
in_scope_branch_ids.add(resolved_branch_id)
|
|
2175
|
+
|
|
2176
|
+
added: list[dict[str, str]] = []
|
|
2177
|
+
for branch in manifest.branches:
|
|
2178
|
+
if branch.id not in in_scope_branch_ids:
|
|
2179
|
+
continue
|
|
2180
|
+
branch_dir = project_root / branch.path
|
|
2181
|
+
if not branch_dir.exists():
|
|
2182
|
+
continue
|
|
2183
|
+
for config_file in branch_dir.rglob(CONFIG_FILENAME):
|
|
2184
|
+
config_dir = config_file.parent
|
|
2185
|
+
# Skip row-level configs (they're under rows/ subdirectory)
|
|
2186
|
+
if "rows" in config_dir.parts:
|
|
2187
|
+
continue
|
|
2188
|
+
# Skip branch-level _config.yml
|
|
2189
|
+
if config_dir == branch_dir:
|
|
2190
|
+
continue
|
|
2191
|
+
if str(config_dir) not in tracked_paths:
|
|
2192
|
+
local_data = self._read_config_file(config_dir)
|
|
2193
|
+
keboola_meta = local_data.get("_keboola", {}) if local_data else {}
|
|
2194
|
+
added.append(
|
|
2195
|
+
{
|
|
2196
|
+
"component_id": keboola_meta.get("component_id", "unknown"),
|
|
2197
|
+
"config_id": keboola_meta.get("config_id", ""),
|
|
2198
|
+
"path": str(config_dir.relative_to(project_root / branch.path)),
|
|
2199
|
+
}
|
|
2200
|
+
)
|
|
2201
|
+
|
|
2202
|
+
return added
|
|
2203
|
+
|
|
2204
|
+
def _find_untracked_rows(self, project_root: Path, manifest: Manifest) -> list[dict[str, Any]]:
|
|
2205
|
+
"""Scan tracked config dirs for ``rows/*/_config.yml`` not in manifest.
|
|
2206
|
+
|
|
2207
|
+
Paralleling :meth:`_find_untracked_configs` at the row level. A user
|
|
2208
|
+
can drop a hand-crafted row directory under a tracked config's
|
|
2209
|
+
``rows/`` folder; this surfaces it so :meth:`diff` can flag it as
|
|
2210
|
+
``"added"`` and :meth:`push` can POST it via ``create_config_row``.
|
|
2211
|
+
|
|
2212
|
+
Each entry contains ``component_id``, ``parent_config_id``,
|
|
2213
|
+
``row_name`` (from the loaded YAML), ``path`` (relative to the parent
|
|
2214
|
+
config dir, e.g. ``rows/new-row``), and ``data`` (the loaded dict).
|
|
2215
|
+
"""
|
|
2216
|
+
added: list[dict[str, Any]] = []
|
|
2217
|
+
for cfg in manifest.configurations:
|
|
2218
|
+
branch_path = self._find_branch_path(manifest, cfg.branch_id)
|
|
2219
|
+
parent_dir = project_root / branch_path / cfg.path
|
|
2220
|
+
rows_dir = parent_dir / "rows"
|
|
2221
|
+
if not rows_dir.is_dir():
|
|
2222
|
+
continue
|
|
2223
|
+
tracked_row_paths = {row.path for row in cfg.rows}
|
|
2224
|
+
for row_subdir in rows_dir.iterdir():
|
|
2225
|
+
if not row_subdir.is_dir():
|
|
2226
|
+
continue
|
|
2227
|
+
row_rel_path = f"rows/{row_subdir.name}"
|
|
2228
|
+
if row_rel_path in tracked_row_paths:
|
|
2229
|
+
continue
|
|
2230
|
+
if not (row_subdir / CONFIG_FILENAME).exists():
|
|
2231
|
+
continue
|
|
2232
|
+
local_data = self._read_config_file(row_subdir)
|
|
2233
|
+
if local_data is None:
|
|
2234
|
+
continue
|
|
2235
|
+
added.append(
|
|
2236
|
+
{
|
|
2237
|
+
"component_id": cfg.component_id,
|
|
2238
|
+
"parent_config_id": cfg.id,
|
|
2239
|
+
"row_name": local_data.get("name", ""),
|
|
2240
|
+
"path": row_rel_path,
|
|
2241
|
+
"data": local_data,
|
|
2242
|
+
}
|
|
2243
|
+
)
|
|
2244
|
+
return added
|