quantology-modelmaker 0.3.3__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. {quantology_modelmaker-0.3.3/quantology_modelmaker.egg-info → quantology_modelmaker-0.3.4}/PKG-INFO +5 -2
  2. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/README.md +4 -1
  3. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/build.py +14 -0
  4. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/catalogue.py +5 -9
  5. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/controller.py +57 -1
  6. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/hint_rules.py +14 -1
  7. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/hints.py +45 -0
  8. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/prompts.py +60 -2
  9. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/tools.py +4 -2
  10. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/api.py +0 -6
  11. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/assistant_api.py +1 -1
  12. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/base.py +15 -0
  13. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/library.py +91 -7
  14. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/cache.py +104 -6
  15. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/capabilities.py +3 -3
  16. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/datasets.py +0 -4
  17. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_api.py +2 -11
  18. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/prompts.py +6 -1
  19. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/run_worker.py +46 -9
  20. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/runner.py +42 -8
  21. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/accumulate.py +0 -18
  22. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/client.py +0 -6
  23. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/screens.py +0 -2
  24. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/util.py +0 -7
  25. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/pyproject.toml +1 -1
  26. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4/quantology_modelmaker.egg-info}/PKG-INFO +5 -2
  27. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/SOURCES.txt +1 -0
  28. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_build.py +31 -1
  29. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_hints.py +32 -0
  30. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_datasets.py +1 -1
  31. quantology_modelmaker-0.3.4/tests/test_large_data.py +207 -0
  32. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/LICENSE +0 -0
  33. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/MANIFEST.in +0 -0
  34. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/NOTICE +0 -0
  35. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/.gitignore +0 -0
  36. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/.npmrc +0 -0
  37. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/.oxlintrc.json +0 -0
  38. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/README.md +0 -0
  39. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/index.html +0 -0
  40. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/package-lock.json +0 -0
  41. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/package.json +0 -0
  42. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/public/brand-quantology/favicon.svg +0 -0
  43. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/public/brand-quantology/icon-mark-light.svg +0 -0
  44. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/public/icons.svg +0 -0
  45. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/scripts/check-host.mjs +0 -0
  46. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/App.tsx +0 -0
  47. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/BlockNode.tsx +0 -0
  48. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/BuildPanel.tsx +0 -0
  49. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/BuildReport.tsx +0 -0
  50. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/CodeEditor.tsx +0 -0
  51. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/DataModal.tsx +0 -0
  52. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/DataPicker.tsx +0 -0
  53. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/DataWireEdge.tsx +0 -0
  54. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/EnvVarField.tsx +0 -0
  55. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/FileBrowser.tsx +0 -0
  56. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/GhostNode.tsx +0 -0
  57. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/GitPanel.tsx +0 -0
  58. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/Inspector.tsx +0 -0
  59. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LaneBand.tsx +0 -0
  60. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LaneLabels.tsx +0 -0
  61. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LibraryPage.tsx +0 -0
  62. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LibraryPanel.tsx +0 -0
  63. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/NodeDisplay.tsx +0 -0
  64. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/Palette.tsx +0 -0
  65. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/ParamsForm.tsx +0 -0
  66. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/PortDataView.tsx +0 -0
  67. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/PortInspector.tsx +0 -0
  68. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/SettingsPanel.tsx +0 -0
  69. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/Toolbar.tsx +0 -0
  70. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/UpdateDialog.tsx +0 -0
  71. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/api.ts +0 -0
  72. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/assets/hero.png +0 -0
  73. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/assets/react.svg +0 -0
  74. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/assets/vite.svg +0 -0
  75. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/features.ts +0 -0
  76. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/host.ts +0 -0
  77. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/index.css +0 -0
  78. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/main.tsx +0 -0
  79. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/roles.ts +0 -0
  80. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/types.ts +0 -0
  81. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/tsconfig.app.json +0 -0
  82. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/tsconfig.json +0 -0
  83. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/tsconfig.node.json +0 -0
  84. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/vite.config.ts +0 -0
  85. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/__init__.py +0 -0
  86. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/__main__.py +0 -0
  87. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/__init__.py +0 -0
  88. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/layout.py +0 -0
  89. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/leakage.py +0 -0
  90. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/loop.py +0 -0
  91. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/mcp_server.py +0 -0
  92. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/assistant_mcp.py +0 -0
  93. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/block_library.py +0 -0
  94. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/__init__.py +0 -0
  95. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/_template.py +0 -0
  96. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/binning.py +0 -0
  97. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/data_quality.py +0 -0
  98. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/feature_analysis.py +0 -0
  99. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/modelling.py +0 -0
  100. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/stat_tests.py +0 -0
  101. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/stochastic.py +0 -0
  102. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/cli.py +0 -0
  103. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/compiler.py +0 -0
  104. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/demo_data.py +0 -0
  105. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/__init__.py +0 -0
  106. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/agent.py +0 -0
  107. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/base.py +0 -0
  108. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/env.py +0 -0
  109. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/limits.py +0 -0
  110. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/local.py +0 -0
  111. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/protocol.py +0 -0
  112. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/remote.py +0 -0
  113. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/safe_pickle.py +0 -0
  114. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/wire.py +0 -0
  115. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/features.py +0 -0
  116. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/forms.py +0 -0
  117. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/gitops.py +0 -0
  118. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/graph.py +0 -0
  119. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_ai.py +0 -0
  120. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_store.py +0 -0
  121. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_updates.py +0 -0
  122. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/__init__.py +0 -0
  123. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/anthropic_provider.py +0 -0
  124. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/base.py +0 -0
  125. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/claude_cli_provider.py +0 -0
  126. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/copilot_provider.py +0 -0
  127. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/gemini_provider.py +0 -0
  128. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/lmstudio_provider.py +0 -0
  129. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/openai_provider.py +0 -0
  130. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/redact.py +0 -0
  131. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/settings.py +0 -0
  132. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/stub_provider.py +0 -0
  133. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/metadata_transforms.py +0 -0
  134. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/packet.py +0 -0
  135. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/project.py +0 -0
  136. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/runslot.py +0 -0
  137. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/session.py +0 -0
  138. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/signature.py +0 -0
  139. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/sources.py +0 -0
  140. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/__init__.py +0 -0
  141. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/credit.py +0 -0
  142. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/dependency.py +0 -0
  143. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/distributions.py +0 -0
  144. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/proxy.py +0 -0
  145. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/seed.py +0 -0
  146. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/var_covar.py +0 -0
  147. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/__init__.py +0 -0
  148. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/app.py +0 -0
  149. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/canvas.py +0 -0
  150. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/charts.py +0 -0
  151. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/inspector.py +0 -0
  152. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/paramspecs.py +0 -0
  153. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/preview.py +0 -0
  154. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/worker_pool.py +0 -0
  155. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/workspace.py +0 -0
  156. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/projects/demo_pd_model/model.json +0 -0
  157. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/dependency_links.txt +0 -0
  158. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/entry_points.txt +0 -0
  159. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/requires.txt +0 -0
  160. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/top_level.txt +0 -0
  161. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/run.bat +0 -0
  162. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/setup.cfg +0 -0
  163. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/setup.py +0 -0
  164. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_api.py +0 -0
  165. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_foundations.py +0 -0
  166. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_leakage.py +0 -0
  167. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_loops.py +0 -0
  168. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_api.py +0 -0
  169. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_assistant_mcp.py +0 -0
  170. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_binning.py +0 -0
  171. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_block_library.py +0 -0
  172. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_cache.py +0 -0
  173. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_compiler.py +0 -0
  174. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_data_quality.py +0 -0
  175. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_demo_data.py +0 -0
  176. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_demo_project.py +0 -0
  177. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_display_blocks.py +0 -0
  178. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_embedding.py +0 -0
  179. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_env_vars.py +0 -0
  180. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_agent.py +0 -0
  181. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_executor.py +0 -0
  182. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_limits.py +0 -0
  183. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_security.py +0 -0
  184. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_wire.py +0 -0
  185. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_feature_analysis.py +0 -0
  186. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_features.py +0 -0
  187. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_forms.py +0 -0
  188. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_gitops.py +0 -0
  189. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_graph.py +0 -0
  190. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_hosting_hooks.py +0 -0
  191. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_input_blocks.py +0 -0
  192. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_library.py +0 -0
  193. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_llm.py +0 -0
  194. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_llm_connections.py +0 -0
  195. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_master_scale.py +0 -0
  196. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_modelling.py +0 -0
  197. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_no_exec_in_api.py +0 -0
  198. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_packet.py +0 -0
  199. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_prep_blocks.py +0 -0
  200. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_project.py +0 -0
  201. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_risk_parameter_blocks.py +0 -0
  202. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_runner.py +0 -0
  203. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_runner_streaming.py +0 -0
  204. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_session.py +0 -0
  205. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stat_tests.py +0 -0
  206. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_blocks.py +0 -0
  207. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_core.py +0 -0
  208. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_credit.py +0 -0
  209. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_fan_out.py +0 -0
  210. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_runner.py +0 -0
  211. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_tui_app.py +0 -0
  212. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_tui_charts.py +0 -0
  213. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_tui_client.py +0 -0
  214. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_validation_blocks.py +0 -0
  215. {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_workspaces.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quantology-modelmaker
3
- Version: 0.3.3
3
+ Version: 0.3.4
4
4
  Summary: Visual pipeline builder for financial model development
5
5
  Author-email: Maurits Malkus <10948172+mmalkus@users.noreply.github.com>
6
6
  License-Expression: Apache-2.0
@@ -197,7 +197,10 @@ then run `modelmaker-api` (or `modelmaker-tui`) from inside that folder.
197
197
  A few things worth knowing beyond the canvas itself.
198
198
 
199
199
  - **Sample mode** — the `Sample` toggle in the top bar caps *every* source at
200
- the first N rows, so a whole pipeline stays fast to iterate on. It changes
200
+ N rows, so a whole pipeline stays fast to iterate on. A source with a
201
+ column tagged as the ID (on the input block itself) keeps whole IDs
202
+ instead of the first N rows: about N rows, every row of each kept ID, so
203
+ a panel's histories stay complete and the sample spans the whole file. It changes
201
204
  every block's cache key, so sampled and full results never mix; a banner
202
205
  keeps it obvious that what you're looking at isn't the full population.
203
206
  Switching back off needs the sources re-read.
@@ -145,7 +145,10 @@ then run `modelmaker-api` (or `modelmaker-tui`) from inside that folder.
145
145
  A few things worth knowing beyond the canvas itself.
146
146
 
147
147
  - **Sample mode** — the `Sample` toggle in the top bar caps *every* source at
148
- the first N rows, so a whole pipeline stays fast to iterate on. It changes
148
+ N rows, so a whole pipeline stays fast to iterate on. A source with a
149
+ column tagged as the ID (on the input block itself) keeps whole IDs
150
+ instead of the first N rows: about N rows, every row of each kept ID, so
151
+ a panel's histories stay complete and the sample spans the whole file. It changes
149
152
  every block's cache key, so sampled and full results never mix; a banner
150
153
  keeps it obvious that what you're looking at isn't the full population.
151
154
  Switching back off needs the sources re-read.
@@ -200,6 +200,14 @@ class AgentBuild:
200
200
  self.owned_lanes: set[str] = set()
201
201
  self.lane_map: dict[str, str] = {} # plan lane key / name asked for -> lane id
202
202
  self.sample_rows_used: int | None = None
203
+ # The anchors' size on the full data (rows, estimated bytes) when
204
+ # known -- preflight reads it before the build turns sample mode on
205
+ # -- and how much bigger the full data is than the sample the build
206
+ # runs on (full rows / sampled rows at the anchors; 1.0 on the full
207
+ # data, None when unknown). For large-data guidance and size hints.
208
+ self.full_rows: int | None = None
209
+ self.full_bytes: int | None = None
210
+ self.sample_scale: float | None = 1.0
203
211
  self.changed_sample_mode = False # the build turned sample mode on (and must turn it off)
204
212
  self.finished = False # the model called finish
205
213
  # Set by a successful terminal tool (submit_plan, ask_user,
@@ -323,6 +331,9 @@ class AgentBuild:
323
331
  "owned_lanes": sorted(self.owned_lanes),
324
332
  "lane_map": self.lane_map,
325
333
  "sample_rows_used": self.sample_rows_used,
334
+ "full_rows": self.full_rows,
335
+ "full_bytes": self.full_bytes,
336
+ "sample_scale": self.sample_scale,
326
337
  "counters": self.counters,
327
338
  "usage": self.usage,
328
339
  "error": self.error,
@@ -380,6 +391,9 @@ class AgentBuild:
380
391
  done = [i for i, st in enumerate(b.stages) if st["status"] in (STAGE_DONE, STAGE_SKIPPED)]
381
392
  b.stage_index = min(max(done, default=-1) + 1, max(len(b.stages) - 1, 0))
382
393
  b.sample_rows_used = record.get("sample_rows_used")
394
+ b.full_rows = record.get("full_rows")
395
+ b.full_bytes = record.get("full_bytes")
396
+ b.sample_scale = record.get("sample_scale", 1.0 if not b.sample_rows_used else None)
383
397
  b.counters.update({k: v for k, v in (record.get("counters") or {}).items() if k in b.counters})
384
398
  b.usage.update(record.get("usage") or {})
385
399
  b._events = list(record.get("events") or [])
@@ -10,8 +10,7 @@ from typing import Any
10
10
 
11
11
  from .. import capabilities
12
12
  from ..block_library import INJECTED_PARAMS, load_builtin
13
- from ..blocks.base import BLOCK_REGISTRY, BlockDefinition
14
- from ..packet import ColumnRole
13
+ from ..blocks.base import BLOCK_REGISTRY, BlockDefinition, streaming_note
15
14
  from ..signature import ParamSpec
16
15
  from ..util import ROLE_PARAM_NAMES, find_role_param
17
16
 
@@ -32,6 +31,7 @@ AGENT_DISALLOWED: dict[str, str] = {
32
31
  "read_dataset": "input blocks are prepared by the user, not the AI build",
33
32
  "read_source": "input blocks are prepared by the user, not the AI build",
34
33
  "write_csv": "writing files is left to the user",
34
+ "write_parquet": "writing files is left to the user",
35
35
  "export": "exporting results is left to the user",
36
36
  "iterate": "fan-out (iterate/collect) isn't supported in AI builds yet",
37
37
  "collect": "fan-out (iterate/collect) isn't supported in AI builds yet",
@@ -247,6 +247,9 @@ def catalogue_entry(spec: BlockDefinition, detail: bool = False) -> dict[str, An
247
247
  if detail:
248
248
  entry["doc"] = doc
249
249
  entry["params"] = block_params(spec)
250
+ # What it costs on large data: a block that doesn't stream holds its
251
+ # whole input and output tables in memory on the full run.
252
+ entry["streaming"] = streaming_note(spec) or "no: holds its whole input and output tables in memory"
250
253
  return entry
251
254
 
252
255
 
@@ -286,10 +289,3 @@ def describe_block_type(category: str, include_library: bool = True) -> dict[str
286
289
  if spec is None:
287
290
  raise KeyError(f"unknown block category {category!r}")
288
291
  return catalogue_entry(spec, detail=True)
289
-
290
-
291
- ROLE_NOTES = {
292
- ColumnRole.TARGET.value: "the modelling target; params named target/target_col auto-fill from it",
293
- ColumnRole.PREDICTED.value: "set automatically on model outputs; params named score_col/predicted_col auto-fill from it",
294
- ColumnRole.EXCLUDED.value: "must never be used as a model feature",
295
- }
@@ -120,9 +120,51 @@ def run_preflight(build: AgentBuild) -> dict[str, Any]:
120
120
  ):
121
121
  warnings.append(_issue("no_date", "The goal mentions out-of-time/vintages but no column is tagged date."))
122
122
  max_rows = max((f.data.height for f in frames), default=None)
123
+ if runner.sample_rows is None and frames:
124
+ # The full data: what the build's large-data guidance and size
125
+ # hints scale a sample's numbers up to (see AgentBuild.full_rows).
126
+ build.full_rows = max_rows
127
+ build.full_bytes = max(f.data.estimated_size() for f in frames)
128
+ warnings += _large_data_issues(build, frames, max_rows)
123
129
  return {"blocking": blocking, "warnings": warnings, "max_rows": max_rows}
124
130
 
125
131
 
132
+ def _large_data_issues(build: AgentBuild, frames: list[DataFramePacket], max_rows: int | None) -> list[dict[str, Any]]:
133
+ """Preflight warnings for a source big enough to build on a sample:
134
+ without an ID tagged on the input block itself, sample mode cuts the
135
+ first N rows -- for a panel sorted by id and date, a few ids' histories
136
+ and nothing else."""
137
+ big = (
138
+ (max_rows or 0) > AUTO_SAMPLE_THRESHOLD_ROWS
139
+ or bool(build.options.sample_rows)
140
+ or build.session.runner.sample_rows is not None
141
+ )
142
+ if not big:
143
+ return []
144
+ from ..runner import sample_key
145
+
146
+ graph = build.session.graph
147
+ scope = build.scope_block_ids()
148
+ inputs = [bid for bid in graph.topo_order() if bid in scope and graph.blocks[bid].block_type == "input"]
149
+ if not inputs or any(sample_key(graph.blocks[bid]) for bid in inputs):
150
+ return []
151
+ tagged = sorted({n for f in frames for n, m in f.schema_meta.items() if m.role == ColumnRole.ID})
152
+ where = (
153
+ f"{tagged[0]} is tagged id downstream, but not on the input block, where sample mode reads it"
154
+ if tagged
155
+ else "No column is tagged id on the input block"
156
+ )
157
+ return [
158
+ _issue(
159
+ "sample_without_id",
160
+ f"{where}. The data is large, so the build runs on a sample: with the id (loan, account) tagged on "
161
+ "the input block, the sample keeps every row of a random set of ids -- complete histories from "
162
+ "across the file -- instead of the file's first rows.",
163
+ inputs,
164
+ )
165
+ ]
166
+
167
+
126
168
  def _record_resolved_model(choice: LLMChoice, loop: AgentLoop) -> None:
127
169
  """When no model was chosen, the loop picks one (a provider default, or
128
170
  whatever a local server has loaded) -- record which, so provenance and
@@ -583,14 +625,28 @@ class BuildController:
583
625
  current = session.runner.sample_rows
584
626
  if not requested or current is not None:
585
627
  # Off, or the user already has sample mode on -- build on
586
- # whatever they're looking at and leave the setting alone.
628
+ # whatever they're looking at and leave the setting alone. A
629
+ # sample the user turned on hides the full size: unknown.
587
630
  b.sample_rows_used = current
631
+ b.sample_scale = 1.0 if current is None else None
588
632
  return
589
633
  session.set_sample_rows(requested)
590
634
  b.sample_rows_used = requested
591
635
  b.changed_sample_mode = True
592
636
  b.log("sample_mode", rows=requested)
593
637
  self._reread_inputs()
638
+ sampled = self._anchor_rows()
639
+ b.sample_scale = b.full_rows / sampled if b.full_rows and sampled else None
640
+
641
+ def _anchor_rows(self) -> int | None:
642
+ """The anchors' largest current output, in rows."""
643
+ runner = self.session.runner
644
+ rows = []
645
+ for a in self.build.anchors:
646
+ st = runner.state.get(a)
647
+ entry = runner.cache.get(st.last_successful_key) if st and st.last_successful_key else None
648
+ rows += [v.data.height for v in (entry.outputs.values() if entry else []) if isinstance(v, DataFramePacket)]
649
+ return max(rows, default=None)
594
650
 
595
651
  def _read_stale_inputs(self) -> None:
596
652
  """(Re)read the in-scope input blocks that aren't current. Caller
@@ -12,7 +12,7 @@ from __future__ import annotations
12
12
 
13
13
  from typing import Any, Callable
14
14
 
15
-
15
+ import polars as pl
16
16
 
17
17
  from ..packet import DataFramePacket
18
18
  from . import hints as h
@@ -47,6 +47,19 @@ def _split_hints(outputs: dict[str, Any]) -> list[str]:
47
47
  for port, packet in outputs.items():
48
48
  if isinstance(packet, DataFramePacket) and packet.data.height == 0:
49
49
  hints.append(f"{port} is empty -- check the split settings (test_size, cutoff, oot_end).")
50
+ # A random row split of a panel (several rows per id) puts rows of one
51
+ # id on both sides. Only for train/test: an out-of-time split of a
52
+ # panel is meant to see the same ids again, later.
53
+ train, test = outputs.get("train"), outputs.get("test")
54
+ key = h.id_column(train)
55
+ if key is not None and isinstance(test, DataFramePacket) and key in test.data.columns:
56
+ shared = train.data.select(pl.col(key).unique()).filter(pl.col(key).is_in(test.data[key].implode())).height
57
+ if shared:
58
+ hints.append(
59
+ f"{shared:,} {key} values have rows in both train and test -- the data has several rows per "
60
+ f"{key} (a panel), so a random row split leaks: test isn't unseen. Split by time "
61
+ "(time_split), or derive a per-id split flag and filter on it."
62
+ )
50
63
  thin = [f"{p} has {ev}" for p, (_, _, ev) in samples.items() if ev is not None and ev < h.MIN_EVENTS]
51
64
  if thin:
52
65
  hints.append(f"Few events (< {h.MIN_EVENTS}): {h.names(thin)} -- metrics on it will be noisy.")
@@ -36,6 +36,11 @@ MIN_EVENTS = 50 # defaults a sample needs for a stable validation
36
36
  SPLIT_RATE_GAP = 0.2 # relative event-rate gap between two samples
37
37
  MAX_NULL_SHARE = 0.0 # computed LGD/CCF nulls tolerated before a fit
38
38
 
39
+ # Large data. A table past either size, on the full data, is worth reducing
40
+ # before the next block that doesn't stream (it holds the whole table).
41
+ LARGE_TABLE_ROWS = 10_000_000
42
+ LARGE_TABLE_BYTES = 2 * 1024**3
43
+
39
44
  # Feature screens. fit_binning/iv_table IV bands (see binning._iv_band):
40
45
  # below USELESS_IV a feature carries no signal; at or above SUSPICIOUS_IV
41
46
  # it usually leaks the target.
@@ -108,6 +113,46 @@ def target_column(packet: Any) -> str | None:
108
113
  )
109
114
 
110
115
 
116
+ def id_column(packet: Any) -> str | None:
117
+ """The column tagged as the ID, when the packet has one."""
118
+ if not isinstance(packet, DataFramePacket):
119
+ return None
120
+ return next(
121
+ (n for n, m in packet.schema_meta.items() if m.role == ColumnRole.ID and n in packet.data.columns), None
122
+ )
123
+
124
+
125
+ def human_bytes(n: float) -> str:
126
+ for unit in ("bytes", "KB", "MB", "GB"):
127
+ if n < 1024 or unit == "GB":
128
+ return f"{n:,.0f} {unit}" if unit == "bytes" else f"{n:,.1f} {unit}"
129
+ n /= 1024
130
+ return f"{n:,.1f} GB"
131
+
132
+
133
+ def size_hints(outputs: dict[str, Any], scale: float | None) -> list[str]:
134
+ """Any block's dataframe outputs that are large on the full data.
135
+ `scale` turns a size seen on a sample into the full data's (full rows /
136
+ sampled rows at the source); 1 on the full data; None when the build
137
+ runs on a sample whose full size isn't known -- then only what's seen."""
138
+ hints = []
139
+ for port, packet in outputs.items():
140
+ if not isinstance(packet, DataFramePacket):
141
+ continue
142
+ rows, size = packet.data.height, packet.data.estimated_size()
143
+ k = scale or 1.0
144
+ if rows * k < LARGE_TABLE_ROWS and size * k < LARGE_TABLE_BYTES:
145
+ continue
146
+ where = "on the full data, about " if scale and scale > 1 else ""
147
+ hints.append(
148
+ f"{port} is {where}{rows * k:,.0f} rows, {human_bytes(size * k)} in memory -- every block after it that "
149
+ "doesn't stream (describe_block_type: streaming) holds that whole table. Reduce it next with "
150
+ "streaming blocks: select only the columns the model needs, filter to the population, and "
151
+ "group_by/join down to one row per id, snapshot or cell -- before splitting or fitting."
152
+ )
153
+ return hints
154
+
155
+
111
156
  def iv_rows(rows: list[dict[str, Any]], keep_monotonic: bool) -> list[str]:
112
157
  """fit_binning's summary / iv_table's rows: which features to keep,
113
158
  drop or question."""
@@ -65,6 +65,13 @@ stratify_col for a low default rate, derive_columns for ratios/flags and \
65
65
  one_hot_encode for categorical regressors rather than custom code.
66
66
  - For a PD target from a monthly account panel with no outcome flag yet, \
67
67
  use forward_default_flag (tags the flag as the target) rather than derive_columns.
68
+ - Panel data (several rows per id, e.g. one per loan per month): derive \
69
+ anything that looks ahead or back per id (forward_default_flag, lags) on the \
70
+ complete monthly history first, then keep one observation per id per outcome \
71
+ window (every 12th month for a 12-month default flag -- overlapping windows \
72
+ repeat the same outcome and add rows, not information). Split it by time \
73
+ (time_split): train_test_split splits rows, so one id lands in both train \
74
+ and test.
68
75
  - For LGD: discount_recoveries -> compute_lgd (tags lgd as the target); for \
69
76
  CCF: compute_ccf -> lgd_regression -> compute_ead. Report compare_samples \
70
77
  (train/test/OOT side by side) and, for a rating scale, grade_backtest."""
@@ -83,7 +90,12 @@ expressions (select/filter/with_columns/group_by/join), select only the \
83
90
  columns it needs, and reduce early (to one row per account, snapshot date or \
84
91
  period); if it must collect a LazyFrame, do so only after reducing, with \
85
92
  .collect(engine="streaming"). Never .to_pandas(), .iter_rows(), \
86
- .map_elements() or Python loops over rows.
93
+ .map_elements() or Python loops over rows. If the block is only polars \
94
+ expressions, mark its dataframe inputs "lazy": true so it streams with its \
95
+ neighbours. Per-id logic on a panel uses .over(id) or a group_by on the id \
96
+ joined back -- never a sort-and-loop. Downcast wide numeric outputs \
97
+ (Float64 -> Float32, Int64 -> Int32) and use Categorical for repeated text \
98
+ when a table stays large.
87
99
  - Never hardcode an existing input column's name in the body: take it as a \
88
100
  `<descriptive>_col` str parameter defaulted to the real column name (or \
89
101
  `<descriptive>_cols` for a list) and use pl.col(param). New columns you \
@@ -192,8 +204,52 @@ If the goal needs something no registry block does, say so -- in \
192
204
  `questions` while planning, with ask_user while building. """
193
205
 
194
206
 
207
+ LARGE_DATA = """## Large data
208
+ The data is large ({size}). You build on a sample; the finished graph runs \
209
+ on all of it, and there every block that doesn't stream (describe_block_type \
210
+ shows `streaming`) holds its whole input and output tables in memory. Design \
211
+ for that:
212
+ - Reduce first, straight after the anchors, with blocks that stream \
213
+ (select, filter, derive_columns, groupby_agg, join): only the columns the \
214
+ model needs, only the population in scope, and down to the grain the model \
215
+ needs (one row per id, per snapshot, or per cell) -- before any split, fit, \
216
+ binning or custom block. A run_to result whose `next` says a table is large \
217
+ is the signal to reduce before going on.
218
+ - Don't fan a full-size table out to several branches; reduce once, then \
219
+ branch.
220
+ - If the model needs the full panel (e.g. a lifetime PD on loan-years) and \
221
+ it would still be too big, say so and ask_user: the options are a sample of \
222
+ ids (whole histories, weighted back if stratified), or aggregating to cells \
223
+ of banded drivers with event and exposure counts.
224
+ - In your report, say which block holds the largest table on the full data."""
225
+
226
+
227
+ def _large_data_note(build: AgentBuild) -> str | None:
228
+ """The large-data section, for a build whose data is big enough to
229
+ run on a sample (see controller._maybe_sample), else None."""
230
+ from ..cache import memory_budget_bytes
231
+ from .controller import AUTO_SAMPLE_THRESHOLD_ROWS
232
+ from .hints import human_bytes
233
+
234
+ rows = build.full_rows
235
+ if not (rows and rows > AUTO_SAMPLE_THRESHOLD_ROWS) and not build.sample_rows_used:
236
+ return None
237
+ if rows:
238
+ size = f"{rows:,} rows at the anchors"
239
+ if build.full_bytes:
240
+ size += f", {human_bytes(build.full_bytes)} in memory"
241
+ else:
242
+ size = "its full size isn't known: the build started on the user's sample"
243
+ budget = memory_budget_bytes()
244
+ if budget:
245
+ size += f"; the app keeps up to {human_bytes(budget)} of results in memory"
246
+ return LARGE_DATA.format(size=size)
247
+
248
+
195
249
  def _common(build: AgentBuild) -> str:
196
- return COMMON.replace("{custom_rule}", CUSTOM_ON if build.options.allow_new_blocks else CUSTOM_OFF)
250
+ common = COMMON.replace("{custom_rule}", CUSTOM_ON if build.options.allow_new_blocks else CUSTOM_OFF)
251
+ note = _large_data_note(build)
252
+ return common if note is None else f"{common}\n\n{note}"
197
253
 
198
254
 
199
255
  def _custom_contract(build: AgentBuild) -> list[str]:
@@ -292,6 +348,8 @@ def build_system(build: AgentBuild, compact: bool = False) -> str:
292
348
  outline)."""
293
349
  if build.sample_rows_used:
294
350
  sample_note = f"a {build.sample_rows_used:,}-row sample of the data"
351
+ if build.full_rows:
352
+ sample_note += f" (the full data has {build.full_rows:,} rows)"
295
353
  else:
296
354
  sample_note = "the full data"
297
355
  return "\n\n".join(
@@ -21,7 +21,7 @@ from ..runslot import RunBusy, RunFailed
21
21
  from ..session import new_id, wire_is_valid
22
22
  from . import catalogue, leakage
23
23
  from .build import AWAITING_APPROVAL, AWAITING_INPUT, BUILDING, PLANNING, STAGE_ACTIVE, STAGE_DONE, STAGE_SKIPPED, AgentBuild, ToolError
24
- from .hints import decision_hints
24
+ from .hints import decision_hints, size_hints
25
25
  from .layout import Placer, next_lane_order
26
26
 
27
27
  READ = frozenset({PLANNING, BUILDING})
@@ -267,7 +267,9 @@ def block_hints(b: AgentBuild, block_id: str) -> list[str]:
267
267
  outputs[p.name] = b.current_output(block_id, p.name)[1]
268
268
  except ToolError:
269
269
  continue
270
- return decision_hints(blk.resolve(), outputs)
270
+ # Size applies to every block: a large table is worth reducing before
271
+ # whatever comes next, whichever block made it.
272
+ return decision_hints(blk.resolve(), outputs) + size_hints(outputs, b.sample_scale)
271
273
 
272
274
 
273
275
  def suspicious_features(b: AgentBuild) -> dict[str, float | None]:
@@ -280,12 +280,6 @@ CLAUDE_CLI_KNOWN_MODELS = [
280
280
  # ---- schemas -----------------------------------------------------------
281
281
 
282
282
 
283
- class PortSpecOut(BaseModel):
284
- name: str
285
- type: str
286
- required: bool = True
287
-
288
-
289
283
  class BlockCreate(BaseModel):
290
284
  # A built-in's id; ignored when `library` is given.
291
285
  category: str = ""
@@ -106,7 +106,7 @@ IO_TAGS = {
106
106
  "exports": ("Write results out to a file or the workspace's exports.", "output"),
107
107
  }
108
108
  IO_BLOCKS = {"input": ("read_csv", "read_parquet", "read_json", "read_excel", "read_sql", "read_dataset", "read_source", "demo_credit_data"),
109
- "output": ("write_csv", "export")}
109
+ "output": ("write_csv", "write_parquet", "export")}
110
110
 
111
111
 
112
112
  def _io_blocks(kind: str) -> list[dict[str, Any]]:
@@ -233,3 +233,18 @@ class BlockDefinition:
233
233
  # Built-in blocks by id, filled by block_library.load_builtin() at package
234
234
  # import (see modelmaker/__init__.py).
235
235
  BLOCK_REGISTRY: dict[str, BlockDefinition] = {}
236
+
237
+
238
+ def streaming_note(d: BlockDefinition) -> str | None:
239
+ """How a block takes part in a streaming run, from its port flags (see
240
+ runner._fusable_block / _sink_block), or None when it doesn't: it then
241
+ holds its whole input and output tables in memory."""
242
+ df_inputs = [p for p in d.inputs if p.type == "dataframe"]
243
+ single_df_out = len(d.outputs) == 1 and d.outputs[0].type == "dataframe"
244
+ if d.block_type == "input" and single_df_out and d.outputs[0].lazy:
245
+ return "fuses: its lazy scan streams into the blocks after it"
246
+ if single_df_out and df_inputs and len(df_inputs) == len(d.inputs) and all(p.lazy for p in df_inputs):
247
+ return "fuses with neighbouring lazy blocks in a streaming run"
248
+ if d.block_type == "output" and not d.outputs and len(d.inputs) == 1 and d.inputs[0].lazy:
249
+ return "streams its input straight to its destination (a sink)"
250
+ return None
@@ -19,7 +19,21 @@ BLOCKS = {
19
19
  "block_type": "input",
20
20
  "inputs": [],
21
21
  "outputs": [{"name": "out", "lazy": True}],
22
- "form": [{"key": "path", "label": "File path", "kind": "path", "options": [".csv"]}],
22
+ "form": [
23
+ {"key": "path", "label": "File path", "kind": "path", "options": [".csv", ".txt"]},
24
+ {"key": "separator", "label": "Separator (e.g. , or | or \\t)", "kind": "text", "placeholder": ","},
25
+ {"key": "has_header", "label": "First row is a header", "kind": "checkbox"},
26
+ {
27
+ "key": "column_names",
28
+ "label": "Column names (comma-separated; optional, replaces the header or column_1, column_2, ...)",
29
+ "kind": "text",
30
+ },
31
+ {
32
+ "key": "infer_schema_rows",
33
+ "label": "Rows to infer dtypes from (0 = read every column as text)",
34
+ "kind": "number",
35
+ },
36
+ ],
23
37
  "meta": {"inherit": "none"},
24
38
  },
25
39
  "demo_credit_data": {
@@ -249,6 +263,30 @@ BLOCKS = {
249
263
  ],
250
264
  "meta": {"inherit": "none"},
251
265
  },
266
+ "write_parquet": {
267
+ "capabilities": ["filesystem"],
268
+ "use_instead": "export",
269
+ "display_name": "Write Parquet",
270
+ "version": 1,
271
+ "block_type": "output",
272
+ "inputs": [{"name": "df", "lazy": True}],
273
+ "outputs": [],
274
+ "form": [
275
+ {
276
+ "key": "filename",
277
+ "label": "Output filename",
278
+ "kind": "text",
279
+ "placeholder": "output.parquet",
280
+ },
281
+ {
282
+ "key": "compression",
283
+ "label": "Compression",
284
+ "kind": "select",
285
+ "options": ["zstd", "snappy", "lz4", "gzip", "uncompressed"],
286
+ },
287
+ ],
288
+ "meta": {"inherit": "none"},
289
+ },
252
290
  "export": {
253
291
  "display_name": "Export",
254
292
  "version": 1,
@@ -340,15 +378,39 @@ BLOCKS = {
340
378
  }
341
379
 
342
380
 
343
- def read_csv(path: str) -> pl.LazyFrame:
344
- """Reads a CSV file into a dataframe on the `out` port; no inputs.
345
- `path` is the file path (relative to the working directory the engine
346
- runs in). Column dtypes are inferred by polars; column roles start
347
- untagged."""
381
+ def read_csv(
382
+ path: str,
383
+ separator: str = ",",
384
+ has_header: bool = True,
385
+ column_names: str = "",
386
+ infer_schema_rows: int = 100,
387
+ ) -> pl.LazyFrame:
388
+ """Reads a delimited text file into a dataframe on the `out` port; no
389
+ inputs. `path` is the file path (relative to the working directory the
390
+ engine runs in). `separator` is one character ("," by default; "|" for
391
+ pipe-delimited files, "\\t" or "tab" for tab). `has_header` says whether
392
+ the first row holds column names; without one the columns are named
393
+ column_1, column_2, ... unless `column_names` (comma-separated) names
394
+ them -- given with a header, it replaces the header's names.
395
+ `infer_schema_rows` is how many rows polars reads to infer dtypes
396
+ (default 100); 0 reads every column as text, the safe choice for a
397
+ large file whose first rows aren't representative (cast afterwards).
398
+ Column roles start untagged."""
399
+ if separator in ("\\t", "tab", "TAB"):
400
+ separator = "\t"
401
+ if len(separator) != 1:
402
+ raise ValueError(f"separator must be a single character, got {separator!r}")
403
+ names = [n.strip() for n in column_names.split(",") if n.strip()] if column_names else None
348
404
  # A lazy scan: the engine collects it, pushing sample mode's row cap
349
405
  # down into the scan first, and a streaming run fuses it with whatever
350
406
  # follows (see the output port's "lazy" flag).
351
- return pl.scan_csv(path)
407
+ return pl.scan_csv(
408
+ path,
409
+ separator=separator,
410
+ has_header=has_header,
411
+ new_columns=names,
412
+ infer_schema_length=infer_schema_rows,
413
+ )
352
414
 
353
415
 
354
416
  def demo_credit_data(n_rows: int = 5000, seed: int = 42) -> pl.DataFrame:
@@ -622,6 +684,28 @@ def write_csv(df: pl.DataFrame, filename: str, output_dir: str = ".") -> None:
622
684
  df.lazy().sink_csv(os.path.join(output_dir, filename))
623
685
 
624
686
 
687
+ PARQUET_COMPRESSIONS = ("zstd", "snappy", "lz4", "gzip", "uncompressed")
688
+
689
+
690
+ def write_parquet(df: pl.DataFrame, filename: str, compression: str = "zstd", output_dir: str = ".") -> None:
691
+ """Writes the `df` input to a Parquet file; no output ports. `filename`
692
+ is the file name (e.g. "loans.parquet"), written inside `output_dir`,
693
+ which the engine injects as the run's output directory -- leave it
694
+ unset. `compression` is "zstd" (default), "snappy", "lz4", "gzip" or
695
+ "uncompressed". Overwrites any existing file of the same name. In a
696
+ streaming run (Run all (streaming)) the data goes straight from its
697
+ source to disk without ever being held in memory whole -- the way to
698
+ convert a large CSV to Parquet once, so later runs read only the
699
+ columns they use."""
700
+ import os
701
+
702
+ if compression not in PARQUET_COMPRESSIONS:
703
+ raise ValueError(f"compression must be one of {', '.join(PARQUET_COMPRESSIONS)}, got {compression!r}")
704
+ # As write_csv: .lazy() is a no-op on a LazyFrame, so a streaming run
705
+ # hands this its upstream's uncollected plan and it sinks to disk.
706
+ df.lazy().sink_parquet(os.path.join(output_dir, filename), compression=compression)
707
+
708
+
625
709
  EXPORT_FORMATS = ("csv", "parquet")
626
710
 
627
711