quantology-modelmaker 0.3.3__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantology_modelmaker-0.3.3/quantology_modelmaker.egg-info → quantology_modelmaker-0.3.4}/PKG-INFO +5 -2
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/README.md +4 -1
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/build.py +14 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/catalogue.py +5 -9
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/controller.py +57 -1
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/hint_rules.py +14 -1
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/hints.py +45 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/prompts.py +60 -2
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/tools.py +4 -2
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/api.py +0 -6
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/assistant_api.py +1 -1
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/base.py +15 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/library.py +91 -7
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/cache.py +104 -6
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/capabilities.py +3 -3
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/datasets.py +0 -4
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_api.py +2 -11
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/prompts.py +6 -1
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/run_worker.py +46 -9
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/runner.py +42 -8
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/accumulate.py +0 -18
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/client.py +0 -6
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/screens.py +0 -2
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/util.py +0 -7
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/pyproject.toml +1 -1
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4/quantology_modelmaker.egg-info}/PKG-INFO +5 -2
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/SOURCES.txt +1 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_build.py +31 -1
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_hints.py +32 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_datasets.py +1 -1
- quantology_modelmaker-0.3.4/tests/test_large_data.py +207 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/LICENSE +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/MANIFEST.in +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/NOTICE +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/.gitignore +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/.npmrc +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/.oxlintrc.json +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/README.md +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/index.html +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/package-lock.json +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/package.json +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/public/brand-quantology/favicon.svg +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/public/brand-quantology/icon-mark-light.svg +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/public/icons.svg +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/scripts/check-host.mjs +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/App.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/BlockNode.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/BuildPanel.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/BuildReport.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/CodeEditor.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/DataModal.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/DataPicker.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/DataWireEdge.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/EnvVarField.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/FileBrowser.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/GhostNode.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/GitPanel.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/Inspector.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LaneBand.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LaneLabels.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LibraryPage.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/LibraryPanel.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/NodeDisplay.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/Palette.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/ParamsForm.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/PortDataView.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/PortInspector.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/SettingsPanel.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/Toolbar.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/UpdateDialog.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/api.ts +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/assets/hero.png +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/assets/react.svg +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/assets/vite.svg +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/features.ts +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/host.ts +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/index.css +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/main.tsx +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/roles.ts +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/src/types.ts +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/tsconfig.app.json +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/tsconfig.json +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/tsconfig.node.json +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/frontend/vite.config.ts +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/__init__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/__main__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/__init__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/layout.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/leakage.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/loop.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/agent/mcp_server.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/assistant_mcp.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/block_library.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/__init__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/_template.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/binning.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/data_quality.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/feature_analysis.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/modelling.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/stat_tests.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/blocks/stochastic.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/cli.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/compiler.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/demo_data.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/__init__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/agent.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/base.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/env.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/limits.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/local.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/protocol.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/remote.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/safe_pickle.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/execution/wire.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/features.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/forms.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/gitops.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/graph.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_ai.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_store.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/library_updates.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/__init__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/anthropic_provider.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/base.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/claude_cli_provider.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/copilot_provider.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/gemini_provider.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/lmstudio_provider.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/openai_provider.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/redact.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/settings.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/llm/stub_provider.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/metadata_transforms.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/packet.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/project.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/runslot.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/session.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/signature.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/sources.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/__init__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/credit.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/dependency.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/distributions.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/proxy.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/seed.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/stochastic/var_covar.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/__init__.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/app.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/canvas.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/charts.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/inspector.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/paramspecs.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/tui/preview.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/worker_pool.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/modelmaker/workspace.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/projects/demo_pd_model/model.json +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/dependency_links.txt +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/entry_points.txt +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/requires.txt +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/quantology_modelmaker.egg-info/top_level.txt +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/run.bat +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/setup.cfg +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/setup.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_api.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_foundations.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_leakage.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_agent_loops.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_api.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_assistant_mcp.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_binning.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_block_library.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_cache.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_compiler.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_data_quality.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_demo_data.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_demo_project.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_display_blocks.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_embedding.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_env_vars.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_agent.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_executor.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_limits.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_security.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_execution_wire.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_feature_analysis.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_features.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_forms.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_gitops.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_graph.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_hosting_hooks.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_input_blocks.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_library.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_llm.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_llm_connections.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_master_scale.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_modelling.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_no_exec_in_api.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_packet.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_prep_blocks.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_project.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_risk_parameter_blocks.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_runner.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_runner_streaming.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_session.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stat_tests.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_blocks.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_core.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_credit.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_fan_out.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_stochastic_runner.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_tui_app.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_tui_charts.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_tui_client.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_validation_blocks.py +0 -0
- {quantology_modelmaker-0.3.3 → quantology_modelmaker-0.3.4}/tests/test_workspaces.py +0 -0
{quantology_modelmaker-0.3.3/quantology_modelmaker.egg-info → quantology_modelmaker-0.3.4}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quantology-modelmaker
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Visual pipeline builder for financial model development
|
|
5
5
|
Author-email: Maurits Malkus <10948172+mmalkus@users.noreply.github.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -197,7 +197,10 @@ then run `modelmaker-api` (or `modelmaker-tui`) from inside that folder.
|
|
|
197
197
|
A few things worth knowing beyond the canvas itself.
|
|
198
198
|
|
|
199
199
|
- **Sample mode** — the `Sample` toggle in the top bar caps *every* source at
|
|
200
|
-
|
|
200
|
+
N rows, so a whole pipeline stays fast to iterate on. A source with a
|
|
201
|
+
column tagged as the ID (on the input block itself) keeps whole IDs
|
|
202
|
+
instead of the first N rows: about N rows, every row of each kept ID, so
|
|
203
|
+
a panel's histories stay complete and the sample spans the whole file. It changes
|
|
201
204
|
every block's cache key, so sampled and full results never mix; a banner
|
|
202
205
|
keeps it obvious that what you're looking at isn't the full population.
|
|
203
206
|
Switching back off needs the sources re-read.
|
|
@@ -145,7 +145,10 @@ then run `modelmaker-api` (or `modelmaker-tui`) from inside that folder.
|
|
|
145
145
|
A few things worth knowing beyond the canvas itself.
|
|
146
146
|
|
|
147
147
|
- **Sample mode** — the `Sample` toggle in the top bar caps *every* source at
|
|
148
|
-
|
|
148
|
+
N rows, so a whole pipeline stays fast to iterate on. A source with a
|
|
149
|
+
column tagged as the ID (on the input block itself) keeps whole IDs
|
|
150
|
+
instead of the first N rows: about N rows, every row of each kept ID, so
|
|
151
|
+
a panel's histories stay complete and the sample spans the whole file. It changes
|
|
149
152
|
every block's cache key, so sampled and full results never mix; a banner
|
|
150
153
|
keeps it obvious that what you're looking at isn't the full population.
|
|
151
154
|
Switching back off needs the sources re-read.
|
|
@@ -200,6 +200,14 @@ class AgentBuild:
|
|
|
200
200
|
self.owned_lanes: set[str] = set()
|
|
201
201
|
self.lane_map: dict[str, str] = {} # plan lane key / name asked for -> lane id
|
|
202
202
|
self.sample_rows_used: int | None = None
|
|
203
|
+
# The anchors' size on the full data (rows, estimated bytes) when
|
|
204
|
+
# known -- preflight reads it before the build turns sample mode on
|
|
205
|
+
# -- and how much bigger the full data is than the sample the build
|
|
206
|
+
# runs on (full rows / sampled rows at the anchors; 1.0 on the full
|
|
207
|
+
# data, None when unknown). For large-data guidance and size hints.
|
|
208
|
+
self.full_rows: int | None = None
|
|
209
|
+
self.full_bytes: int | None = None
|
|
210
|
+
self.sample_scale: float | None = 1.0
|
|
203
211
|
self.changed_sample_mode = False # the build turned sample mode on (and must turn it off)
|
|
204
212
|
self.finished = False # the model called finish
|
|
205
213
|
# Set by a successful terminal tool (submit_plan, ask_user,
|
|
@@ -323,6 +331,9 @@ class AgentBuild:
|
|
|
323
331
|
"owned_lanes": sorted(self.owned_lanes),
|
|
324
332
|
"lane_map": self.lane_map,
|
|
325
333
|
"sample_rows_used": self.sample_rows_used,
|
|
334
|
+
"full_rows": self.full_rows,
|
|
335
|
+
"full_bytes": self.full_bytes,
|
|
336
|
+
"sample_scale": self.sample_scale,
|
|
326
337
|
"counters": self.counters,
|
|
327
338
|
"usage": self.usage,
|
|
328
339
|
"error": self.error,
|
|
@@ -380,6 +391,9 @@ class AgentBuild:
|
|
|
380
391
|
done = [i for i, st in enumerate(b.stages) if st["status"] in (STAGE_DONE, STAGE_SKIPPED)]
|
|
381
392
|
b.stage_index = min(max(done, default=-1) + 1, max(len(b.stages) - 1, 0))
|
|
382
393
|
b.sample_rows_used = record.get("sample_rows_used")
|
|
394
|
+
b.full_rows = record.get("full_rows")
|
|
395
|
+
b.full_bytes = record.get("full_bytes")
|
|
396
|
+
b.sample_scale = record.get("sample_scale", 1.0 if not b.sample_rows_used else None)
|
|
383
397
|
b.counters.update({k: v for k, v in (record.get("counters") or {}).items() if k in b.counters})
|
|
384
398
|
b.usage.update(record.get("usage") or {})
|
|
385
399
|
b._events = list(record.get("events") or [])
|
|
@@ -10,8 +10,7 @@ from typing import Any
|
|
|
10
10
|
|
|
11
11
|
from .. import capabilities
|
|
12
12
|
from ..block_library import INJECTED_PARAMS, load_builtin
|
|
13
|
-
from ..blocks.base import BLOCK_REGISTRY, BlockDefinition
|
|
14
|
-
from ..packet import ColumnRole
|
|
13
|
+
from ..blocks.base import BLOCK_REGISTRY, BlockDefinition, streaming_note
|
|
15
14
|
from ..signature import ParamSpec
|
|
16
15
|
from ..util import ROLE_PARAM_NAMES, find_role_param
|
|
17
16
|
|
|
@@ -32,6 +31,7 @@ AGENT_DISALLOWED: dict[str, str] = {
|
|
|
32
31
|
"read_dataset": "input blocks are prepared by the user, not the AI build",
|
|
33
32
|
"read_source": "input blocks are prepared by the user, not the AI build",
|
|
34
33
|
"write_csv": "writing files is left to the user",
|
|
34
|
+
"write_parquet": "writing files is left to the user",
|
|
35
35
|
"export": "exporting results is left to the user",
|
|
36
36
|
"iterate": "fan-out (iterate/collect) isn't supported in AI builds yet",
|
|
37
37
|
"collect": "fan-out (iterate/collect) isn't supported in AI builds yet",
|
|
@@ -247,6 +247,9 @@ def catalogue_entry(spec: BlockDefinition, detail: bool = False) -> dict[str, An
|
|
|
247
247
|
if detail:
|
|
248
248
|
entry["doc"] = doc
|
|
249
249
|
entry["params"] = block_params(spec)
|
|
250
|
+
# What it costs on large data: a block that doesn't stream holds its
|
|
251
|
+
# whole input and output tables in memory on the full run.
|
|
252
|
+
entry["streaming"] = streaming_note(spec) or "no: holds its whole input and output tables in memory"
|
|
250
253
|
return entry
|
|
251
254
|
|
|
252
255
|
|
|
@@ -286,10 +289,3 @@ def describe_block_type(category: str, include_library: bool = True) -> dict[str
|
|
|
286
289
|
if spec is None:
|
|
287
290
|
raise KeyError(f"unknown block category {category!r}")
|
|
288
291
|
return catalogue_entry(spec, detail=True)
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
ROLE_NOTES = {
|
|
292
|
-
ColumnRole.TARGET.value: "the modelling target; params named target/target_col auto-fill from it",
|
|
293
|
-
ColumnRole.PREDICTED.value: "set automatically on model outputs; params named score_col/predicted_col auto-fill from it",
|
|
294
|
-
ColumnRole.EXCLUDED.value: "must never be used as a model feature",
|
|
295
|
-
}
|
|
@@ -120,9 +120,51 @@ def run_preflight(build: AgentBuild) -> dict[str, Any]:
|
|
|
120
120
|
):
|
|
121
121
|
warnings.append(_issue("no_date", "The goal mentions out-of-time/vintages but no column is tagged date."))
|
|
122
122
|
max_rows = max((f.data.height for f in frames), default=None)
|
|
123
|
+
if runner.sample_rows is None and frames:
|
|
124
|
+
# The full data: what the build's large-data guidance and size
|
|
125
|
+
# hints scale a sample's numbers up to (see AgentBuild.full_rows).
|
|
126
|
+
build.full_rows = max_rows
|
|
127
|
+
build.full_bytes = max(f.data.estimated_size() for f in frames)
|
|
128
|
+
warnings += _large_data_issues(build, frames, max_rows)
|
|
123
129
|
return {"blocking": blocking, "warnings": warnings, "max_rows": max_rows}
|
|
124
130
|
|
|
125
131
|
|
|
132
|
+
def _large_data_issues(build: AgentBuild, frames: list[DataFramePacket], max_rows: int | None) -> list[dict[str, Any]]:
|
|
133
|
+
"""Preflight warnings for a source big enough to build on a sample:
|
|
134
|
+
without an ID tagged on the input block itself, sample mode cuts the
|
|
135
|
+
first N rows -- for a panel sorted by id and date, a few ids' histories
|
|
136
|
+
and nothing else."""
|
|
137
|
+
big = (
|
|
138
|
+
(max_rows or 0) > AUTO_SAMPLE_THRESHOLD_ROWS
|
|
139
|
+
or bool(build.options.sample_rows)
|
|
140
|
+
or build.session.runner.sample_rows is not None
|
|
141
|
+
)
|
|
142
|
+
if not big:
|
|
143
|
+
return []
|
|
144
|
+
from ..runner import sample_key
|
|
145
|
+
|
|
146
|
+
graph = build.session.graph
|
|
147
|
+
scope = build.scope_block_ids()
|
|
148
|
+
inputs = [bid for bid in graph.topo_order() if bid in scope and graph.blocks[bid].block_type == "input"]
|
|
149
|
+
if not inputs or any(sample_key(graph.blocks[bid]) for bid in inputs):
|
|
150
|
+
return []
|
|
151
|
+
tagged = sorted({n for f in frames for n, m in f.schema_meta.items() if m.role == ColumnRole.ID})
|
|
152
|
+
where = (
|
|
153
|
+
f"{tagged[0]} is tagged id downstream, but not on the input block, where sample mode reads it"
|
|
154
|
+
if tagged
|
|
155
|
+
else "No column is tagged id on the input block"
|
|
156
|
+
)
|
|
157
|
+
return [
|
|
158
|
+
_issue(
|
|
159
|
+
"sample_without_id",
|
|
160
|
+
f"{where}. The data is large, so the build runs on a sample: with the id (loan, account) tagged on "
|
|
161
|
+
"the input block, the sample keeps every row of a random set of ids -- complete histories from "
|
|
162
|
+
"across the file -- instead of the file's first rows.",
|
|
163
|
+
inputs,
|
|
164
|
+
)
|
|
165
|
+
]
|
|
166
|
+
|
|
167
|
+
|
|
126
168
|
def _record_resolved_model(choice: LLMChoice, loop: AgentLoop) -> None:
|
|
127
169
|
"""When no model was chosen, the loop picks one (a provider default, or
|
|
128
170
|
whatever a local server has loaded) -- record which, so provenance and
|
|
@@ -583,14 +625,28 @@ class BuildController:
|
|
|
583
625
|
current = session.runner.sample_rows
|
|
584
626
|
if not requested or current is not None:
|
|
585
627
|
# Off, or the user already has sample mode on -- build on
|
|
586
|
-
# whatever they're looking at and leave the setting alone.
|
|
628
|
+
# whatever they're looking at and leave the setting alone. A
|
|
629
|
+
# sample the user turned on hides the full size: unknown.
|
|
587
630
|
b.sample_rows_used = current
|
|
631
|
+
b.sample_scale = 1.0 if current is None else None
|
|
588
632
|
return
|
|
589
633
|
session.set_sample_rows(requested)
|
|
590
634
|
b.sample_rows_used = requested
|
|
591
635
|
b.changed_sample_mode = True
|
|
592
636
|
b.log("sample_mode", rows=requested)
|
|
593
637
|
self._reread_inputs()
|
|
638
|
+
sampled = self._anchor_rows()
|
|
639
|
+
b.sample_scale = b.full_rows / sampled if b.full_rows and sampled else None
|
|
640
|
+
|
|
641
|
+
def _anchor_rows(self) -> int | None:
|
|
642
|
+
"""The anchors' largest current output, in rows."""
|
|
643
|
+
runner = self.session.runner
|
|
644
|
+
rows = []
|
|
645
|
+
for a in self.build.anchors:
|
|
646
|
+
st = runner.state.get(a)
|
|
647
|
+
entry = runner.cache.get(st.last_successful_key) if st and st.last_successful_key else None
|
|
648
|
+
rows += [v.data.height for v in (entry.outputs.values() if entry else []) if isinstance(v, DataFramePacket)]
|
|
649
|
+
return max(rows, default=None)
|
|
594
650
|
|
|
595
651
|
def _read_stale_inputs(self) -> None:
|
|
596
652
|
"""(Re)read the in-scope input blocks that aren't current. Caller
|
|
@@ -12,7 +12,7 @@ from __future__ import annotations
|
|
|
12
12
|
|
|
13
13
|
from typing import Any, Callable
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
import polars as pl
|
|
16
16
|
|
|
17
17
|
from ..packet import DataFramePacket
|
|
18
18
|
from . import hints as h
|
|
@@ -47,6 +47,19 @@ def _split_hints(outputs: dict[str, Any]) -> list[str]:
|
|
|
47
47
|
for port, packet in outputs.items():
|
|
48
48
|
if isinstance(packet, DataFramePacket) and packet.data.height == 0:
|
|
49
49
|
hints.append(f"{port} is empty -- check the split settings (test_size, cutoff, oot_end).")
|
|
50
|
+
# A random row split of a panel (several rows per id) puts rows of one
|
|
51
|
+
# id on both sides. Only for train/test: an out-of-time split of a
|
|
52
|
+
# panel is meant to see the same ids again, later.
|
|
53
|
+
train, test = outputs.get("train"), outputs.get("test")
|
|
54
|
+
key = h.id_column(train)
|
|
55
|
+
if key is not None and isinstance(test, DataFramePacket) and key in test.data.columns:
|
|
56
|
+
shared = train.data.select(pl.col(key).unique()).filter(pl.col(key).is_in(test.data[key].implode())).height
|
|
57
|
+
if shared:
|
|
58
|
+
hints.append(
|
|
59
|
+
f"{shared:,} {key} values have rows in both train and test -- the data has several rows per "
|
|
60
|
+
f"{key} (a panel), so a random row split leaks: test isn't unseen. Split by time "
|
|
61
|
+
"(time_split), or derive a per-id split flag and filter on it."
|
|
62
|
+
)
|
|
50
63
|
thin = [f"{p} has {ev}" for p, (_, _, ev) in samples.items() if ev is not None and ev < h.MIN_EVENTS]
|
|
51
64
|
if thin:
|
|
52
65
|
hints.append(f"Few events (< {h.MIN_EVENTS}): {h.names(thin)} -- metrics on it will be noisy.")
|
|
@@ -36,6 +36,11 @@ MIN_EVENTS = 50 # defaults a sample needs for a stable validation
|
|
|
36
36
|
SPLIT_RATE_GAP = 0.2 # relative event-rate gap between two samples
|
|
37
37
|
MAX_NULL_SHARE = 0.0 # computed LGD/CCF nulls tolerated before a fit
|
|
38
38
|
|
|
39
|
+
# Large data. A table past either size, on the full data, is worth reducing
|
|
40
|
+
# before the next block that doesn't stream (it holds the whole table).
|
|
41
|
+
LARGE_TABLE_ROWS = 10_000_000
|
|
42
|
+
LARGE_TABLE_BYTES = 2 * 1024**3
|
|
43
|
+
|
|
39
44
|
# Feature screens. fit_binning/iv_table IV bands (see binning._iv_band):
|
|
40
45
|
# below USELESS_IV a feature carries no signal; at or above SUSPICIOUS_IV
|
|
41
46
|
# it usually leaks the target.
|
|
@@ -108,6 +113,46 @@ def target_column(packet: Any) -> str | None:
|
|
|
108
113
|
)
|
|
109
114
|
|
|
110
115
|
|
|
116
|
+
def id_column(packet: Any) -> str | None:
|
|
117
|
+
"""The column tagged as the ID, when the packet has one."""
|
|
118
|
+
if not isinstance(packet, DataFramePacket):
|
|
119
|
+
return None
|
|
120
|
+
return next(
|
|
121
|
+
(n for n, m in packet.schema_meta.items() if m.role == ColumnRole.ID and n in packet.data.columns), None
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def human_bytes(n: float) -> str:
|
|
126
|
+
for unit in ("bytes", "KB", "MB", "GB"):
|
|
127
|
+
if n < 1024 or unit == "GB":
|
|
128
|
+
return f"{n:,.0f} {unit}" if unit == "bytes" else f"{n:,.1f} {unit}"
|
|
129
|
+
n /= 1024
|
|
130
|
+
return f"{n:,.1f} GB"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def size_hints(outputs: dict[str, Any], scale: float | None) -> list[str]:
|
|
134
|
+
"""Any block's dataframe outputs that are large on the full data.
|
|
135
|
+
`scale` turns a size seen on a sample into the full data's (full rows /
|
|
136
|
+
sampled rows at the source); 1 on the full data; None when the build
|
|
137
|
+
runs on a sample whose full size isn't known -- then only what's seen."""
|
|
138
|
+
hints = []
|
|
139
|
+
for port, packet in outputs.items():
|
|
140
|
+
if not isinstance(packet, DataFramePacket):
|
|
141
|
+
continue
|
|
142
|
+
rows, size = packet.data.height, packet.data.estimated_size()
|
|
143
|
+
k = scale or 1.0
|
|
144
|
+
if rows * k < LARGE_TABLE_ROWS and size * k < LARGE_TABLE_BYTES:
|
|
145
|
+
continue
|
|
146
|
+
where = "on the full data, about " if scale and scale > 1 else ""
|
|
147
|
+
hints.append(
|
|
148
|
+
f"{port} is {where}{rows * k:,.0f} rows, {human_bytes(size * k)} in memory -- every block after it that "
|
|
149
|
+
"doesn't stream (describe_block_type: streaming) holds that whole table. Reduce it next with "
|
|
150
|
+
"streaming blocks: select only the columns the model needs, filter to the population, and "
|
|
151
|
+
"group_by/join down to one row per id, snapshot or cell -- before splitting or fitting."
|
|
152
|
+
)
|
|
153
|
+
return hints
|
|
154
|
+
|
|
155
|
+
|
|
111
156
|
def iv_rows(rows: list[dict[str, Any]], keep_monotonic: bool) -> list[str]:
|
|
112
157
|
"""fit_binning's summary / iv_table's rows: which features to keep,
|
|
113
158
|
drop or question."""
|
|
@@ -65,6 +65,13 @@ stratify_col for a low default rate, derive_columns for ratios/flags and \
|
|
|
65
65
|
one_hot_encode for categorical regressors rather than custom code.
|
|
66
66
|
- For a PD target from a monthly account panel with no outcome flag yet, \
|
|
67
67
|
use forward_default_flag (tags the flag as the target) rather than derive_columns.
|
|
68
|
+
- Panel data (several rows per id, e.g. one per loan per month): derive \
|
|
69
|
+
anything that looks ahead or back per id (forward_default_flag, lags) on the \
|
|
70
|
+
complete monthly history first, then keep one observation per id per outcome \
|
|
71
|
+
window (every 12th month for a 12-month default flag -- overlapping windows \
|
|
72
|
+
repeat the same outcome and add rows, not information). Split it by time \
|
|
73
|
+
(time_split): train_test_split splits rows, so one id lands in both train \
|
|
74
|
+
and test.
|
|
68
75
|
- For LGD: discount_recoveries -> compute_lgd (tags lgd as the target); for \
|
|
69
76
|
CCF: compute_ccf -> lgd_regression -> compute_ead. Report compare_samples \
|
|
70
77
|
(train/test/OOT side by side) and, for a rating scale, grade_backtest."""
|
|
@@ -83,7 +90,12 @@ expressions (select/filter/with_columns/group_by/join), select only the \
|
|
|
83
90
|
columns it needs, and reduce early (to one row per account, snapshot date or \
|
|
84
91
|
period); if it must collect a LazyFrame, do so only after reducing, with \
|
|
85
92
|
.collect(engine="streaming"). Never .to_pandas(), .iter_rows(), \
|
|
86
|
-
.map_elements() or Python loops over rows.
|
|
93
|
+
.map_elements() or Python loops over rows. If the block is only polars \
|
|
94
|
+
expressions, mark its dataframe inputs "lazy": true so it streams with its \
|
|
95
|
+
neighbours. Per-id logic on a panel uses .over(id) or a group_by on the id \
|
|
96
|
+
joined back -- never a sort-and-loop. Downcast wide numeric outputs \
|
|
97
|
+
(Float64 -> Float32, Int64 -> Int32) and use Categorical for repeated text \
|
|
98
|
+
when a table stays large.
|
|
87
99
|
- Never hardcode an existing input column's name in the body: take it as a \
|
|
88
100
|
`<descriptive>_col` str parameter defaulted to the real column name (or \
|
|
89
101
|
`<descriptive>_cols` for a list) and use pl.col(param). New columns you \
|
|
@@ -192,8 +204,52 @@ If the goal needs something no registry block does, say so -- in \
|
|
|
192
204
|
`questions` while planning, with ask_user while building. """
|
|
193
205
|
|
|
194
206
|
|
|
207
|
+
LARGE_DATA = """## Large data
|
|
208
|
+
The data is large ({size}). You build on a sample; the finished graph runs \
|
|
209
|
+
on all of it, and there every block that doesn't stream (describe_block_type \
|
|
210
|
+
shows `streaming`) holds its whole input and output tables in memory. Design \
|
|
211
|
+
for that:
|
|
212
|
+
- Reduce first, straight after the anchors, with blocks that stream \
|
|
213
|
+
(select, filter, derive_columns, groupby_agg, join): only the columns the \
|
|
214
|
+
model needs, only the population in scope, and down to the grain the model \
|
|
215
|
+
needs (one row per id, per snapshot, or per cell) -- before any split, fit, \
|
|
216
|
+
binning or custom block. A run_to result whose `next` says a table is large \
|
|
217
|
+
is the signal to reduce before going on.
|
|
218
|
+
- Don't fan a full-size table out to several branches; reduce once, then \
|
|
219
|
+
branch.
|
|
220
|
+
- If the model needs the full panel (e.g. a lifetime PD on loan-years) and \
|
|
221
|
+
it would still be too big, say so and ask_user: the options are a sample of \
|
|
222
|
+
ids (whole histories, weighted back if stratified), or aggregating to cells \
|
|
223
|
+
of banded drivers with event and exposure counts.
|
|
224
|
+
- In your report, say which block holds the largest table on the full data."""
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _large_data_note(build: AgentBuild) -> str | None:
|
|
228
|
+
"""The large-data section, for a build whose data is big enough to
|
|
229
|
+
run on a sample (see controller._maybe_sample), else None."""
|
|
230
|
+
from ..cache import memory_budget_bytes
|
|
231
|
+
from .controller import AUTO_SAMPLE_THRESHOLD_ROWS
|
|
232
|
+
from .hints import human_bytes
|
|
233
|
+
|
|
234
|
+
rows = build.full_rows
|
|
235
|
+
if not (rows and rows > AUTO_SAMPLE_THRESHOLD_ROWS) and not build.sample_rows_used:
|
|
236
|
+
return None
|
|
237
|
+
if rows:
|
|
238
|
+
size = f"{rows:,} rows at the anchors"
|
|
239
|
+
if build.full_bytes:
|
|
240
|
+
size += f", {human_bytes(build.full_bytes)} in memory"
|
|
241
|
+
else:
|
|
242
|
+
size = "its full size isn't known: the build started on the user's sample"
|
|
243
|
+
budget = memory_budget_bytes()
|
|
244
|
+
if budget:
|
|
245
|
+
size += f"; the app keeps up to {human_bytes(budget)} of results in memory"
|
|
246
|
+
return LARGE_DATA.format(size=size)
|
|
247
|
+
|
|
248
|
+
|
|
195
249
|
def _common(build: AgentBuild) -> str:
|
|
196
|
-
|
|
250
|
+
common = COMMON.replace("{custom_rule}", CUSTOM_ON if build.options.allow_new_blocks else CUSTOM_OFF)
|
|
251
|
+
note = _large_data_note(build)
|
|
252
|
+
return common if note is None else f"{common}\n\n{note}"
|
|
197
253
|
|
|
198
254
|
|
|
199
255
|
def _custom_contract(build: AgentBuild) -> list[str]:
|
|
@@ -292,6 +348,8 @@ def build_system(build: AgentBuild, compact: bool = False) -> str:
|
|
|
292
348
|
outline)."""
|
|
293
349
|
if build.sample_rows_used:
|
|
294
350
|
sample_note = f"a {build.sample_rows_used:,}-row sample of the data"
|
|
351
|
+
if build.full_rows:
|
|
352
|
+
sample_note += f" (the full data has {build.full_rows:,} rows)"
|
|
295
353
|
else:
|
|
296
354
|
sample_note = "the full data"
|
|
297
355
|
return "\n\n".join(
|
|
@@ -21,7 +21,7 @@ from ..runslot import RunBusy, RunFailed
|
|
|
21
21
|
from ..session import new_id, wire_is_valid
|
|
22
22
|
from . import catalogue, leakage
|
|
23
23
|
from .build import AWAITING_APPROVAL, AWAITING_INPUT, BUILDING, PLANNING, STAGE_ACTIVE, STAGE_DONE, STAGE_SKIPPED, AgentBuild, ToolError
|
|
24
|
-
from .hints import decision_hints
|
|
24
|
+
from .hints import decision_hints, size_hints
|
|
25
25
|
from .layout import Placer, next_lane_order
|
|
26
26
|
|
|
27
27
|
READ = frozenset({PLANNING, BUILDING})
|
|
@@ -267,7 +267,9 @@ def block_hints(b: AgentBuild, block_id: str) -> list[str]:
|
|
|
267
267
|
outputs[p.name] = b.current_output(block_id, p.name)[1]
|
|
268
268
|
except ToolError:
|
|
269
269
|
continue
|
|
270
|
-
|
|
270
|
+
# Size applies to every block: a large table is worth reducing before
|
|
271
|
+
# whatever comes next, whichever block made it.
|
|
272
|
+
return decision_hints(blk.resolve(), outputs) + size_hints(outputs, b.sample_scale)
|
|
271
273
|
|
|
272
274
|
|
|
273
275
|
def suspicious_features(b: AgentBuild) -> dict[str, float | None]:
|
|
@@ -280,12 +280,6 @@ CLAUDE_CLI_KNOWN_MODELS = [
|
|
|
280
280
|
# ---- schemas -----------------------------------------------------------
|
|
281
281
|
|
|
282
282
|
|
|
283
|
-
class PortSpecOut(BaseModel):
|
|
284
|
-
name: str
|
|
285
|
-
type: str
|
|
286
|
-
required: bool = True
|
|
287
|
-
|
|
288
|
-
|
|
289
283
|
class BlockCreate(BaseModel):
|
|
290
284
|
# A built-in's id; ignored when `library` is given.
|
|
291
285
|
category: str = ""
|
|
@@ -106,7 +106,7 @@ IO_TAGS = {
|
|
|
106
106
|
"exports": ("Write results out to a file or the workspace's exports.", "output"),
|
|
107
107
|
}
|
|
108
108
|
IO_BLOCKS = {"input": ("read_csv", "read_parquet", "read_json", "read_excel", "read_sql", "read_dataset", "read_source", "demo_credit_data"),
|
|
109
|
-
"output": ("write_csv", "export")}
|
|
109
|
+
"output": ("write_csv", "write_parquet", "export")}
|
|
110
110
|
|
|
111
111
|
|
|
112
112
|
def _io_blocks(kind: str) -> list[dict[str, Any]]:
|
|
@@ -233,3 +233,18 @@ class BlockDefinition:
|
|
|
233
233
|
# Built-in blocks by id, filled by block_library.load_builtin() at package
|
|
234
234
|
# import (see modelmaker/__init__.py).
|
|
235
235
|
BLOCK_REGISTRY: dict[str, BlockDefinition] = {}
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def streaming_note(d: BlockDefinition) -> str | None:
|
|
239
|
+
"""How a block takes part in a streaming run, from its port flags (see
|
|
240
|
+
runner._fusable_block / _sink_block), or None when it doesn't: it then
|
|
241
|
+
holds its whole input and output tables in memory."""
|
|
242
|
+
df_inputs = [p for p in d.inputs if p.type == "dataframe"]
|
|
243
|
+
single_df_out = len(d.outputs) == 1 and d.outputs[0].type == "dataframe"
|
|
244
|
+
if d.block_type == "input" and single_df_out and d.outputs[0].lazy:
|
|
245
|
+
return "fuses: its lazy scan streams into the blocks after it"
|
|
246
|
+
if single_df_out and df_inputs and len(df_inputs) == len(d.inputs) and all(p.lazy for p in df_inputs):
|
|
247
|
+
return "fuses with neighbouring lazy blocks in a streaming run"
|
|
248
|
+
if d.block_type == "output" and not d.outputs and len(d.inputs) == 1 and d.inputs[0].lazy:
|
|
249
|
+
return "streams its input straight to its destination (a sink)"
|
|
250
|
+
return None
|
|
@@ -19,7 +19,21 @@ BLOCKS = {
|
|
|
19
19
|
"block_type": "input",
|
|
20
20
|
"inputs": [],
|
|
21
21
|
"outputs": [{"name": "out", "lazy": True}],
|
|
22
|
-
"form": [
|
|
22
|
+
"form": [
|
|
23
|
+
{"key": "path", "label": "File path", "kind": "path", "options": [".csv", ".txt"]},
|
|
24
|
+
{"key": "separator", "label": "Separator (e.g. , or | or \\t)", "kind": "text", "placeholder": ","},
|
|
25
|
+
{"key": "has_header", "label": "First row is a header", "kind": "checkbox"},
|
|
26
|
+
{
|
|
27
|
+
"key": "column_names",
|
|
28
|
+
"label": "Column names (comma-separated; optional, replaces the header or column_1, column_2, ...)",
|
|
29
|
+
"kind": "text",
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"key": "infer_schema_rows",
|
|
33
|
+
"label": "Rows to infer dtypes from (0 = read every column as text)",
|
|
34
|
+
"kind": "number",
|
|
35
|
+
},
|
|
36
|
+
],
|
|
23
37
|
"meta": {"inherit": "none"},
|
|
24
38
|
},
|
|
25
39
|
"demo_credit_data": {
|
|
@@ -249,6 +263,30 @@ BLOCKS = {
|
|
|
249
263
|
],
|
|
250
264
|
"meta": {"inherit": "none"},
|
|
251
265
|
},
|
|
266
|
+
"write_parquet": {
|
|
267
|
+
"capabilities": ["filesystem"],
|
|
268
|
+
"use_instead": "export",
|
|
269
|
+
"display_name": "Write Parquet",
|
|
270
|
+
"version": 1,
|
|
271
|
+
"block_type": "output",
|
|
272
|
+
"inputs": [{"name": "df", "lazy": True}],
|
|
273
|
+
"outputs": [],
|
|
274
|
+
"form": [
|
|
275
|
+
{
|
|
276
|
+
"key": "filename",
|
|
277
|
+
"label": "Output filename",
|
|
278
|
+
"kind": "text",
|
|
279
|
+
"placeholder": "output.parquet",
|
|
280
|
+
},
|
|
281
|
+
{
|
|
282
|
+
"key": "compression",
|
|
283
|
+
"label": "Compression",
|
|
284
|
+
"kind": "select",
|
|
285
|
+
"options": ["zstd", "snappy", "lz4", "gzip", "uncompressed"],
|
|
286
|
+
},
|
|
287
|
+
],
|
|
288
|
+
"meta": {"inherit": "none"},
|
|
289
|
+
},
|
|
252
290
|
"export": {
|
|
253
291
|
"display_name": "Export",
|
|
254
292
|
"version": 1,
|
|
@@ -340,15 +378,39 @@ BLOCKS = {
|
|
|
340
378
|
}
|
|
341
379
|
|
|
342
380
|
|
|
343
|
-
def read_csv(
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
381
|
+
def read_csv(
|
|
382
|
+
path: str,
|
|
383
|
+
separator: str = ",",
|
|
384
|
+
has_header: bool = True,
|
|
385
|
+
column_names: str = "",
|
|
386
|
+
infer_schema_rows: int = 100,
|
|
387
|
+
) -> pl.LazyFrame:
|
|
388
|
+
"""Reads a delimited text file into a dataframe on the `out` port; no
|
|
389
|
+
inputs. `path` is the file path (relative to the working directory the
|
|
390
|
+
engine runs in). `separator` is one character ("," by default; "|" for
|
|
391
|
+
pipe-delimited files, "\\t" or "tab" for tab). `has_header` says whether
|
|
392
|
+
the first row holds column names; without one the columns are named
|
|
393
|
+
column_1, column_2, ... unless `column_names` (comma-separated) names
|
|
394
|
+
them -- given with a header, it replaces the header's names.
|
|
395
|
+
`infer_schema_rows` is how many rows polars reads to infer dtypes
|
|
396
|
+
(default 100); 0 reads every column as text, the safe choice for a
|
|
397
|
+
large file whose first rows aren't representative (cast afterwards).
|
|
398
|
+
Column roles start untagged."""
|
|
399
|
+
if separator in ("\\t", "tab", "TAB"):
|
|
400
|
+
separator = "\t"
|
|
401
|
+
if len(separator) != 1:
|
|
402
|
+
raise ValueError(f"separator must be a single character, got {separator!r}")
|
|
403
|
+
names = [n.strip() for n in column_names.split(",") if n.strip()] if column_names else None
|
|
348
404
|
# A lazy scan: the engine collects it, pushing sample mode's row cap
|
|
349
405
|
# down into the scan first, and a streaming run fuses it with whatever
|
|
350
406
|
# follows (see the output port's "lazy" flag).
|
|
351
|
-
return pl.scan_csv(
|
|
407
|
+
return pl.scan_csv(
|
|
408
|
+
path,
|
|
409
|
+
separator=separator,
|
|
410
|
+
has_header=has_header,
|
|
411
|
+
new_columns=names,
|
|
412
|
+
infer_schema_length=infer_schema_rows,
|
|
413
|
+
)
|
|
352
414
|
|
|
353
415
|
|
|
354
416
|
def demo_credit_data(n_rows: int = 5000, seed: int = 42) -> pl.DataFrame:
|
|
@@ -622,6 +684,28 @@ def write_csv(df: pl.DataFrame, filename: str, output_dir: str = ".") -> None:
|
|
|
622
684
|
df.lazy().sink_csv(os.path.join(output_dir, filename))
|
|
623
685
|
|
|
624
686
|
|
|
687
|
+
PARQUET_COMPRESSIONS = ("zstd", "snappy", "lz4", "gzip", "uncompressed")
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def write_parquet(df: pl.DataFrame, filename: str, compression: str = "zstd", output_dir: str = ".") -> None:
|
|
691
|
+
"""Writes the `df` input to a Parquet file; no output ports. `filename`
|
|
692
|
+
is the file name (e.g. "loans.parquet"), written inside `output_dir`,
|
|
693
|
+
which the engine injects as the run's output directory -- leave it
|
|
694
|
+
unset. `compression` is "zstd" (default), "snappy", "lz4", "gzip" or
|
|
695
|
+
"uncompressed". Overwrites any existing file of the same name. In a
|
|
696
|
+
streaming run (Run all (streaming)) the data goes straight from its
|
|
697
|
+
source to disk without ever being held in memory whole -- the way to
|
|
698
|
+
convert a large CSV to Parquet once, so later runs read only the
|
|
699
|
+
columns they use."""
|
|
700
|
+
import os
|
|
701
|
+
|
|
702
|
+
if compression not in PARQUET_COMPRESSIONS:
|
|
703
|
+
raise ValueError(f"compression must be one of {', '.join(PARQUET_COMPRESSIONS)}, got {compression!r}")
|
|
704
|
+
# As write_csv: .lazy() is a no-op on a LazyFrame, so a streaming run
|
|
705
|
+
# hands this its upstream's uncollected plan and it sinks to disk.
|
|
706
|
+
df.lazy().sink_parquet(os.path.join(output_dir, filename), compression=compression)
|
|
707
|
+
|
|
708
|
+
|
|
625
709
|
EXPORT_FORMATS = ("csv", "parquet")
|
|
626
710
|
|
|
627
711
|
|