TalkPipe 0.11.2__tar.gz → 0.11.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.github/workflows/ci-cd.yml +2 -2
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.gitignore +2 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/AGENTS.md +2 -1
- {talkpipe-0.11.2 → talkpipe-0.11.4}/CHANGELOG.md +35 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/CLAUDE.md +2 -1
- {talkpipe-0.11.2/src/talkpipe.egg-info → talkpipe-0.11.4}/PKG-INFO +12 -6
- {talkpipe-0.11.2 → talkpipe-0.11.4}/README.md +7 -3
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/README.md +1 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/api-reference/lazy-loading.md +43 -79
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/api-reference/talkpipe-plugin-manager.md +13 -14
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/api-reference/talkpipe-ref.md +4 -2
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/chatterlang.md +53 -227
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/configuration.md +2 -0
- talkpipe-0.11.4/docs/architecture/documentation-formatting.md +58 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/extending-talkpipe.md +74 -5
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/metadata-stream.md +20 -4
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/pipe-api.md +47 -4
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/protocol.md +52 -2
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/streaming.md +58 -21
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/README.md +1 -9
- talkpipe-0.11.4/docs/tutorials/Tutorial_1-Document_Indexing/README.md +191 -0
- talkpipe-0.11.4/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/README.md +213 -0
- talkpipe-0.11.4/docs/tutorials/Tutorial_3_Report_Writing/README.md +206 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/pyproject.toml +16 -12
- talkpipe-0.11.4/scripts/extract_documentation_examples.py +194 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/chatterlang_generate_entry_points.py +55 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/chatterlang_serve.py +26 -1
- talkpipe-0.11.4/src/talkpipe/app/static/unit-docs.html +1 -0
- talkpipe-0.11.4/src/talkpipe/app/static/unit-docs.txt +1 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/extraction.py +67 -1
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipe/basic.py +47 -100
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipe/core.py +4 -14
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipe/fork.py +43 -28
- talkpipe-0.11.4/src/talkpipe/pipe/math.py +95 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/collections.py +34 -3
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/config.py +83 -90
- talkpipe-0.11.4/src/talkpipe/util/constants.py +16 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/doc_extraction.py +40 -40
- {talkpipe-0.11.2 → talkpipe-0.11.4/src/talkpipe.egg-info}/PKG-INFO +12 -6
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe.egg-info/SOURCES.txt +2 -2
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe.egg-info/entry_points.txt +8 -7
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe.egg-info/requires.txt +4 -1
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/app/test_chatterlang_generate_entry_points.py +27 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/app/test_chatterlang_serve.py +11 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_extraction.py +111 -2
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipe/test_math.py +8 -8
- talkpipe-0.11.2/METADATA_DESIGN_ALTERNATIVES.md +0 -178
- talkpipe-0.11.2/METADATA_STREAM_DESIGN.md +0 -192
- talkpipe-0.11.2/docs/tutorials/Tutorial_1-Document_Indexing/README.md +0 -268
- talkpipe-0.11.2/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/README.md +0 -243
- talkpipe-0.11.2/docs/tutorials/Tutorial_3_Report_Writing/README.md +0 -300
- talkpipe-0.11.2/src/talkpipe/app/static/unit-docs.html +0 -1
- talkpipe-0.11.2/src/talkpipe/app/static/unit-docs.txt +0 -1
- talkpipe-0.11.2/src/talkpipe/pipe/math.py +0 -185
- talkpipe-0.11.2/src/talkpipe/util/constants.py +0 -5
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.bandit +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.claude/commands/check_examples.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.claude/commands/review_documentation.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.dockerignore +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.github/CICD.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.github/ISSUE_TEMPLATE/release-checklist.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.github/SECURITY.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.github/copilot-instructions.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.github/dependabot.yml +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.safety-project.ini +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.vscode/settings.json +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/.vscode/tasks.json +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/COPYRIGHT +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/Dockerfile +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/LICENSE +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/bin/config-analyzer.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/bin/config_bash +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/.gitignore +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/LLMChat.png +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/LLMEvaluateWebPage.png +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/TalkPipe.png +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/TalkPipe.xcf +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/api-reference/README.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/api-reference/chatterlang-script.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/api-reference/chatterlang-server.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/api-reference/chatterlang-workbench.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/architecture/README.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/make_structure_diagram.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/quickstart.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/structure.tex +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/talkpipe_architecture.png +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/talkpipe_architecture.tex +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/talkpipe_diagram.png +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/.gitignore +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_1_CreateSyntheticData.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_1_CreateSyntheticData.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_1_CreateSyntheticData.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_2_IndexStories.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_2_IndexStories.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_2_IndexStories.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_3_SearchStories.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_3_SearchStories.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/Step_3_SearchStories.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/stories.json +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_1-Document_Indexing/story_search_ui.yml +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_1_CreateVectorDatabase.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_1_CreateVectorDatabase.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_1_CreateVectorDatabase.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_2_SearchByExample.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_2_SearchByExample.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_2_SearchByExample.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_3_SpecializedRag.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_3_SpecializedRag.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/Step_3_SpecializedRag.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/step_3_extras.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_2-Search_by_Example_and_RAG/story_by_example_ui.yml +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_1_ExecutiveSummaryGeneration.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_1_ExecutiveSummaryGeneration.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_1_ExecutiveSummaryGeneration.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_2_DetailedAnalysisReportGeneration.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_2_DetailedAnalysisReportGeneration.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_2_DetailedAnalysisReportGeneration.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_3_MultiFormatReportGeneration.bat +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_3_MultiFormatReportGeneration.script +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/Step_3_MultiFormatReportGeneration.sh +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/file_structure.md +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/multi_format_ui.yml +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/report_topic_ui.yml +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/step3_extras.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/step_1_extras.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/docs/tutorials/Tutorial_3_Report_Writing/step_2_extras.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/pytest.ini +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/setup.cfg +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/setup.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/chatterlang_reference_browser.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/chatterlang_reference_generator.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/chatterlang_script.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/chatterlang_workbench.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/static/.gitignore +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/static/favicon.ico +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/app/talkpipe_plugin_manager.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/chatterlang/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/chatterlang/compiler.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/chatterlang/parsers.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/chatterlang/registry.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/email.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/html.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/mongo.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/rss.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/text/chunking_units.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/data/text/operations.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/llm/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/llm/chat.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/llm/config.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/llm/embedding.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/llm/embedding_adapters.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/llm/prompt_adapters.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/operations/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/operations/filtering.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/operations/matrices.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/operations/thread_ops.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/operations/transforms.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipe/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipe/io.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipe/metadata.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipelines/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipelines/basic_rag.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/pipelines/vector_databases.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/search/abstract.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/search/lancedb.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/search/whoosh.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/__init__.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/data_manipulation.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/iterators.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/os.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe/util/plugin_loader.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe.egg-info/dependency_links.txt +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/src/talkpipe.egg-info/top_level.txt +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/stories.json +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/test-workflow.yml +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/conftest.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/app/test_chatterlang_reference_browser.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/app/test_chatterlang_reference_generator.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/app/test_chatterlang_script.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/app/test_talkpipe_plugin_manager.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/chatterlang/test_chatterlang_workbench.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/chatterlang/test_compiler.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/chatterlang/test_entry_points.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/chatterlang/test_parsers.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/chatterlang/test_registry.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/sample_feed.pkl +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test.docx +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_email.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_html.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_html_exceptions.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_mogomock.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_mongo.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_mongo_integration.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/test_rss.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/text/test_chunking_units.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/data/text/test_operations.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/llm/test_chat.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/llm/test_embedding.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/operations/test_filtering.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/operations/test_matrices.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/operations/test_threading.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/operations/test_transforms.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipe/test_basic.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipe/test_core.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipe/test_fork.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipe/test_io.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipe/test_metadata.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipelines/test_basic_rag.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/pipelines/test_vector_databases.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/search/test_abstract.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/search/test_lancedb.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/search/test_lancedb_cache.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/search/test_whoosh.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/test_documentation_examples.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_collections.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_config.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_data_manipulation.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_doc_extraction.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_iterators.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_os.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_plugin_loader.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/talkpipe/util/test_util.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/test_security_vulnerabilities.py +0 -0
- {talkpipe-0.11.2 → talkpipe-0.11.4}/tests/testutils.py +0 -0
|
@@ -2,9 +2,9 @@ name: CI/CD Pipeline
|
|
|
2
2
|
|
|
3
3
|
on:
|
|
4
4
|
push:
|
|
5
|
-
branches: [ main, develop, 'bugfix/**' ]
|
|
5
|
+
branches: [ main, develop, 'bugfix/**', 'feature/**' ]
|
|
6
6
|
pull_request:
|
|
7
|
-
branches: [ main, develop, 'bugfix/**' ]
|
|
7
|
+
branches: [ main, develop, 'bugfix/**', 'feature/**' ]
|
|
8
8
|
release:
|
|
9
9
|
types: [ published ]
|
|
10
10
|
|
|
@@ -38,7 +38,8 @@ TalkPipe is a Python toolkit that makes it easy to create, test, and deploy work
|
|
|
38
38
|
- Unit tests should be as short as possible while still testing the functionality as completely as possible.
|
|
39
39
|
- User interfaces should be visually appealing and maintain consistency with each other.
|
|
40
40
|
- When writing or updating pipeline examples in documentation, write examples with stand-alone code and
|
|
41
|
-
test the examples by running them.
|
|
41
|
+
test the examples by running them. Prefer unindented \`\`\`python fences for easier extraction; see
|
|
42
|
+
docs/architecture/documentation-formatting.md.
|
|
42
43
|
- When creating a new source or segment, update pyproject.toml to include it in the endpoints.
|
|
43
44
|
docs/architecture/protocol.md names protocol conventions to use when writing new sources and segments.
|
|
44
45
|
- When writing documentation for segments or sources, do not document the parameters.
|
|
@@ -1,5 +1,40 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.11.4
|
|
4
|
+
- Documentation example fixes: added missing imports and stubs to extending-talkpipe.md,
|
|
5
|
+
metadata-stream.md, pipe-api.md; introduced `# skip-extract` for blocks that cannot run
|
|
6
|
+
standalone (setup.py, relative imports, illustrative fragments); updated
|
|
7
|
+
documentation-formatting.md with skip-extract guidance.
|
|
8
|
+
- Improved Tutorial 1 (Document Indexing), Tutorial 2 (Search by Example and RAG), and Tutorial 3
|
|
9
|
+
(Report Generation) documentation: clearer structure, prerequisites and quick-start section,
|
|
10
|
+
value-focused intro, and step-by-step guidance for running the tutorials.
|
|
11
|
+
- Fixed documentation examples: corrected lazy-loading.md ChatterLang syntax (range as source
|
|
12
|
+
needs INPUT FROM, uses lower/upper not start/stop; scale uses multiplier not factor);
|
|
13
|
+
corrected protocol.md assign_property example (nested paths require direct assignment).
|
|
14
|
+
Replaced optional-dependency examples (tensorflow, pandas) in lazy-loading.md with stdlib
|
|
15
|
+
equivalents (csv, sqlite3) so all snippets run out of the box.
|
|
16
|
+
- Fixed ReadFile segment naming discrepency.
|
|
17
|
+
- Internal simplifications: removed unreachable code, extracted shared hash validation logic,
|
|
18
|
+
consolidated boolean filter segments (isIn/isNotIn/isTrue/isFalse) with common helper.
|
|
19
|
+
- Additional simplifications: consolidated imports and simplified firstN (itertools.islice),
|
|
20
|
+
removed verbose per-item debug logging in FilterExpression, factored comparison segments
|
|
21
|
+
(eq/neq/gt/gte/lt/lte) via factory, fixed hash_data to parse string field_list, removed
|
|
22
|
+
redundant empty __init__ from DescribeData/ToList/ToDataFrame.
|
|
23
|
+
- chatterlang_generate_entry_points now discovers factory-created segments (e.g. gt, eq, lt).
|
|
24
|
+
Fixed pyproject.toml entry points for comparison segments to reference correct object names
|
|
25
|
+
(GT, EQ, etc.) instead of lowercase.
|
|
26
|
+
|
|
27
|
+
## 0.11.3
|
|
28
|
+
- Fixed chatterlang_serve stream interface so search results display reliably. The UI now
|
|
29
|
+
shows results from the `/process` response directly instead of relying solely on
|
|
30
|
+
Server-Sent Events, avoiding a race where SSE sometimes did not deliver all items before
|
|
31
|
+
the next interaction. SSE events received during a request are buffered and discarded
|
|
32
|
+
when the response arrives, preventing duplicate display of results.
|
|
33
|
+
- Added optional PDF extraction via `pypdf`. Install with `pip install talkpipe[pypdf]` or
|
|
34
|
+
`talkpipe[all]` to enable reading PDF files with readFile and the new readpdf segment.
|
|
35
|
+
Without the optional dependency, if no other pdf hander is registered, PDF extraction
|
|
36
|
+
fails with a clear ImportError directing users to install the pypdf extra.
|
|
37
|
+
|
|
3
38
|
## 0.11.2
|
|
4
39
|
- Fix for prompt segment error that would display an error message if history_file is none.
|
|
5
40
|
|
|
@@ -38,7 +38,8 @@ TalkPipe is a Python toolkit that makes it easy to create, test, and deploy work
|
|
|
38
38
|
- Unit tests should be as short as possible while still testing the functionality as completely as possible.
|
|
39
39
|
- User interfaces should be visually appealing and maintain consistency with each other.
|
|
40
40
|
- When writing or updating pipeline examples in documentation, write examples with stand-alone code and
|
|
41
|
-
test the examples by running them.
|
|
41
|
+
test the examples by running them. Prefer unindented \`\`\`python fences for easier extraction; see
|
|
42
|
+
docs/architecture/documentation-formatting.md.
|
|
42
43
|
- When creating a new source or segment, update pyproject.toml to include it in the endpoints.
|
|
43
44
|
docs/architecture/protocol.md names protocol conventions to use when writing new sources and segments.
|
|
44
45
|
- When writing documentation for segments or sources, do not document the parameters.
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: talkpipe
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.4
|
|
4
4
|
Summary: Python internal and external DSL for writing generative AI analytics
|
|
5
5
|
Author-email: Travis Bauer <tlbauer@sandia.gov>
|
|
6
6
|
Maintainer-email: Travis Bauer <tlbauer@sandia.gov>
|
|
7
7
|
License-Expression: Apache-2.0
|
|
8
8
|
Project-URL: Homepage, https://github.com/sandialabs/talkpipe
|
|
9
9
|
Keywords: ai
|
|
10
|
-
Classifier: Development Status ::
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Requires-Python: >=3.11
|
|
@@ -47,8 +47,10 @@ Provides-Extra: openai
|
|
|
47
47
|
Requires-Dist: openai; extra == "openai"
|
|
48
48
|
Provides-Extra: anthropic
|
|
49
49
|
Requires-Dist: anthropic; extra == "anthropic"
|
|
50
|
+
Provides-Extra: pypdf
|
|
51
|
+
Requires-Dist: pypdf; extra == "pypdf"
|
|
50
52
|
Provides-Extra: all
|
|
51
|
-
Requires-Dist: talkpipe[anthropic,ollama,openai]; extra == "all"
|
|
53
|
+
Requires-Dist: talkpipe[anthropic,ollama,openai,pypdf]; extra == "all"
|
|
52
54
|
Dynamic: license-file
|
|
53
55
|
|
|
54
56
|
<center><img src="docs/TalkPipe.png" width=500></center>
|
|
@@ -318,7 +320,7 @@ script = """
|
|
|
318
320
|
"""
|
|
319
321
|
|
|
320
322
|
analyzer = compiler.compile(script).as_function(single_in=True)
|
|
321
|
-
analyzer("
|
|
323
|
+
analyzer("http://example.com/")
|
|
322
324
|
```
|
|
323
325
|
|
|
324
326
|
## Example 4: Content Evaluation Pipeline
|
|
@@ -396,7 +398,8 @@ indexing_script = """
|
|
|
396
398
|
path="./my_knowledge_base",
|
|
397
399
|
embedding_model="nomic-embed-text",
|
|
398
400
|
embedding_source="ollama",
|
|
399
|
-
embedding_field="text"
|
|
401
|
+
embedding_field="text",
|
|
402
|
+
overwrite=True
|
|
400
403
|
]
|
|
401
404
|
"""
|
|
402
405
|
indexer = compiler.compile(indexing_script).as_function(single_in=False)
|
|
@@ -577,13 +580,16 @@ result = pipeline.as_function(single_out=False)()
|
|
|
577
580
|
|
|
578
581
|
**Segments** transform data:
|
|
579
582
|
```python
|
|
583
|
+
from talkpipe.pipe.math import arange
|
|
584
|
+
from talkpipe.pipe import core, io
|
|
585
|
+
|
|
580
586
|
@core.segment()
|
|
581
587
|
def multiplyBy(items, factor=2):
|
|
582
588
|
for item in items:
|
|
583
589
|
yield item * factor
|
|
584
590
|
|
|
585
591
|
# Use it to double the Fibonacci numbers
|
|
586
|
-
pipeline =
|
|
592
|
+
pipeline = arange(lower=5, upper=10) | multiplyBy(factor=3) | io.Print()
|
|
587
593
|
result = pipeline.as_function(single_out=False)()
|
|
588
594
|
|
|
589
595
|
# Output:
|
|
@@ -265,7 +265,7 @@ script = """
|
|
|
265
265
|
"""
|
|
266
266
|
|
|
267
267
|
analyzer = compiler.compile(script).as_function(single_in=True)
|
|
268
|
-
analyzer("
|
|
268
|
+
analyzer("http://example.com/")
|
|
269
269
|
```
|
|
270
270
|
|
|
271
271
|
## Example 4: Content Evaluation Pipeline
|
|
@@ -343,7 +343,8 @@ indexing_script = """
|
|
|
343
343
|
path="./my_knowledge_base",
|
|
344
344
|
embedding_model="nomic-embed-text",
|
|
345
345
|
embedding_source="ollama",
|
|
346
|
-
embedding_field="text"
|
|
346
|
+
embedding_field="text",
|
|
347
|
+
overwrite=True
|
|
347
348
|
]
|
|
348
349
|
"""
|
|
349
350
|
indexer = compiler.compile(indexing_script).as_function(single_in=False)
|
|
@@ -524,13 +525,16 @@ result = pipeline.as_function(single_out=False)()
|
|
|
524
525
|
|
|
525
526
|
**Segments** transform data:
|
|
526
527
|
```python
|
|
528
|
+
from talkpipe.pipe.math import arange
|
|
529
|
+
from talkpipe.pipe import core, io
|
|
530
|
+
|
|
527
531
|
@core.segment()
|
|
528
532
|
def multiplyBy(items, factor=2):
|
|
529
533
|
for item in items:
|
|
530
534
|
yield item * factor
|
|
531
535
|
|
|
532
536
|
# Use it to double the Fibonacci numbers
|
|
533
|
-
pipeline =
|
|
537
|
+
pipeline = arange(lower=5, upper=10) | multiplyBy(factor=3) | io.Print()
|
|
534
538
|
result = pipeline.as_function(single_out=False)()
|
|
535
539
|
|
|
536
540
|
# Output:
|
|
@@ -36,6 +36,7 @@ Found a gap or error? Documentation improvements are welcome! Please:
|
|
|
36
36
|
1. Check existing issues for the topic
|
|
37
37
|
2. Submit a pull request with your improvements
|
|
38
38
|
3. Follow the existing documentation style and structure
|
|
39
|
+
4. See [Documentation Formatting](architecture/documentation-formatting.md) for code block and example guidelines
|
|
39
40
|
|
|
40
41
|
---
|
|
41
42
|
|
|
@@ -52,7 +52,7 @@ enable_lazy_imports()
|
|
|
52
52
|
|
|
53
53
|
# Your TalkPipe code here
|
|
54
54
|
from talkpipe import compile
|
|
55
|
-
pipeline = compile("range[
|
|
55
|
+
pipeline = compile("INPUT FROM range[lower=0, upper=100] | print")
|
|
56
56
|
|
|
57
57
|
# Disable lazy loading if needed
|
|
58
58
|
disable_lazy_imports()
|
|
@@ -73,6 +73,7 @@ TalkPipe uses a hybrid registry that combines decorator-based registration with
|
|
|
73
73
|
When you request a component by name:
|
|
74
74
|
|
|
75
75
|
```python
|
|
76
|
+
from talkpipe.chatterlang.registry import segment_registry
|
|
76
77
|
segment_registry.get("print")
|
|
77
78
|
```
|
|
78
79
|
|
|
@@ -90,6 +91,7 @@ The registry follows this process:
|
|
|
90
91
|
When you access all components:
|
|
91
92
|
|
|
92
93
|
```python
|
|
94
|
+
from talkpipe.chatterlang.registry import segment_registry
|
|
93
95
|
all_segments = segment_registry.all
|
|
94
96
|
```
|
|
95
97
|
|
|
@@ -121,8 +123,8 @@ from talkpipe import compile
|
|
|
121
123
|
|
|
122
124
|
# With lazy loading enabled, this starts fast
|
|
123
125
|
pipeline = compile("""
|
|
124
|
-
range[
|
|
125
|
-
| scale[
|
|
126
|
+
INPUT FROM range[lower=0, upper=100]
|
|
127
|
+
| scale[multiplier=2]
|
|
126
128
|
| print
|
|
127
129
|
""")
|
|
128
130
|
|
|
@@ -280,6 +282,7 @@ Place components that are often used together in the same module:
|
|
|
280
282
|
```python
|
|
281
283
|
# Good: data_io.py - components commonly used together
|
|
282
284
|
from talkpipe.chatterlang.registry import register_segment
|
|
285
|
+
from talkpipe.pipe.core import AbstractSegment
|
|
283
286
|
|
|
284
287
|
@register_segment("readJson")
|
|
285
288
|
class ReadJson(AbstractSegment):
|
|
@@ -292,31 +295,12 @@ class WriteJson(AbstractSegment):
|
|
|
292
295
|
pass
|
|
293
296
|
```
|
|
294
297
|
|
|
295
|
-
#### 2. Isolate Heavy Dependencies
|
|
296
|
-
|
|
297
|
-
Place components with heavy or optional dependencies in separate modules:
|
|
298
|
-
|
|
299
|
-
```python
|
|
300
|
-
# ml_segments.py - isolated because it requires tensorflow
|
|
301
|
-
import tensorflow as tf # Heavy import
|
|
302
|
-
|
|
303
|
-
@register_segment("tensorflowPredict")
|
|
304
|
-
class TensorFlowPredict(AbstractSegment):
|
|
305
|
-
pass
|
|
306
|
-
|
|
307
|
-
# basic_segments.py - no heavy dependencies
|
|
308
|
-
@register_segment("filter")
|
|
309
|
-
class Filter(AbstractSegment):
|
|
310
|
-
pass
|
|
311
|
-
```
|
|
312
|
-
|
|
313
|
-
This way, users who don't need TensorFlow won't pay the cost of importing it.
|
|
314
298
|
|
|
315
|
-
####
|
|
299
|
+
#### 2. Organize by Domain
|
|
316
300
|
|
|
317
301
|
Structure components by domain or functionality:
|
|
318
302
|
|
|
319
|
-
```
|
|
303
|
+
```
|
|
320
304
|
# Project structure
|
|
321
305
|
src/myproject/
|
|
322
306
|
sources/
|
|
@@ -359,6 +343,7 @@ Enable lazy loading in these scenarios:
|
|
|
359
343
|
|
|
360
344
|
2. **Documentation generators**: Need to discover all components but may not use them all
|
|
361
345
|
```python
|
|
346
|
+
from talkpipe.chatterlang.registry import segment_registry, enable_lazy_imports
|
|
362
347
|
enable_lazy_imports()
|
|
363
348
|
all_segments = segment_registry.list_entry_points() # Fast discovery
|
|
364
349
|
```
|
|
@@ -386,33 +371,7 @@ Enable lazy loading in these scenarios:
|
|
|
386
371
|
|
|
387
372
|
When developing TalkPipe plugins, follow these guidelines:
|
|
388
373
|
|
|
389
|
-
#### 1.
|
|
390
|
-
|
|
391
|
-
Avoid importing heavy dependencies at module level:
|
|
392
|
-
|
|
393
|
-
```python
|
|
394
|
-
# Bad: Heavy import at module level
|
|
395
|
-
from talkpipe.chatterlang.registry import register_segment
|
|
396
|
-
import tensorflow as tf # Loaded even if segment never used
|
|
397
|
-
|
|
398
|
-
@register_segment("tfPredict")
|
|
399
|
-
class TensorFlowPredict(AbstractSegment):
|
|
400
|
-
def transform(self, input_iter):
|
|
401
|
-
# Use tf here
|
|
402
|
-
pass
|
|
403
|
-
|
|
404
|
-
# Good: Import only when needed
|
|
405
|
-
from talkpipe.chatterlang.registry import register_segment
|
|
406
|
-
|
|
407
|
-
@register_segment("tfPredict")
|
|
408
|
-
class TensorFlowPredict(AbstractSegment):
|
|
409
|
-
def transform(self, input_iter):
|
|
410
|
-
import tensorflow as tf # Loaded only when segment actually runs
|
|
411
|
-
# Use tf here
|
|
412
|
-
pass
|
|
413
|
-
```
|
|
414
|
-
|
|
415
|
-
#### 2. Provide Multiple Granularity Levels
|
|
374
|
+
#### 1. Provide Multiple Granularity Levels
|
|
416
375
|
|
|
417
376
|
Offer both bundled and granular entry points:
|
|
418
377
|
|
|
@@ -429,20 +388,22 @@ mlTrain = "myplugin.ml.train"
|
|
|
429
388
|
mlEvaluate = "myplugin.ml.evaluate"
|
|
430
389
|
```
|
|
431
390
|
|
|
432
|
-
####
|
|
391
|
+
#### 2. Document Dependency Requirements
|
|
433
392
|
|
|
434
393
|
Make it clear which components have special requirements:
|
|
435
394
|
|
|
436
395
|
```python
|
|
437
|
-
|
|
438
|
-
|
|
396
|
+
from talkpipe.chatterlang.registry import register_segment
|
|
397
|
+
from talkpipe.pipe.core import AbstractSegment
|
|
398
|
+
|
|
399
|
+
@register_segment("csvProcess")
|
|
400
|
+
class CsvProcess(AbstractSegment):
|
|
439
401
|
"""
|
|
440
|
-
|
|
402
|
+
Process CSV data using the csv module.
|
|
441
403
|
|
|
442
|
-
Requires:
|
|
404
|
+
Requires: csv (stdlib)
|
|
443
405
|
|
|
444
|
-
|
|
445
|
-
Install with: pip install tensorflow
|
|
406
|
+
For large datasets, consider: pip install pandas
|
|
446
407
|
"""
|
|
447
408
|
pass
|
|
448
409
|
```
|
|
@@ -468,7 +429,7 @@ print(f"Registered: {stats['registered']}/{stats['entry_points']} segments")
|
|
|
468
429
|
print(f"Modules loaded: {stats['loaded_modules']}")
|
|
469
430
|
|
|
470
431
|
# Run your pipeline
|
|
471
|
-
pipeline = compile("range[
|
|
432
|
+
pipeline = compile("INPUT FROM range[lower=0, upper=10] | print")
|
|
472
433
|
|
|
473
434
|
# Check what got loaded
|
|
474
435
|
stats_after = segment_registry.stats()
|
|
@@ -484,18 +445,18 @@ Here's a real-world example of restructuring for lazy loading:
|
|
|
484
445
|
```python
|
|
485
446
|
# components.py (slow to load)
|
|
486
447
|
from talkpipe.chatterlang.registry import register_segment
|
|
487
|
-
|
|
488
|
-
import
|
|
489
|
-
import
|
|
490
|
-
import
|
|
491
|
-
|
|
448
|
+
from talkpipe.pipe.core import AbstractSegment
|
|
449
|
+
import csv
|
|
450
|
+
import sqlite3
|
|
451
|
+
import xml.etree.ElementTree as ET
|
|
452
|
+
import decimal
|
|
492
453
|
|
|
493
454
|
# 50+ segments all in one file
|
|
494
|
-
@register_segment("
|
|
495
|
-
class
|
|
455
|
+
@register_segment("csvFilter")
|
|
456
|
+
class CsvFilter(AbstractSegment): pass
|
|
496
457
|
|
|
497
|
-
@register_segment("
|
|
498
|
-
class
|
|
458
|
+
@register_segment("sqliteQuery")
|
|
459
|
+
class SqliteQuery(AbstractSegment): pass
|
|
499
460
|
|
|
500
461
|
# ... 48 more segments
|
|
501
462
|
```
|
|
@@ -504,6 +465,7 @@ class TFPredict(AbstractSegment): pass
|
|
|
504
465
|
```python
|
|
505
466
|
# segments/basic.py (fast to load, no heavy deps)
|
|
506
467
|
from talkpipe.chatterlang.registry import register_segment
|
|
468
|
+
from talkpipe.pipe.core import AbstractSegment
|
|
507
469
|
|
|
508
470
|
@register_segment("filter")
|
|
509
471
|
class Filter(AbstractSegment): pass
|
|
@@ -511,26 +473,28 @@ class Filter(AbstractSegment): pass
|
|
|
511
473
|
@register_segment("map")
|
|
512
474
|
class Map(AbstractSegment): pass
|
|
513
475
|
|
|
514
|
-
# segments/
|
|
476
|
+
# segments/data_io.py (moderate deps)
|
|
515
477
|
from talkpipe.chatterlang.registry import register_segment
|
|
478
|
+
from talkpipe.pipe.core import AbstractSegment
|
|
516
479
|
|
|
517
|
-
@register_segment("
|
|
518
|
-
class
|
|
480
|
+
@register_segment("csvFilter")
|
|
481
|
+
class CsvFilter(AbstractSegment):
|
|
519
482
|
def transform(self, input_iter):
|
|
520
|
-
import
|
|
521
|
-
# Use
|
|
483
|
+
import csv # Lazy import
|
|
484
|
+
# Use csv here
|
|
522
485
|
|
|
523
|
-
# segments/
|
|
486
|
+
# segments/database.py (stdlib deps)
|
|
524
487
|
from talkpipe.chatterlang.registry import register_segment
|
|
488
|
+
from talkpipe.pipe.core import AbstractSegment
|
|
525
489
|
|
|
526
|
-
@register_segment("
|
|
527
|
-
class
|
|
490
|
+
@register_segment("sqliteQuery")
|
|
491
|
+
class SqliteQuery(AbstractSegment):
|
|
528
492
|
def transform(self, input_iter):
|
|
529
|
-
import
|
|
530
|
-
# Use
|
|
493
|
+
import sqlite3 # Lazy import
|
|
494
|
+
# Use sqlite3 here
|
|
531
495
|
```
|
|
532
496
|
|
|
533
|
-
**Result**: Users who only need basic filtering get **10x faster startup** because
|
|
497
|
+
**Result**: Users who only need basic filtering get **10x faster startup** because csv and sqlite3 are never imported. The same pattern applies to heavy ML libraries like TensorFlow or PyTorch.
|
|
534
498
|
|
|
535
499
|
## Troubleshooting
|
|
536
500
|
|
|
@@ -60,7 +60,7 @@ The plugin manager discovers plugins through Python entry points in the `talkpip
|
|
|
60
60
|
Plugins should define entry points in their `setup.py` or `pyproject.toml`:
|
|
61
61
|
|
|
62
62
|
**setup.py:**
|
|
63
|
-
```
|
|
63
|
+
```
|
|
64
64
|
setup(
|
|
65
65
|
name="my-talkpipe-plugin",
|
|
66
66
|
entry_points={
|
|
@@ -138,8 +138,17 @@ You can also register components at the module level:
|
|
|
138
138
|
```python
|
|
139
139
|
# my_plugin/__init__.py
|
|
140
140
|
from talkpipe.chatterlang import registry
|
|
141
|
-
from .
|
|
142
|
-
|
|
141
|
+
from talkpipe.pipe.core import AbstractSource, AbstractSegment
|
|
142
|
+
|
|
143
|
+
# In a real plugin: from .sources import MySource; from .segments import MySegment
|
|
144
|
+
class MySource(AbstractSource):
|
|
145
|
+
def generate(self):
|
|
146
|
+
yield "example"
|
|
147
|
+
|
|
148
|
+
class MySegment(AbstractSegment):
|
|
149
|
+
def transform(self, items):
|
|
150
|
+
for item in items:
|
|
151
|
+
yield item
|
|
143
152
|
|
|
144
153
|
# Registration happens on import
|
|
145
154
|
registry.input_registry.register(MySource, "mySource")
|
|
@@ -178,18 +187,8 @@ talkpipe_plugins --list
|
|
|
178
187
|
|
|
179
188
|
## Integration with TalkPipe
|
|
180
189
|
|
|
181
|
-
Plugins are automatically loaded when you import TalkPipe
|
|
190
|
+
Plugins are automatically loaded when you import TalkPipe.
|
|
182
191
|
|
|
183
|
-
```python
|
|
184
|
-
import talkpipe # Plugins loaded automatically
|
|
185
|
-
|
|
186
|
-
# Use plugin components
|
|
187
|
-
from talkpipe.chatterlang import compiler
|
|
188
|
-
|
|
189
|
-
# Plugin components available in ChatterLang scripts
|
|
190
|
-
script = "| httpGet[url='https://api.example.com'] | jsonParse | print"
|
|
191
|
-
pipeline = compiler.compile(script)
|
|
192
|
-
```
|
|
193
192
|
|
|
194
193
|
## Troubleshooting
|
|
195
194
|
|
|
@@ -337,9 +337,8 @@ chatterlang_reference_browser
|
|
|
337
337
|
#### Documentation Validation
|
|
338
338
|
|
|
339
339
|
**Check component registration**:
|
|
340
|
-
```
|
|
340
|
+
```
|
|
341
341
|
# After adding new components, verify they appear
|
|
342
|
-
chatterlang_reference_generator
|
|
343
342
|
chatterlang_reference_browser
|
|
344
343
|
# > list your.package.name
|
|
345
344
|
# > show yourNewComponent
|
|
@@ -392,6 +391,9 @@ Error running talkpipe_ref command: [Errno 2] No such file or directory
|
|
|
392
391
|
|
|
393
392
|
**Documentation Standards**:
|
|
394
393
|
```python
|
|
394
|
+
from talkpipe.chatterlang import registry
|
|
395
|
+
from talkpipe.pipe import core
|
|
396
|
+
|
|
395
397
|
@registry.register_segment("mySegment")
|
|
396
398
|
@core.field_segment()
|
|
397
399
|
def my_segment(item: str, multiplier: int = 1,
|