TalkPipe 0.4.1__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {talkpipe-0.4.1 → talkpipe-0.5.0}/.github/ISSUE_TEMPLATE/release-checklist.md +1 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/.vscode/settings.json +0 -5
- talkpipe-0.5.0/CHANGELOG.md +85 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/Dockerfile +4 -6
- {talkpipe-0.4.1 → talkpipe-0.5.0}/PKG-INFO +4 -2
- talkpipe-0.5.0/base-image/Dockerfile +19 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/pyproject.toml +9 -4
- talkpipe-0.5.0/pytest.ini +3 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/__init__.py +2 -0
- talkpipe-0.5.0/src/talkpipe/app/__init__.py +0 -0
- talkpipe-0.5.0/src/talkpipe/app/apiendpoint.py +407 -0
- talkpipe-0.4.1/src/talkpipe/app/scriptendpoint.py → talkpipe-0.5.0/src/talkpipe/app/chatterlang_server.py +1 -1
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/app/static/unit-docs.html +364 -29
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/app/static/unit-docs.txt +272 -22
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/chatterlang/compiler.py +30 -3
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/chatterlang/parsers.py +5 -3
- talkpipe-0.5.0/src/talkpipe/data/email.py +457 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/data/html.py +23 -6
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/data/mongo.py +20 -10
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/data/rss.py +2 -2
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/llm/prompt_adapters.py +47 -1
- talkpipe-0.5.0/src/talkpipe/operations/__init__.py +1 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/operations/matrices.py +5 -5
- talkpipe-0.5.0/src/talkpipe/operations/signatures.py +452 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/pipe/basic.py +110 -1
- talkpipe-0.5.0/src/talkpipe/pipe/math.py +201 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/util/data_manipulation.py +61 -2
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe.egg-info/PKG-INFO +4 -2
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe.egg-info/SOURCES.txt +8 -3
- talkpipe-0.5.0/src/talkpipe.egg-info/entry_points.txt +6 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe.egg-info/requires.txt +2 -0
- talkpipe-0.5.0/tests/conftest.py +153 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/app/test_chatcli.py +1 -2
- talkpipe-0.4.1/tests/talkpipe/chatterlang/test_scriptendpoint.py → talkpipe-0.5.0/tests/talkpipe/chatterlang/test_chatterlang_server.py +5 -5
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/chatterlang/test_compiler.py +60 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/chatterlang/test_parsers.py +21 -0
- talkpipe-0.5.0/tests/talkpipe/data/test_email.py +265 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test_html.py +7 -5
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test_mongo.py +1 -1
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test_mongo_integration.py +2 -2
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/llm/test_chat.py +14 -14
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/llm/test_embedding.py +2 -4
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/operations/test_matrices.py +4 -4
- talkpipe-0.5.0/tests/talkpipe/operations/test_signatures.py +652 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/operations/test_threading.py +1 -1
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/pipe/test_basic.py +43 -0
- talkpipe-0.5.0/tests/talkpipe/pipe/test_math.py +43 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/test_documentation_examples.py +4 -7
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/testutils.py +60 -25
- talkpipe-0.4.1/CHANGELOG.md +0 -42
- talkpipe-0.4.1/pytest.ini +0 -5
- talkpipe-0.4.1/src/talkpipe/data/email.py +0 -184
- talkpipe-0.4.1/src/talkpipe/operations/__init__.py +0 -46
- talkpipe-0.4.1/src/talkpipe/pipe/math.py +0 -81
- talkpipe-0.4.1/src/talkpipe.egg-info/entry_points.txt +0 -3
- talkpipe-0.4.1/tests/conftest.py +0 -78
- talkpipe-0.4.1/tests/talkpipe/data/test_email.py +0 -110
- talkpipe-0.4.1/tests/talkpipe/pipe/test_math.py +0 -15
- {talkpipe-0.4.1 → talkpipe-0.5.0}/.gitignore +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/.vscode/tasks.json +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/COPYRIGHT +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/LICENSE +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/README.md +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/bin/config-analyzer.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/bin/config_bash +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/docs/LLMChat.png +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/docs/LLMEvaluateWebPage.png +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/docs/TalkPipe.png +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/docs/TalkPipe.xcf +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/setup.cfg +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/setup.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/app/chatcli.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/app/runscript.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/app/static/.gitignore +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/app/static/favicon.ico +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/app/unit_documentation_analyzer.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/chatterlang/__init__.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/chatterlang/registry.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/data/__init__.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/data/extraction.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/jupyter/basic.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/jupyter/examples.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/jupyter/interactive.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/llm/__init__.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/llm/chat.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/llm/config.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/llm/embedding.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/llm/embedding_adapters.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/operations/filtering.py +0 -0
- /talkpipe-0.4.1/src/talkpipe/operations/threading.py → /talkpipe-0.5.0/src/talkpipe/operations/thread_ops.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/operations/transforms.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/pipe/__init__.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/pipe/core.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/pipe/fork.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/pipe/io.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/util/__init__.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/util/collections.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/util/config.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe/util/os.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe.egg-info/dependency_links.txt +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/src/talkpipe.egg-info/top_level.txt +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/app/test_runscript.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/sample_feed.pkl +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test.docx +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test_extraction.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test_html_exceptions.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test_mogomock.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/data/test_rss.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/jupyter/test_basic_jupyter.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/jupyter/test_examples.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/jupyter/test_interactive.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/operations/test_filtering.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/operations/test_transforms.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/pipe/test_core.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/pipe/test_fork.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/pipe/test_io.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/util/test_collections.py +0 -0
- {talkpipe-0.4.1 → talkpipe-0.5.0}/tests/talkpipe/util/test_util.py +0 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## in development
|
|
4
|
+
### New Segments and Sources
|
|
5
|
+
- lambda expression integration
|
|
6
|
+
- **lambda** - lets one write lambda expressions with limited, but common data manipulation segments. This will
|
|
7
|
+
eliminate the need for most small, simple segments, even gt, lt, etc.
|
|
8
|
+
- **lambdaFilter** - filters based on a lambda expression. Uses the same syntax as lambda.
|
|
9
|
+
- **readEmail** - source for reading email from an IMAP server, along with helper functions
|
|
10
|
+
- **sign** and **verify** for signing and verifying data. Also added utility functions for creating keys and associated
|
|
11
|
+
functions.
|
|
12
|
+
- **jsonReceiver** - both a source and an app for opening an endpoint that can receive json and pass it through a
|
|
13
|
+
chatterlang script
|
|
14
|
+
|
|
15
|
+
### Improvements
|
|
16
|
+
- Broad support for configuration and environment variables. When a parameter to a segment or source starts with a dollar sign (e.g. $myval),
|
|
17
|
+
the system will look first in the current configuration for the key "myval" and then in the OS' environment variables for TALKPIPE_myval.
|
|
18
|
+
- Added a safe lambda expression compiler to data_manipulation. Enables only basic data manipulation functionality
|
|
19
|
+
- Use can specify custom user-agent strings via a configuration file or environment variable for web page
|
|
20
|
+
downloader.
|
|
21
|
+
- Renamed threading.py to thread_ops.py to avoid name collision
|
|
22
|
+
- Renamed scriptendpoint.py to chatterlang_server.py for clarity
|
|
23
|
+
- Documentation improvements
|
|
24
|
+
- Moved code closing mongo connections to the end of the internal loop and out of the del code. It was causing
|
|
25
|
+
error when being disposed as python was exiting.
|
|
26
|
+
- Changed n_iter to max_iter to resolve warning with tsne.
|
|
27
|
+
- Adjust n_neighbors in umap when larger than the number of points to resolve warning
|
|
28
|
+
- Added chatterlang_scipt to the package installer so the user can run talkpipe.app.runscript from the commandline
|
|
29
|
+
|
|
30
|
+
## 0.4.2
|
|
31
|
+
### New Segments
|
|
32
|
+
- snippet - lets the script writer either load chatterlang code from a file or provide it as a parameter.
|
|
33
|
+
- Abstracted the comparison filter used for gt and lt
|
|
34
|
+
- Added filters for gte (greater than or equal), lte (less than or equal), eq (equal) and neq (not equal)
|
|
35
|
+
### Improvements
|
|
36
|
+
- Reconfigured the Docker containers so that there is a base container with the pre-requisites and one for talkpipe.
|
|
37
|
+
This reduces the time needed to build a talkpipe container if the base already exists.
|
|
38
|
+
- Retooled unit tests to eliminate the "online" fixture, replacing it with something that checks if ollama and mongo are
|
|
39
|
+
available locally and skipping tests that require them if they are not available.
|
|
40
|
+
### Bug Fixes
|
|
41
|
+
- Fixed error in rss.py causing a crash if the URL was specified in the configuration file or environment variable
|
|
42
|
+
- Handled code where robots.txt response is returned compressed. It now decompresses it rather than erroring out.
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
## 0.4.1
|
|
47
|
+
- Updated syntax so that pipelines in forks can contain more than one segment. This was an oversight in the original design.
|
|
48
|
+
- Added the mongoInsert and mongoSearch segments
|
|
49
|
+
- Generalized LLM source so that new ones can be registered and used without modifying TalkPipe
|
|
50
|
+
- Updated the Embedding API to match the chat api (with sources, etc)
|
|
51
|
+
- Added ability to create and reduce matrices using UMAP and t-SNE
|
|
52
|
+
- Added template segment for populating a string template
|
|
53
|
+
- Update chatterlang so that a double-double quote ("") gets resolved to one double quote
|
|
54
|
+
- Fixed bug in extract_property that would cause it to fail seeing some properties
|
|
55
|
+
- Improved unit testing and coverage
|
|
56
|
+
- Updated the guided generation classes to make it easier to do guided generations problems in talkpipe
|
|
57
|
+
- Split the talkpipe.util module into a talkpipe.util subpackage with different modules for different types of utilities.
|
|
58
|
+
- Updated the logo
|
|
59
|
+
- Improved and refined the wording in the README
|
|
60
|
+
- Move the documentation script into apps and added a script so you can generate documentation files when talkpipe is installed
|
|
61
|
+
|
|
62
|
+
## 0.3.1
|
|
63
|
+
- jupyter notebook widgets
|
|
64
|
+
- a hash segment
|
|
65
|
+
|
|
66
|
+
## 0.3.0
|
|
67
|
+
- Basic web app for writing and executing ChatterLang scripts, along with endpoints for the underlying functionality. Includes the ability to:
|
|
68
|
+
- write scripts in a text area and compile/execute them.
|
|
69
|
+
- interactive provide text input to the script if the script requires it.
|
|
70
|
+
- the ability to see logging statements emitted.
|
|
71
|
+
- run the server using a command "chatterlang_server" that is included in the whl file.
|
|
72
|
+
- view documentation for the include sources and segments. Note that the documentation is rebuilt
|
|
73
|
+
everytime a whl file is generated.
|
|
74
|
+
- view and load various example scripts.
|
|
75
|
+
installed. This works on at least windows and linux
|
|
76
|
+
- Added comment support for ChatterLang, anything from a hash mark (#) until the end of the line.
|
|
77
|
+
- Added configureLogging segment for configuring logging levels and files from within a script. This is probably temporary and will be removed before 0.3.0 as a better solution is written.
|
|
78
|
+
- Consolidated the logic around configuring logging, migrating the various ways of specifying the configuration strings from runscript into the configure_logging method itself.
|
|
79
|
+
|
|
80
|
+
## 0.2.2 (2025-02-21)
|
|
81
|
+
- The rss segment can get the url to download from the rss_url configuration setting.
|
|
82
|
+
- Updated OpenAI source so that it can support guided generation (and thus the llmScore segment)
|
|
83
|
+
|
|
84
|
+
## 0.2.1 (2025-02-21)
|
|
85
|
+
- Catch all ConnectionErrors when downloading the robots.txt file.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
FROM
|
|
1
|
+
FROM talkpipe:base
|
|
2
2
|
|
|
3
3
|
WORKDIR /app
|
|
4
4
|
|
|
@@ -17,10 +17,8 @@ COPY .git/ .git/
|
|
|
17
17
|
|
|
18
18
|
RUN pip install --upgrade pip
|
|
19
19
|
RUN pip install -e .[dev]
|
|
20
|
-
RUN pytest
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
# Install the basics of the scientific computing stack for analytics
|
|
24
|
-
RUN pip install numpy pandas matplotlib scikit-learn scipy
|
|
20
|
+
RUN pytest --log-cli-level=DEBUG
|
|
21
|
+
RUN rm -fr .git
|
|
22
|
+
RUN rm -fr tests
|
|
25
23
|
|
|
26
24
|
CMD ["python", "-m", "talkpipe.app.runscript", "--load_module", "data/custom_module.py", "--env", "TALKPIPE_SCRIPT"]
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: talkpipe
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Python internal and external DSL for writing generative AI analytics
|
|
5
5
|
Author-email: Travis Bauer <tlbauer@sandia.gov>
|
|
6
6
|
Maintainer-email: Travis Bauer <tlbauer@sandia.gov>
|
|
7
|
+
License-Expression: Apache-2.0
|
|
7
8
|
Project-URL: Homepage, https://github.com/sandialabs/talkpipe
|
|
8
9
|
Keywords: ai
|
|
9
10
|
Classifier: Development Status :: 3 - Alpha
|
|
10
11
|
Classifier: Programming Language :: Python :: 3
|
|
11
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Requires-Python: >=3.11
|
|
14
14
|
Description-Content-Type: text/markdown
|
|
@@ -31,6 +31,8 @@ Requires-Dist: ipywidgets
|
|
|
31
31
|
Requires-Dist: pymongo
|
|
32
32
|
Requires-Dist: umap-learn
|
|
33
33
|
Requires-Dist: scikit-learn
|
|
34
|
+
Requires-Dist: cryptography
|
|
35
|
+
Requires-Dist: uvicorn
|
|
34
36
|
Provides-Extra: dev
|
|
35
37
|
Requires-Dist: pytest; extra == "dev"
|
|
36
38
|
Requires-Dist: pytest-mock; extra == "dev"
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
FROM python:3.11
|
|
2
|
+
|
|
3
|
+
# Set working directory
|
|
4
|
+
WORKDIR /app
|
|
5
|
+
|
|
6
|
+
# Install PDM
|
|
7
|
+
RUN pip install pdm
|
|
8
|
+
|
|
9
|
+
# Copy pyproject.toml into the container
|
|
10
|
+
COPY pyproject.toml ./pyproject.toml
|
|
11
|
+
|
|
12
|
+
# Run pdm lock, export deps to requirements.txt, and install
|
|
13
|
+
RUN pdm lock && \
|
|
14
|
+
pdm export --without-hashes --dev -o requirements.txt && \
|
|
15
|
+
pip install -r requirements.txt && \
|
|
16
|
+
rm requirements.txt
|
|
17
|
+
|
|
18
|
+
# Install the basics of the scientific computing stack for analytics
|
|
19
|
+
RUN pip install numpy pandas matplotlib scikit-learn scipy
|
|
@@ -14,10 +14,10 @@ description = "Python internal and external DSL for writing generative AI analyt
|
|
|
14
14
|
readme = "README.md"
|
|
15
15
|
requires-python = ">=3.11"
|
|
16
16
|
keywords = ["ai"]
|
|
17
|
+
license = "Apache-2.0"
|
|
17
18
|
classifiers = [
|
|
18
19
|
"Development Status :: 3 - Alpha",
|
|
19
20
|
"Programming Language :: Python :: 3",
|
|
20
|
-
"License :: OSI Approved :: Apache Software License",
|
|
21
21
|
"Operating System :: OS Independent",
|
|
22
22
|
]
|
|
23
23
|
dependencies = [
|
|
@@ -38,7 +38,9 @@ dependencies = [
|
|
|
38
38
|
'ipywidgets',
|
|
39
39
|
'pymongo',
|
|
40
40
|
'umap-learn',
|
|
41
|
-
'scikit-learn'
|
|
41
|
+
'scikit-learn',
|
|
42
|
+
'cryptography',
|
|
43
|
+
'uvicorn'
|
|
42
44
|
]
|
|
43
45
|
dynamic = ["version"]
|
|
44
46
|
|
|
@@ -51,8 +53,11 @@ dev = [
|
|
|
51
53
|
]
|
|
52
54
|
|
|
53
55
|
[project.scripts]
|
|
54
|
-
chatterlang_server = "talkpipe.app.
|
|
56
|
+
chatterlang_server = "talkpipe.app.chatterlang_server:main"
|
|
57
|
+
chatterlang_script = "talkpipe.app.runscript:main"
|
|
55
58
|
talkpipe_ref = "talkpipe.app.unit_documentation_analyzer:go"
|
|
59
|
+
talkpipe_genkeys = "talkpipe.operations.crypto:generate_keys_cli"
|
|
60
|
+
talkpipe_endpoint = "talkpipe.app.apiendpoint:go"
|
|
56
61
|
|
|
57
62
|
[tool.setuptools.packages.find]
|
|
58
63
|
where = ["src"]
|
|
@@ -66,4 +71,4 @@ where = ["src"]
|
|
|
66
71
|
#use defaults
|
|
67
72
|
|
|
68
73
|
[project.urls]
|
|
69
|
-
Homepage = "https://github.com/sandialabs/talkpipe"
|
|
74
|
+
Homepage = "https://github.com/sandialabs/talkpipe"
|
|
File without changes
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
"""
|
|
2
|
+
FastAPI JSON Receiver Server
|
|
3
|
+
Receives JSON data via HTTP and processes it with a configurable function
|
|
4
|
+
"""
|
|
5
|
+
from typing import Union
|
|
6
|
+
import logging
|
|
7
|
+
import argparse
|
|
8
|
+
from fastapi.responses import FileResponse
|
|
9
|
+
from fastapi.staticfiles import StaticFiles
|
|
10
|
+
from fastapi import FastAPI, HTTPException, Depends, Header
|
|
11
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
12
|
+
from fastapi.responses import HTMLResponse
|
|
13
|
+
from pydantic import BaseModel
|
|
14
|
+
from typing import Optional, List, Any, Dict, Callable
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
import uvicorn
|
|
17
|
+
import json
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
import threading
|
|
20
|
+
from queue import Queue
|
|
21
|
+
from talkpipe.pipe.core import AbstractSource
|
|
22
|
+
from talkpipe.chatterlang import register_source
|
|
23
|
+
from talkpipe.chatterlang import compile
|
|
24
|
+
from talkpipe.util.config import get_config
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
# Models
|
|
29
|
+
class DataResponse(BaseModel):
|
|
30
|
+
status: str
|
|
31
|
+
message: str
|
|
32
|
+
data: Dict[str, Any]
|
|
33
|
+
timestamp: datetime
|
|
34
|
+
|
|
35
|
+
class DataHistory(BaseModel):
|
|
36
|
+
entries: List[dict]
|
|
37
|
+
count: int
|
|
38
|
+
|
|
39
|
+
class JSONReceiver:
|
|
40
|
+
"""JSON Data Receiver Server Class - supports multiple instances"""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
host: str = "0.0.0.0",
|
|
45
|
+
port: int = 9999,
|
|
46
|
+
api_key: str = "your-secret-key-here",
|
|
47
|
+
require_auth: bool = False,
|
|
48
|
+
processor_func: Callable[[Dict[str, Any]], Any] = None,
|
|
49
|
+
title: str = "JSON Data Receiver",
|
|
50
|
+
history_length: int = 1000
|
|
51
|
+
):
|
|
52
|
+
self.host = host
|
|
53
|
+
self.port = port
|
|
54
|
+
self.api_key = api_key
|
|
55
|
+
self.require_auth = require_auth
|
|
56
|
+
self.history = []
|
|
57
|
+
self.title = title
|
|
58
|
+
self.history_length = history_length
|
|
59
|
+
|
|
60
|
+
# Set up processor function
|
|
61
|
+
self.processor_function = processor_func or self._default_print_processor
|
|
62
|
+
|
|
63
|
+
# Create FastAPI app instance
|
|
64
|
+
self.app = FastAPI(title=f"{title} (Port {port})", version="1.0.0")
|
|
65
|
+
|
|
66
|
+
# Mount favicon.ico directly to root
|
|
67
|
+
@self.app.get("/favicon.ico")
|
|
68
|
+
async def favicon():
|
|
69
|
+
favicon_path = Path(__file__).parent / 'static' / 'favicon.ico'
|
|
70
|
+
if favicon_path.exists():
|
|
71
|
+
return FileResponse(favicon_path)
|
|
72
|
+
else:
|
|
73
|
+
raise HTTPException(status_code=404, detail="Favicon not found")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# Configure middleware
|
|
77
|
+
self._setup_middleware()
|
|
78
|
+
|
|
79
|
+
# Configure routes
|
|
80
|
+
self._setup_routes()
|
|
81
|
+
|
|
82
|
+
# Server instance for stopping
|
|
83
|
+
self.server = None
|
|
84
|
+
self.server_thread = None
|
|
85
|
+
|
|
86
|
+
def _setup_middleware(self):
|
|
87
|
+
"""Configure CORS middleware"""
|
|
88
|
+
self.app.add_middleware(
|
|
89
|
+
CORSMiddleware,
|
|
90
|
+
allow_origins=["*"],
|
|
91
|
+
allow_credentials=True,
|
|
92
|
+
allow_methods=["*"],
|
|
93
|
+
allow_headers=["*"],
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def _setup_routes(self):
|
|
97
|
+
"""Configure all API routes"""
|
|
98
|
+
|
|
99
|
+
@self.app.get("/", response_class=HTMLResponse)
|
|
100
|
+
async def root():
|
|
101
|
+
return self._get_html_interface()
|
|
102
|
+
|
|
103
|
+
@self.app.post("/process", response_model=DataResponse)
|
|
104
|
+
async def process_json(data: Dict[str, Any], api_key: str = Depends(self._verify_api_key)):
|
|
105
|
+
return await self._process_json(data)
|
|
106
|
+
|
|
107
|
+
@self.app.get("/history", response_model=DataHistory)
|
|
108
|
+
async def get_history(limit: int = 50, api_key: str = Depends(self._verify_api_key)):
|
|
109
|
+
return self._get_history(limit)
|
|
110
|
+
|
|
111
|
+
@self.app.delete("/history")
|
|
112
|
+
async def clear_history(api_key: str = Depends(self._verify_api_key)):
|
|
113
|
+
return self._clear_history()
|
|
114
|
+
|
|
115
|
+
@self.app.get("/health")
|
|
116
|
+
async def health_check():
|
|
117
|
+
return {"status": "healthy", "timestamp": datetime.now(), "port": self.port}
|
|
118
|
+
|
|
119
|
+
async def _verify_api_key(self, x_api_key: Optional[str] = Header(None)):
|
|
120
|
+
"""Dependency for API key validation"""
|
|
121
|
+
if self.require_auth and x_api_key != self.api_key:
|
|
122
|
+
raise HTTPException(status_code=403, detail="Invalid API key")
|
|
123
|
+
return x_api_key
|
|
124
|
+
|
|
125
|
+
def _default_print_processor(self, data: Dict[str, Any]) -> str:
|
|
126
|
+
"""Default processor: print JSON data to stdout"""
|
|
127
|
+
try:
|
|
128
|
+
print(f"[Port {self.port}] Received JSON: {json.dumps(data, indent=2, default=str)}")
|
|
129
|
+
return "Data printed successfully"
|
|
130
|
+
except Exception as e:
|
|
131
|
+
logger.error(f"Error processing data: {e}")
|
|
132
|
+
raise HTTPException(status_code=500, detail=f"Processing error: {e}")
|
|
133
|
+
|
|
134
|
+
def _save_to_history(self, data: Dict[str, Any]):
|
|
135
|
+
self.history.append(data)
|
|
136
|
+
while len(self.history) > self.history_length:
|
|
137
|
+
self.history.pop(0)
|
|
138
|
+
|
|
139
|
+
async def _process_json(self, data: Dict[str, Any]) -> DataResponse:
|
|
140
|
+
"""Process JSON data with the configured processor function"""
|
|
141
|
+
|
|
142
|
+
# Log the request
|
|
143
|
+
logger.info(f"[{datetime.now().strftime('%H:%M:%S')}] Port {self.port}: Processing JSON data")
|
|
144
|
+
|
|
145
|
+
try:
|
|
146
|
+
# Process the data
|
|
147
|
+
result = self.processor_function(data)
|
|
148
|
+
|
|
149
|
+
# Save to history
|
|
150
|
+
self._save_to_history(data)
|
|
151
|
+
|
|
152
|
+
# Determine message based on result
|
|
153
|
+
if isinstance(result, str):
|
|
154
|
+
message = result
|
|
155
|
+
else:
|
|
156
|
+
message = "Data processed successfully"
|
|
157
|
+
|
|
158
|
+
return DataResponse(
|
|
159
|
+
status="success",
|
|
160
|
+
message=message,
|
|
161
|
+
data=data,
|
|
162
|
+
timestamp=datetime.now()
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
except HTTPException:
|
|
166
|
+
# Re-raise HTTP exceptions from the processor
|
|
167
|
+
raise
|
|
168
|
+
except Exception as e:
|
|
169
|
+
logger.error(f"Error processing data: {e}")
|
|
170
|
+
raise HTTPException(status_code=500, detail=f"Processing failed: {str(e)}")
|
|
171
|
+
|
|
172
|
+
def _get_history(self, limit: int) -> DataHistory:
|
|
173
|
+
"""Get processing history"""
|
|
174
|
+
return DataHistory(entries=self.history, count=len(self.history))
|
|
175
|
+
|
|
176
|
+
def _clear_history(self):
|
|
177
|
+
"""Clear processing history"""
|
|
178
|
+
self.history.clear()
|
|
179
|
+
logger.info(f"Port {self.port}: History cleared")
|
|
180
|
+
return {"status": "success", "message": "History cleared", "timestamp": datetime.now()}
|
|
181
|
+
|
|
182
|
+
def _get_html_interface(self) -> str:
|
|
183
|
+
"""Get the HTML interface with port-specific URLs"""
|
|
184
|
+
return f"""
|
|
185
|
+
<!DOCTYPE html>
|
|
186
|
+
<html>
|
|
187
|
+
<head>
|
|
188
|
+
<title>{self.title} - Port {self.port}</title>
|
|
189
|
+
<link rel="icon" type="image/ico" href="/favicon.ico">
|
|
190
|
+
<style>
|
|
191
|
+
body {{ font-family: Arial, sans-serif; max-width: 800px; margin: 50px auto; padding: 20px; }}
|
|
192
|
+
input, button, textarea {{ font-size: 16px; padding: 10px; margin: 5px; }}
|
|
193
|
+
textarea {{ width: 500px; height: 150px; font-family: monospace; }}
|
|
194
|
+
.status {{ margin-top: 20px; padding: 10px; border-radius: 5px; }}
|
|
195
|
+
.success {{ background-color: #d4edda; color: #155724; }}
|
|
196
|
+
.error {{ background-color: #f8d7da; color: #721c24; }}
|
|
197
|
+
.history {{ margin-top: 30px; }}
|
|
198
|
+
.history-item {{ padding: 5px; border-bottom: 1px solid #ddd; }}
|
|
199
|
+
code {{ background-color: #f4f4f4; padding: 2px 5px; border-radius: 3px; }}
|
|
200
|
+
.port-info {{ background-color: #e3f2fd; padding: 10px; border-radius: 5px; margin-bottom: 20px; }}
|
|
201
|
+
</style>
|
|
202
|
+
</head>
|
|
203
|
+
<body>
|
|
204
|
+
<div class="port-info">
|
|
205
|
+
<h1>{self.title}</h1>
|
|
206
|
+
<p><strong>Server Port:</strong> {self.port}</p>
|
|
207
|
+
<p><strong>History Length:</strong> {len(self.history)}, max length={self.history_length}</p>
|
|
208
|
+
</div>
|
|
209
|
+
|
|
210
|
+
<p>Send JSON data to process it on this machine:</p>
|
|
211
|
+
|
|
212
|
+
<div>
|
|
213
|
+
<textarea id="jsonData" placeholder='{{"example": "data", "number": 42}}'></textarea><br>
|
|
214
|
+
<button onclick="sendData()">Send JSON Data</button>
|
|
215
|
+
</div>
|
|
216
|
+
|
|
217
|
+
<div id="status"></div>
|
|
218
|
+
|
|
219
|
+
<div class="history">
|
|
220
|
+
<h3>API Endpoints:</h3>
|
|
221
|
+
<ul>
|
|
222
|
+
<li><code>POST /process</code> - Send JSON data for processing</li>
|
|
223
|
+
<li><code>GET /history</code> - Get processing history</li>
|
|
224
|
+
<li><code>DELETE /history</code> - Clear processing history</li>
|
|
225
|
+
<li><code>GET /health</code> - Health check</li>
|
|
226
|
+
<li><code>GET /docs</code> - API documentation</li>
|
|
227
|
+
</ul>
|
|
228
|
+
|
|
229
|
+
<h3>Usage Examples:</h3>
|
|
230
|
+
<pre>
|
|
231
|
+
# Using curl:
|
|
232
|
+
curl -X POST http://localhost:{self.port}/process \\
|
|
233
|
+
-H "Content-Type: application/json" \\
|
|
234
|
+
-d '{{"message": "hello", "value": 123}}'
|
|
235
|
+
|
|
236
|
+
# Using PowerShell:
|
|
237
|
+
Invoke-RestMethod -Uri http://localhost:{self.port}/process \\
|
|
238
|
+
-Method POST -ContentType "application/json" \\
|
|
239
|
+
-Body '{{"key": "value", "number": 42}}'
|
|
240
|
+
|
|
241
|
+
# Using Python:
|
|
242
|
+
import requests
|
|
243
|
+
response = requests.post('http://localhost:{self.port}/process',
|
|
244
|
+
json={{'key': 'value', 'number': 42}})
|
|
245
|
+
print(response.json())</pre>
|
|
246
|
+
</div>
|
|
247
|
+
|
|
248
|
+
<script>
|
|
249
|
+
async function sendData() {{
|
|
250
|
+
const jsonText = document.getElementById('jsonData').value;
|
|
251
|
+
const status = document.getElementById('status');
|
|
252
|
+
|
|
253
|
+
try {{
|
|
254
|
+
const data = JSON.parse(jsonText);
|
|
255
|
+
|
|
256
|
+
const response = await fetch('/process', {{
|
|
257
|
+
method: 'POST',
|
|
258
|
+
headers: {{'Content-Type': 'application/json'}},
|
|
259
|
+
body: JSON.stringify(data)
|
|
260
|
+
}});
|
|
261
|
+
|
|
262
|
+
const result = await response.json();
|
|
263
|
+
|
|
264
|
+
if (response.ok) {{
|
|
265
|
+
status.className = 'status success';
|
|
266
|
+
status.textContent = 'JSON data processed successfully on port {self.port}!';
|
|
267
|
+
}} else {{
|
|
268
|
+
status.className = 'status error';
|
|
269
|
+
status.textContent = 'Error: ' + result.detail;
|
|
270
|
+
}}
|
|
271
|
+
}} catch (error) {{
|
|
272
|
+
status.className = 'status error';
|
|
273
|
+
status.textContent = 'Error: ' + error.message;
|
|
274
|
+
}}
|
|
275
|
+
|
|
276
|
+
setTimeout(() => status.textContent = '', 3000);
|
|
277
|
+
}}
|
|
278
|
+
</script>
|
|
279
|
+
</body>
|
|
280
|
+
</html>
|
|
281
|
+
"""
|
|
282
|
+
|
|
283
|
+
def set_processor_function(self, func: Callable[[Dict[str, Any]], Any]):
|
|
284
|
+
"""Set the function used to process incoming JSON data"""
|
|
285
|
+
self.processor_function = func
|
|
286
|
+
logger.info(f"Port {self.port}: Processor function set to: {func.__name__}")
|
|
287
|
+
|
|
288
|
+
def start(self, background: bool = False):
|
|
289
|
+
"""Start the FastAPI server"""
|
|
290
|
+
logger.info(f"Starting JSON Data Receiver on http://{self.host}:{self.port}")
|
|
291
|
+
logger.info(f"API documentation: http://{self.host}:{self.port}/docs")
|
|
292
|
+
if self.require_auth:
|
|
293
|
+
logger.info(f"Port {self.port}: API key authentication: ENABLED")
|
|
294
|
+
|
|
295
|
+
processor_name = self.processor_function.__name__
|
|
296
|
+
logger.info(f"Port {self.port}: Using processor function: {processor_name}")
|
|
297
|
+
|
|
298
|
+
if background:
|
|
299
|
+
self.server_thread = threading.Thread(
|
|
300
|
+
target=lambda: uvicorn.run(self.app, host=self.host, port=self.port),
|
|
301
|
+
daemon=True
|
|
302
|
+
)
|
|
303
|
+
self.server_thread.start()
|
|
304
|
+
logger.info(f"Port {self.port}: Server started in background thread")
|
|
305
|
+
else:
|
|
306
|
+
uvicorn.run(self.app, host=self.host, port=self.port)
|
|
307
|
+
|
|
308
|
+
def stop(self):
|
|
309
|
+
"""Stop the server (only works if started in background)"""
|
|
310
|
+
if self.server_thread and self.server_thread.is_alive():
|
|
311
|
+
logger.info(f"Port {self.port}: Stopping server...")
|
|
312
|
+
# Note: uvicorn doesn't provide a clean stop method when run this way
|
|
313
|
+
# For production use, consider using uvicorn.Server with proper lifecycle management
|
|
314
|
+
|
|
315
|
+
@register_source("jsonReceiver")
|
|
316
|
+
class JSONReceiverSegment(AbstractSource):
|
|
317
|
+
"""Segment for receiving JSON data via FastAPI"""
|
|
318
|
+
|
|
319
|
+
def __init__(self, port: Union[int,str] = 9999, host: str = "0.0.0.0",
|
|
320
|
+
api_key: str = None, require_auth: bool = False):
|
|
321
|
+
super().__init__()
|
|
322
|
+
self.port = int(port)
|
|
323
|
+
self.host = host
|
|
324
|
+
self.api_key = api_key
|
|
325
|
+
self.require_auth = require_auth
|
|
326
|
+
self.queue = Queue(maxsize=1000)
|
|
327
|
+
|
|
328
|
+
self.receiver = JSONReceiver(
|
|
329
|
+
host=host,
|
|
330
|
+
port=self.port,
|
|
331
|
+
api_key=api_key,
|
|
332
|
+
require_auth=require_auth,
|
|
333
|
+
title=f"JSON Receiver Segment (Port {port})",
|
|
334
|
+
processor_func=self.process_data
|
|
335
|
+
)
|
|
336
|
+
self.receiver.start(background=True)
|
|
337
|
+
logger.info(f"Finished initializing JSONReceiverSegment on port {port}")
|
|
338
|
+
|
|
339
|
+
def process_data(self, data: Dict[str, Any]) -> str:
|
|
340
|
+
"""Process incoming JSON data and add it to the queue"""
|
|
341
|
+
try:
|
|
342
|
+
self.queue.put(data, block=True, timeout=60)
|
|
343
|
+
return f"Data received and queued: {data}"
|
|
344
|
+
except Exception as e:
|
|
345
|
+
logger.error(f"Error processing data: {e}")
|
|
346
|
+
return f"Error processing data: {str(e)}"
|
|
347
|
+
|
|
348
|
+
def generate(self):
|
|
349
|
+
print("Starting JSONReceiverSegment generator...")
|
|
350
|
+
while True:
|
|
351
|
+
# Wait for data to be available in the queue
|
|
352
|
+
print("Waiting for data...")
|
|
353
|
+
# This will block until data is available
|
|
354
|
+
yield self.queue.get(block=True, timeout=None)
|
|
355
|
+
|
|
356
|
+
def go():
|
|
357
|
+
|
|
358
|
+
parser = argparse.ArgumentParser(description='FastAPI JSON Data Receiver')
|
|
359
|
+
parser.add_argument('-p', '--port', type=int, default=9999,
|
|
360
|
+
help='Port to listen on (default: 9999)')
|
|
361
|
+
parser.add_argument('-o', '--host', default='0.0.0.0',
|
|
362
|
+
help='Host to bind to (default: 0.0.0.0)')
|
|
363
|
+
parser.add_argument('--api-key', help='Set API key for authentication')
|
|
364
|
+
parser.add_argument('--require-auth', action='store_true',
|
|
365
|
+
help='Require API key authentication')
|
|
366
|
+
parser.add_argument('--title', default='JSON Data Receiver',
|
|
367
|
+
help='Title for the FastAPI application')
|
|
368
|
+
parser.add_argument('--script-var', default=None,
|
|
369
|
+
help='Configuration key containing script to run')
|
|
370
|
+
parser.add_argument('--script', default=None,
|
|
371
|
+
help='Chatterlang script to run on received data')
|
|
372
|
+
|
|
373
|
+
args = parser.parse_args()
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
script = None
|
|
377
|
+
if args.script_var and args.script:
|
|
378
|
+
raise ValueError("Cannot specify both --script-var and --script. Use one or the other.")
|
|
379
|
+
if args.script_var:
|
|
380
|
+
script = get_config(args.script_var)
|
|
381
|
+
if not script:
|
|
382
|
+
raise ValueError(f"No script found for variable '{args.script_var}' in configuration.")
|
|
383
|
+
else:
|
|
384
|
+
script = args.script
|
|
385
|
+
|
|
386
|
+
if script:
|
|
387
|
+
# Compile the Chatterlang script
|
|
388
|
+
compiled_script = compile(script)
|
|
389
|
+
compiled_script = compiled_script.asFunction(single_in=True, single_out=False)
|
|
390
|
+
else:
|
|
391
|
+
# Default processor function if no script is provided
|
|
392
|
+
compiled_script = None
|
|
393
|
+
|
|
394
|
+
receiver = JSONReceiver(
|
|
395
|
+
host=args.host,
|
|
396
|
+
port=args.port,
|
|
397
|
+
api_key=args.api_key,
|
|
398
|
+
require_auth=args.require_auth,
|
|
399
|
+
title=args.title,
|
|
400
|
+
processor_func=compiled_script,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
# Start the server
|
|
404
|
+
receiver.start(background=False)
|
|
405
|
+
|
|
406
|
+
if __name__ == "__main__":
|
|
407
|
+
go()
|
|
@@ -1011,7 +1011,7 @@ def main():
|
|
|
1011
1011
|
|
|
1012
1012
|
print(f"Starting TalkPipe server at http://{args.host}:{args.port}")
|
|
1013
1013
|
uvicorn.run(
|
|
1014
|
-
"talkpipe.app.
|
|
1014
|
+
"talkpipe.app.chatterlang_server:app",
|
|
1015
1015
|
host=args.host,
|
|
1016
1016
|
port=args.port,
|
|
1017
1017
|
reload=args.reload
|