raglab_bak 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raglab_bak-0.1.3/.github/workflows/cd.yaml +43 -0
- raglab_bak-0.1.3/.github/workflows/ci.yml +48 -0
- raglab_bak-0.1.3/.gitignore +118 -0
- raglab_bak-0.1.3/LICENSE +21 -0
- raglab_bak-0.1.3/PKG-INFO +71 -0
- raglab_bak-0.1.3/README.md +48 -0
- raglab_bak-0.1.3/docsrc/.gitignore +1 -0
- raglab_bak-0.1.3/docsrc/Makefile +33 -0
- raglab_bak-0.1.3/docsrc/conf.py +82 -0
- raglab_bak-0.1.3/docsrc/index.rst +18 -0
- raglab_bak-0.1.3/docsrc/make.bat +35 -0
- raglab_bak-0.1.3/misc/ad_raglab_streamline.ipynb +1609 -0
- raglab_bak-0.1.3/misc/data_ad/QA/eval1.json +29 -0
- raglab_bak-0.1.3/misc/data_ad/QA/mix_wiki.json +9 -0
- raglab_bak-0.1.3/misc/data_ad/QA/story_qa.json +13 -0
- raglab_bak-0.1.3/misc/data_ad/ragas_dataset.json +1 -0
- raglab_bak-0.1.3/misc/data_ad/ragas_dataset_char.json +1 -0
- raglab_bak-0.1.3/misc/data_ad/ragas_results_char.csv +65 -0
- raglab_bak-0.1.3/misc/data_ad/ragas_results_char_600.csv +67 -0
- raglab_bak-0.1.3/misc/data_ad/ragas_results_semantic.csv +51 -0
- raglab_bak-0.1.3/misc/data_ad/ragas_results_semantic_600.csv +59 -0
- raglab_bak-0.1.3/misc/data_ad/test_documents/Contrat_ONE1406.pdf +0 -0
- raglab_bak-0.1.3/misc/data_ad/test_documents/contract1.txt +64 -0
- raglab_bak-0.1.3/misc/data_ad/test_documents/mix_wiki.txt +13 -0
- raglab_bak-0.1.3/misc/data_ad/test_documents/story.txt +11 -0
- raglab_bak-0.1.3/misc/data_ad/test_documents/writers.txt +21 -0
- raglab_bak-0.1.3/misc/embeddings_metadata_story.tsv +25 -0
- raglab_bak-0.1.3/misc/embeddings_story.tsv +25 -0
- raglab_bak-0.1.3/misc/thomas/chunking.ipynb +692 -0
- raglab_bak-0.1.3/misc/thomas/example.py +132 -0
- raglab_bak-0.1.3/misc/tw_raglab_wip.ipynb +1427 -0
- raglab_bak-0.1.3/misc/tw_scrap.ipynb +1408 -0
- raglab_bak-0.1.3/misscutchar_pca.png +0 -0
- raglab_bak-0.1.3/pyproject.toml +169 -0
- raglab_bak-0.1.3/raglab_bak/Dockerfile +12 -0
- raglab_bak-0.1.3/raglab_bak/__init__.py +3 -0
- raglab_bak-0.1.3/raglab_bak/audio.py +18 -0
- raglab_bak-0.1.3/raglab_bak/fiddle_manager/service.py +32 -0
- raglab_bak-0.1.3/raglab_bak/mermaid/prompt_editor.md +13 -0
- raglab_bak-0.1.3/raglab_bak/raglab2.py +174 -0
- raglab_bak-0.1.3/raglab_bak/retrieval/VectorDB.py +163 -0
- raglab_bak-0.1.3/raglab_bak/retrieval/__init__.py +0 -0
- raglab_bak-0.1.3/raglab_bak/retrieval/chroma_client.py +31 -0
- raglab_bak-0.1.3/raglab_bak/retrieval/lib_alexis.py +429 -0
- raglab_bak-0.1.3/raglab_bak/retrieval/segmentation_lib.py +262 -0
- raglab_bak-0.1.3/raglab_bak/stores/__init__.py +1 -0
- raglab_bak-0.1.3/raglab_bak/stores/function_store.py +71 -0
- raglab_bak-0.1.3/raglab_bak/stores/repos.py +39 -0
- raglab_bak-0.1.3/raglab_bak/stores/simple_stores.py +64 -0
- raglab_bak-0.1.3/raglab_bak/stores/sql_app_stores.py +67 -0
- raglab_bak-0.1.3/raglab_bak/stores/stores_util.py +128 -0
- raglab_bak-0.1.3/raglab_bak/stores/user_store.py +9 -0
- raglab_bak-0.1.3/raglab_bak/suggestion_engine/__init__.py +0 -0
- raglab_bak-0.1.3/raglab_bak/suggestion_engine/suggestion_example.ipynb +317 -0
- raglab_bak-0.1.3/raglab_bak/suggestion_engine/suggestion_lib.py +156 -0
- raglab_bak-0.1.3/raglab_bak/templates/chat.html +131 -0
- raglab_bak-0.1.3/raglab_bak/templates/gate.html +31 -0
- raglab_bak-0.1.3/raglab_bak/templates/login.html +37 -0
- raglab_bak-0.1.3/raglab_bak/templates/raglab_home.html +33 -0
- raglab_bak-0.1.3/raglab_bak/templates/raglab_query.html +69 -0
- raglab_bak-0.1.3/raglab_bak/templates/raglab_query_result.html +42 -0
- raglab_bak-0.1.3/raglab_bak/tests/test.db +0 -0
- raglab_bak-0.1.3/raglab_bak/tests/test_app.py +65 -0
- raglab_bak-0.1.3/raglab_bak/tests/test_repo.py +0 -0
- raglab_bak-0.1.3/raglab_bak/tests/test_stores_util.py +60 -0
- raglab_bak-0.1.3/raglab_bak/transformations.py +25 -0
- raglab_bak-0.1.3/raglab_bak/util.py +217 -0
- raglab_bak-0.1.3/raglab_bak/web_services/__init__.py +1 -0
- raglab_bak-0.1.3/raglab_bak/web_services/mockups.py +143 -0
- raglab_bak-0.1.3/raglab_bak/web_services/prompt_pilote.py +73 -0
- raglab_bak-0.1.3/raglab_bak/web_services/prompt_templates_dde.py +110 -0
- raglab_bak-0.1.3/raglab_bak/web_services/store_access.py +47 -0
- raglab_bak-0.1.3/raglab_bak/wips/__init__.py +1 -0
- raglab_bak-0.1.3/raglab_bak/wips/raglab_wips.ipynb +299 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/__init__.py +10 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/components.py +88 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/config.py +31 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/dev_instructions_1.md +843 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/dev_notes.md +6 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/interface.py +78 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/langchain_integration.py +329 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/registry.py +71 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/tests/__init__.py +1 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/tests/test_basic.py +23 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/tests/test_integration.py +207 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/tests/test_qdrant_integration.py +62 -0
- raglab_bak-0.1.3/raglab_bak/wips/vd_facade_1/types.py +73 -0
- raglab_bak-0.1.3/raglab_bak/wips/vectordb_discovery.py +490 -0
- raglab_bak-0.1.3/raglab_bak/wips/vectordb_manager.py +1162 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ "main", "dev", "test" ]
|
|
6
|
+
|
|
7
|
+
env:
|
|
8
|
+
# Use docker.io for Docker Hub if empty
|
|
9
|
+
REGISTRY: ghcr.io
|
|
10
|
+
IMAGE_NAME: ${{ github.actor }}/raglab_api:${{github.GITHUB_REF_NAME}}
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
publish:
|
|
14
|
+
name: publish image
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v3
|
|
19
|
+
- name: Login
|
|
20
|
+
run: |
|
|
21
|
+
echo ${{ secrets.PAT }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
|
|
22
|
+
- name: Build and Publish
|
|
23
|
+
run: |
|
|
24
|
+
docker build . --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
|
25
|
+
docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
|
26
|
+
|
|
27
|
+
deploy:
|
|
28
|
+
needs: publish
|
|
29
|
+
name: deploy image
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
|
|
32
|
+
steps:
|
|
33
|
+
- name: install ssh keys
|
|
34
|
+
# check this thread to understand why its needed:
|
|
35
|
+
# <https://stackoverflow.com/a/70447517>
|
|
36
|
+
run: |
|
|
37
|
+
install -m 600 -D /dev/null ~/.ssh/id_rsa
|
|
38
|
+
echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
|
|
39
|
+
ssh-keyscan -H ${{ secrets.SSH_HOST }} > ~/.ssh/known_hosts
|
|
40
|
+
- name: connect and pull
|
|
41
|
+
run: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "cd ${{ secrets.WORK_DIR }} && docker compose pull && docker compose up -d && exit"
|
|
42
|
+
- name: cleanup
|
|
43
|
+
run: rm -rf ~/.ssh
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# wads CI — calls the reusable workflow hosted in i2mint/wads.
|
|
2
|
+
#
|
|
3
|
+
# All configuration comes from this repo's pyproject.toml [tool.wads.ci.*].
|
|
4
|
+
# To customize the workflow itself (rare), replace this file with the
|
|
5
|
+
# full inline template `wads/data/github_ci_uv.yml` from i2mint/wads.
|
|
6
|
+
#
|
|
7
|
+
# Pinning: `@master` floats with wads. If you need version stability for
|
|
8
|
+
# a release-sensitive repo, change `@master` to a wads tag (e.g. `@v0.1.81`).
|
|
9
|
+
# CI failure does not block a published release — it blocks the publish
|
|
10
|
+
# step itself — so floating master is generally safe.
|
|
11
|
+
#
|
|
12
|
+
# Permissions: GitHub validates that the caller grants AT LEAST the
|
|
13
|
+
# permissions any job in the called workflow requests — at workflow-parse
|
|
14
|
+
# time, not at run-time, even if the job would be skipped via `if:`.
|
|
15
|
+
# The reusable workflow needs:
|
|
16
|
+
# contents: write for the publish job's version-bump push-back
|
|
17
|
+
# and for the github-pages job's gh-pages branch push
|
|
18
|
+
# pages: write for the github-pages job's REST API Pages config
|
|
19
|
+
# Both default to `write` on org-account GITHUB_TOKEN and need to be
|
|
20
|
+
# granted explicitly on personal-account callers (where the default is
|
|
21
|
+
# read-only). No `id-token: write` needed — the publish-github-pages
|
|
22
|
+
# action uses peaceiris/actions-gh-pages (branch-based) + REST API,
|
|
23
|
+
# not the OIDC `actions/deploy-pages` flow.
|
|
24
|
+
name: Continuous Integration
|
|
25
|
+
on: [push, pull_request]
|
|
26
|
+
jobs:
|
|
27
|
+
ci:
|
|
28
|
+
uses: i2mint/wads/.github/workflows/uv-ci.yml@master
|
|
29
|
+
permissions:
|
|
30
|
+
contents: write
|
|
31
|
+
pages: write
|
|
32
|
+
# Explicit pass-through (not `secrets: inherit`) because `inherit` does
|
|
33
|
+
# not reliably propagate caller-repo secrets to a reusable workflow owned
|
|
34
|
+
# by a different account (verified empirically: personal-account caller +
|
|
35
|
+
# i2mint-org workflow → `${{ secrets.PYPI_PASSWORD }}` resolved to empty).
|
|
36
|
+
#
|
|
37
|
+
# This list is the per-repo *transport*: it should contain PYPI_PASSWORD
|
|
38
|
+
# (for publishing) plus every secret your tests/CI need. It is generated
|
|
39
|
+
# from [tool.wads.ci.env] in pyproject.toml. To add one, run
|
|
40
|
+
# wads-secrets add VAR_NAME # updates pyproject + this block
|
|
41
|
+
# or just append a line below. *Which* of these become job env vars (and
|
|
42
|
+
# which are required) is controlled by [tool.wads.ci.env] — passing a
|
|
43
|
+
# secret here does not by itself put it in the environment.
|
|
44
|
+
#
|
|
45
|
+
# A secret name must also be declared in the reusable workflow's superset
|
|
46
|
+
# (wads/ci_secrets.py). `wads-secrets add` warns if it is not.
|
|
47
|
+
secrets:
|
|
48
|
+
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
wads_configs.json
|
|
2
|
+
data/wads_configs.json
|
|
3
|
+
wads/data/wads_configs.json
|
|
4
|
+
|
|
5
|
+
# Byte-compiled / optimized / DLL files
|
|
6
|
+
__pycache__/
|
|
7
|
+
*.py[cod]
|
|
8
|
+
*$py.class
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
.DS_Store
|
|
12
|
+
# C extensions
|
|
13
|
+
*.so
|
|
14
|
+
|
|
15
|
+
# Distribution / packaging
|
|
16
|
+
.Python
|
|
17
|
+
build/
|
|
18
|
+
develop-eggs/
|
|
19
|
+
dist/
|
|
20
|
+
downloads/
|
|
21
|
+
eggs/
|
|
22
|
+
.eggs/
|
|
23
|
+
lib/
|
|
24
|
+
lib64/
|
|
25
|
+
parts/
|
|
26
|
+
sdist/
|
|
27
|
+
var/
|
|
28
|
+
wheels/
|
|
29
|
+
*.egg-info/
|
|
30
|
+
.installed.cfg
|
|
31
|
+
*.egg
|
|
32
|
+
MANIFEST
|
|
33
|
+
_build
|
|
34
|
+
|
|
35
|
+
# PyInstaller
|
|
36
|
+
# Usually these files are written by a python script from a template
|
|
37
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
38
|
+
*.manifest
|
|
39
|
+
*.spec
|
|
40
|
+
|
|
41
|
+
# Installer logs
|
|
42
|
+
pip-log.txt
|
|
43
|
+
pip-delete-this-directory.txt
|
|
44
|
+
|
|
45
|
+
# Unit test / coverage reports
|
|
46
|
+
htmlcov/
|
|
47
|
+
.tox/
|
|
48
|
+
.coverage
|
|
49
|
+
.coverage.*
|
|
50
|
+
.cache
|
|
51
|
+
nosetests.xml
|
|
52
|
+
coverage.xml
|
|
53
|
+
*.cover
|
|
54
|
+
.hypothesis/
|
|
55
|
+
.pytest_cache/
|
|
56
|
+
|
|
57
|
+
# Translations
|
|
58
|
+
*.mo
|
|
59
|
+
*.pot
|
|
60
|
+
|
|
61
|
+
# Django stuff:
|
|
62
|
+
*.log
|
|
63
|
+
local_settings.py
|
|
64
|
+
db.sqlite3
|
|
65
|
+
|
|
66
|
+
# Flask stuff:
|
|
67
|
+
instance/
|
|
68
|
+
.webassets-cache
|
|
69
|
+
|
|
70
|
+
# Scrapy stuff:
|
|
71
|
+
.scrapy
|
|
72
|
+
|
|
73
|
+
# Sphinx documentation
|
|
74
|
+
docs/_build/
|
|
75
|
+
docs/*
|
|
76
|
+
|
|
77
|
+
# PyBuilder
|
|
78
|
+
target/
|
|
79
|
+
|
|
80
|
+
# Jupyter Notebook
|
|
81
|
+
.ipynb_checkpoints
|
|
82
|
+
|
|
83
|
+
# pyenv
|
|
84
|
+
.python-version
|
|
85
|
+
|
|
86
|
+
# celery beat schedule file
|
|
87
|
+
celerybeat-schedule
|
|
88
|
+
|
|
89
|
+
# SageMath parsed files
|
|
90
|
+
*.sage.py
|
|
91
|
+
|
|
92
|
+
# Environments
|
|
93
|
+
.env
|
|
94
|
+
.venv
|
|
95
|
+
env/
|
|
96
|
+
venv/
|
|
97
|
+
ENV/
|
|
98
|
+
env.bak/
|
|
99
|
+
venv.bak/
|
|
100
|
+
|
|
101
|
+
# Spyder project settings
|
|
102
|
+
.spyderproject
|
|
103
|
+
.spyproject
|
|
104
|
+
|
|
105
|
+
# Rope project settings
|
|
106
|
+
.ropeproject
|
|
107
|
+
|
|
108
|
+
# mkdocs documentation
|
|
109
|
+
/site
|
|
110
|
+
|
|
111
|
+
# mypy
|
|
112
|
+
.mypy_cache/
|
|
113
|
+
|
|
114
|
+
# PyCharm
|
|
115
|
+
.idea
|
|
116
|
+
raglab_bak/build_and_run
|
|
117
|
+
raglab_bak/run_container
|
|
118
|
+
misc/thomas.ipynb
|
raglab_bak-0.1.3/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) [year] [fullname]
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: raglab_bak
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: Backend of a system to explore RAG operations (formerly published as raglab)
|
|
5
|
+
Project-URL: Homepage, https://github.com/addaix/raglab_bak
|
|
6
|
+
Project-URL: Repository, https://github.com/addaix/raglab_bak
|
|
7
|
+
Author: addaix
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: config2py
|
|
12
|
+
Requires-Dist: dill
|
|
13
|
+
Requires-Dist: dol
|
|
14
|
+
Requires-Dist: i2
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
17
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
18
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
19
|
+
Provides-Extra: docs
|
|
20
|
+
Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
|
|
21
|
+
Requires-Dist: sphinx>=6.0; extra == 'docs'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# raglab_bak
|
|
25
|
+
|
|
26
|
+
Backend of a system to explore RAG operations.
|
|
27
|
+
|
|
28
|
+
(Formerly published as `raglab`; renamed to `raglab_bak` — "raglab backend" — to
|
|
29
|
+
free the `raglab` name for a new project. Import as `raglab_bak`.)
|
|
30
|
+
|
|
31
|
+
To install: ```pip install raglab_bak```
|
|
32
|
+
|
|
33
|
+
**IMPORTANT NOTE: This is work-in-progress and shouldn't be used, yet.**
|
|
34
|
+
|
|
35
|
+
`raglab_bak` is meant to be a medley of interoperable functionalities around retrieval
|
|
36
|
+
(or "search") and generation (with LLMs).
|
|
37
|
+
It's architecture is structured so that operations can be done in python,
|
|
38
|
+
at the command line, through a web service (or any OpenAPI language binders) or from a
|
|
39
|
+
GUI, where each layer's interface is mapped from the other in a consistent
|
|
40
|
+
(and usually automatic) manner.
|
|
41
|
+
|
|
42
|
+
# Work-in-progress notes
|
|
43
|
+
|
|
44
|
+
## Simple resources CRUD
|
|
45
|
+
|
|
46
|
+
Some functionalities/operations can simply be described as
|
|
47
|
+
(1) a set of operations
|
|
48
|
+
(2) over a kind of data
|
|
49
|
+
|
|
50
|
+
We'll sometimes lazily say CRUD for the "set of operations", but we don't always mean Create-Read-Update-Delete. In fact, most of the time we don't mean those four operations. Instead, since python is our backend language and we [aim at using base interfaces](https://medium.com/@thorwhalen1/mastering-modular-design-the-art-of-using-base-interface-facades-9d4a08d550d1) as much as possible, our "set of operations" will often be inspired from python's [Collections Abstract Base Classes](https://docs.python.org/3/library/collections.abc.html#collections-abstract-base-classes).
|
|
51
|
+
|
|
52
|
+
So many concrete use cases of this abstraction are parametrized by
|
|
53
|
+
* the set of operation (example, read, write, list, delete...)
|
|
54
|
+
* a name designating the kind/type of data
|
|
55
|
+
* optionally a schema that describes what the structure and contents of that kind of data must be
|
|
56
|
+
|
|
57
|
+
Further, by making the interface consistent (for example, and preferably, with the interfaces of [Collections Abstract Base Classes](https://docs.python.org/3/library/collections.abc.html#collections-abstract-base-classes)), it makes it possible to automate how the python interface maps to a REST API interface (see
|
|
58
|
+
[Mapping python collections builtin methods to REST API patterns](https://github.com/i2mint/py2http/discussions/9))
|
|
59
|
+
and/or a UI interface (see [Collections Operations UI Design Patterns](https://github.com/i2mint/oui/discussions/14)).
|
|
60
|
+
|
|
61
|
+
### Examples
|
|
62
|
+
|
|
63
|
+
* `permissions` (keyed by users, groups of users, projects etc.)
|
|
64
|
+
* `datasets` (again keyed by users etc.)
|
|
65
|
+
* `prompt_templates`
|
|
66
|
+
* `embedders`
|
|
67
|
+
* `dataset_query_results`
|
|
68
|
+
* `query_results_aggregators`
|
|
69
|
+
* `query_results_aggregates`
|
|
70
|
+
* `tag`: For example `tag, dataset`, `tag, group_of_datasets`, `tag, anything_really`. Note that tagging, or annotating can be applied to any kind of data, and is a dual of "grouping", and subsumes hierarchical organization. See [Persisting groups of items (tagging stuff)](https://github.com/i2mint/i2mint/blob/main/misc/Persisting%20groups%20of%20items%20(tagging%20stuff).ipynb) for designs regarding this.
|
|
71
|
+
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# raglab_bak
|
|
2
|
+
|
|
3
|
+
Backend of a system to explore RAG operations.
|
|
4
|
+
|
|
5
|
+
(Formerly published as `raglab`; renamed to `raglab_bak` — "raglab backend" — to
|
|
6
|
+
free the `raglab` name for a new project. Import as `raglab_bak`.)
|
|
7
|
+
|
|
8
|
+
To install: ```pip install raglab_bak```
|
|
9
|
+
|
|
10
|
+
**IMPORTANT NOTE: This is work-in-progress and shouldn't be used, yet.**
|
|
11
|
+
|
|
12
|
+
`raglab_bak` is meant to be a medley of interoperable functionalities around retrieval
|
|
13
|
+
(or "search") and generation (with LLMs).
|
|
14
|
+
It's architecture is structured so that operations can be done in python,
|
|
15
|
+
at the command line, through a web service (or any OpenAPI language binders) or from a
|
|
16
|
+
GUI, where each layer's interface is mapped from the other in a consistent
|
|
17
|
+
(and usually automatic) manner.
|
|
18
|
+
|
|
19
|
+
# Work-in-progress notes
|
|
20
|
+
|
|
21
|
+
## Simple resources CRUD
|
|
22
|
+
|
|
23
|
+
Some functionalities/operations can simply be described as
|
|
24
|
+
(1) a set of operations
|
|
25
|
+
(2) over a kind of data
|
|
26
|
+
|
|
27
|
+
We'll sometimes lazily say CRUD for the "set of operations", but we don't always mean Create-Read-Update-Delete. In fact, most of the time we don't mean those four operations. Instead, since python is our backend language and we [aim at using base interfaces](https://medium.com/@thorwhalen1/mastering-modular-design-the-art-of-using-base-interface-facades-9d4a08d550d1) as much as possible, our "set of operations" will often be inspired from python's [Collections Abstract Base Classes](https://docs.python.org/3/library/collections.abc.html#collections-abstract-base-classes).
|
|
28
|
+
|
|
29
|
+
So many concrete use cases of this abstraction are parametrized by
|
|
30
|
+
* the set of operation (example, read, write, list, delete...)
|
|
31
|
+
* a name designating the kind/type of data
|
|
32
|
+
* optionally a schema that describes what the structure and contents of that kind of data must be
|
|
33
|
+
|
|
34
|
+
Further, by making the interface consistent (for example, and preferably, with the interfaces of [Collections Abstract Base Classes](https://docs.python.org/3/library/collections.abc.html#collections-abstract-base-classes)), it makes it possible to automate how the python interface maps to a REST API interface (see
|
|
35
|
+
[Mapping python collections builtin methods to REST API patterns](https://github.com/i2mint/py2http/discussions/9))
|
|
36
|
+
and/or a UI interface (see [Collections Operations UI Design Patterns](https://github.com/i2mint/oui/discussions/14)).
|
|
37
|
+
|
|
38
|
+
### Examples
|
|
39
|
+
|
|
40
|
+
* `permissions` (keyed by users, groups of users, projects etc.)
|
|
41
|
+
* `datasets` (again keyed by users etc.)
|
|
42
|
+
* `prompt_templates`
|
|
43
|
+
* `embedders`
|
|
44
|
+
* `dataset_query_results`
|
|
45
|
+
* `query_results_aggregators`
|
|
46
|
+
* `query_results_aggregates`
|
|
47
|
+
* `tag`: For example `tag, dataset`, `tag, group_of_datasets`, `tag, anything_really`. Note that tagging, or annotating can be applied to any kind of data, and is a dual of "grouping", and subsumes hierarchical organization. See [Persisting groups of items (tagging stuff)](https://github.com/i2mint/i2mint/blob/main/misc/Persisting%20groups%20of%20items%20(tagging%20stuff).ipynb) for designs regarding this.
|
|
48
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
_build/
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Minimal makefile for Sphinx documentation
|
|
2
|
+
#
|
|
3
|
+
|
|
4
|
+
# You can set these variables from the command line, and also
|
|
5
|
+
# from the environment for the first two.
|
|
6
|
+
SPHINXOPTS ?=
|
|
7
|
+
SPHINXBUILD ?= sphinx-build
|
|
8
|
+
SOURCEDIR = .
|
|
9
|
+
BUILDDIR = _build
|
|
10
|
+
GITHUBPAGESDIR = ../docs
|
|
11
|
+
GITLABPAGESDIR = ../public
|
|
12
|
+
|
|
13
|
+
# Put it first so that "make" without argument is like "make help".
|
|
14
|
+
help:
|
|
15
|
+
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
|
16
|
+
|
|
17
|
+
.PHONY: help Makefile
|
|
18
|
+
|
|
19
|
+
github:
|
|
20
|
+
@make html
|
|
21
|
+
@cp -a "$(BUILDDIR)"/html/. $(GITHUBPAGESDIR)
|
|
22
|
+
|
|
23
|
+
gitlab:
|
|
24
|
+
@make html
|
|
25
|
+
@cp -a "$(BUILDDIR)"/html/. $(GITLABPAGESDIR)
|
|
26
|
+
|
|
27
|
+
clean:
|
|
28
|
+
rm -rfv $(BUILDDIR) $(GITHUBPAGESDIR) $(GITLABPAGESDIR)
|
|
29
|
+
|
|
30
|
+
# Catch-all target: route all unknown targets to Sphinx using the new
|
|
31
|
+
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
|
32
|
+
%: Makefile
|
|
33
|
+
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# Configuration file for the Sphinx documentation builder.
|
|
2
|
+
#
|
|
3
|
+
# This file only contains a selection of the most common options.
|
|
4
|
+
# For a full list see the documentation:
|
|
5
|
+
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
|
6
|
+
|
|
7
|
+
# -- Path setup --------------------------------------------------------------
|
|
8
|
+
|
|
9
|
+
# If extensions (or modules to document with autodoc) are in another directory,
|
|
10
|
+
# add these directories to sys.path here. If the directory is relative to the
|
|
11
|
+
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
sys.path.insert(0, os.path.abspath(".."))
|
|
17
|
+
|
|
18
|
+
# -- Project information -----------------------------------------------------
|
|
19
|
+
from epythet.config_parser import parse_config
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
project, copyright, author, release, display_name = parse_config(
|
|
23
|
+
Path(__file__).absolute().parent.parent / "pyproject.toml"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# -- General configuration ---------------------------------------------------
|
|
27
|
+
|
|
28
|
+
# Add any Sphinx extension module names here, as strings. They can be
|
|
29
|
+
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
|
30
|
+
# ones.
|
|
31
|
+
extensions = [
|
|
32
|
+
"sphinx_toggleprompt",
|
|
33
|
+
"sphinx_copybutton",
|
|
34
|
+
"sphinx.ext.autodoc", # Include documentation from docstrings
|
|
35
|
+
"sphinx.ext.doctest", # Test snippets in the documentation
|
|
36
|
+
"sphinx.ext.githubpages", # This extension creates .nojekyll file
|
|
37
|
+
"sphinx.ext.graphviz", # Add Graphviz graphs
|
|
38
|
+
"sphinx.ext.napoleon", # Support for NumPy and Google style docstrings
|
|
39
|
+
"sphinx.ext.todo", # Support for todo items
|
|
40
|
+
"sphinx.ext.viewcode", # Add links to highlighted source code
|
|
41
|
+
"myst_parser", # Parse .md files
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
# Add any paths that contain templates here, relative to this directory.
|
|
45
|
+
templates_path = ["_templates"]
|
|
46
|
+
|
|
47
|
+
# List of patterns, relative to source directory, that match files and
|
|
48
|
+
# directories to ignore when looking for source files.
|
|
49
|
+
# This pattern also affects html_static_path and html_extra_path.
|
|
50
|
+
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
|
51
|
+
|
|
52
|
+
# -- Options for HTML output -------------------------------------------------
|
|
53
|
+
|
|
54
|
+
# The theme to use for HTML and HTML Help pages. See the documentation for
|
|
55
|
+
# a list of builtin themes.
|
|
56
|
+
#
|
|
57
|
+
html_theme = "sphinx_rtd_theme"
|
|
58
|
+
|
|
59
|
+
# Add any paths that contain custom static files (such as style sheets) here,
|
|
60
|
+
# relative to this directory. They are copied after the builtin static files,
|
|
61
|
+
# so a file named "default.css" will overwrite the builtin "default.css".
|
|
62
|
+
html_static_path = ["_static"]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# -- Options for Markdown support -------------------------------------------
|
|
66
|
+
# TODO: fix md support so that it doesn't interfere with rst docs
|
|
67
|
+
# import commonmark
|
|
68
|
+
#
|
|
69
|
+
#
|
|
70
|
+
# def docstring(app, what, name, obj, options, lines):
|
|
71
|
+
# md = '\n'.join(lines)
|
|
72
|
+
# ast = commonmark.Parser().parse(md)
|
|
73
|
+
# rst = commonmark.ReStructuredTextRenderer().render(ast)
|
|
74
|
+
# lines.clear()
|
|
75
|
+
# lines += rst.splitlines()
|
|
76
|
+
#
|
|
77
|
+
#
|
|
78
|
+
# def setup(app):
|
|
79
|
+
# app.connect('autodoc-process-docstring', docstring)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
toggleprompt_offset_right = 30
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Welcome to raglab_bak's documentation!
|
|
2
|
+
======================================
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
.. include:: ./table_of_contents.rst
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
Indices and tables
|
|
9
|
+
==================
|
|
10
|
+
|
|
11
|
+
* :ref:`genindex`
|
|
12
|
+
* :ref:`modindex`
|
|
13
|
+
* :ref:`search`
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
Release: |release|
|
|
17
|
+
|
|
18
|
+
Last change: |today|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
@ECHO OFF
|
|
2
|
+
|
|
3
|
+
pushd %~dp0
|
|
4
|
+
|
|
5
|
+
REM Command file for Sphinx documentation
|
|
6
|
+
|
|
7
|
+
if "%SPHINXBUILD%" == "" (
|
|
8
|
+
set SPHINXBUILD=sphinx-build
|
|
9
|
+
)
|
|
10
|
+
set SOURCEDIR=.
|
|
11
|
+
set BUILDDIR=_build
|
|
12
|
+
|
|
13
|
+
if "%1" == "" goto help
|
|
14
|
+
|
|
15
|
+
%SPHINXBUILD% >NUL 2>NUL
|
|
16
|
+
if errorlevel 9009 (
|
|
17
|
+
echo.
|
|
18
|
+
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
|
19
|
+
echo.installed, then set the SPHINXBUILD environment variable to point
|
|
20
|
+
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
|
21
|
+
echo.may add the Sphinx directory to PATH.
|
|
22
|
+
echo.
|
|
23
|
+
echo.If you don't have Sphinx installed, grab it from
|
|
24
|
+
echo.http://sphinx-doc.org/
|
|
25
|
+
exit /b 1
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
|
29
|
+
goto end
|
|
30
|
+
|
|
31
|
+
:help
|
|
32
|
+
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
|
33
|
+
|
|
34
|
+
:end
|
|
35
|
+
popd
|