MetaList 0.1.6__tar.gz → 0.1.7__tar.gz

MetaList-0.1.7/MetaList.egg-info/PKG-INFO

@@ -0,0 +1,110 @@
+Metadata-Version: 2.1
+Name: MetaList
+Version: 0.1.7
+Summary: A PKM web app integrated with LLMs
+Home-page: https://github.com/evolvingstuff/MetaList
+Author: Thomas Lahore
+Author-email: tom.lahore@gmail.com
+Classifier: Programming Language :: Python :: 3.8
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: bottle==0.12.25
+Requires-Dist: bottle-sqlite==0.2.0
+Requires-Dist: beautifulsoup4~=4.12.2
+Requires-Dist: tqdm==4.66.1
+
+
+# MetaList
+
+<img src="docs/img/MetaList-Logo.png" width="130" />
+
+[![PyPI](https://img.shields.io/pypi/v/metalist)](https://pypi.org/project/metalist/)
+![PyPI - Python Version](https://img.shields.io/badge/python-3.8+-blue)
+![Runs on Linux | MacOS | Windows](https://img.shields.io/badge/runs%20on-Linux%20%7C%20MacOS%20%7C%20Windows-blue)
+
+
+## What is MetaList?
+MetaList is a personal knowledge management web application, 
+tightly integrated with large language models.
+
+> ⚠️ **MetaList is currently in Development**: Expect breaking changes and bugs
+
+## Features
+
+- [x] Chat feature: LLM integration (currently only [OpenAI API](https://openai.com/blog/openai-api))
+- [x] [Chart.js](https://www.chartjs.org/) integration for LLM-generated visualizations
+- [x] Hierarchical, collapsable note structure
+- [x] Reordering notes in one context survives to other search contexts; notes maintain a [total order](https://en.wikipedia.org/wiki/Total_order)
+- [x] Quick, responsive searching (tags, full-text, or combo of both)
+- [x] Search suggestions
+- [x] Tagging
+- [x] Tag suggestions (uses a [Jaccard index](https://en.wikipedia.org/wiki/Jaccard_index))
+- [x] Tags can imply other tags
+- [x] Tag implication rules are also notes; they can themselves be tagged and searched
+- [x] Nested subitems in notes inherit tags from their parents
+- [x] [Markdown](https://markdown-it.github.io/)/[LaTeX](https://katex.org/) support
+- [x] Infinite undo/redo
+- [x] Infinite scrolling
+
+###  Chat Feature Example
+
+Search filters notes down to a relevant subset (a "search context"), which has the nice 
+effect of acting as a form of human-guided [retrieval augmented generation](https://www.promptingguide.ai/techniques/rag).
+The chat feature is fully integrated with the underlying notes. Mousing over a reference in the chat will highlight 
+the referenced note, if it is within view, and clicking on the reference will navigate to and select the note, 
+even opening up collapsed items in the note hierarchy so that all the relevant info becomes visible.
+
+Some examples of interactions with the chat feature (in this case, recommending new movies):
+
+<img src="docs/img/demo-1.gif" width="800"/>
+<img src="docs/img/chat-example-2.png" width="800"/>
+<!--<img src="docs/img/chat-example-3.png" width="800"/>-->
+
+## Installation
+
+It is recommended that you create a Python virtual environment before installing MetaList.
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+Then, install MetaList using pip:
+
+```bash
+pip install metalist
+```
+
+## Usage
+
+From the installation directory, run:
+```
+metalist
+```
+
+Navigate to `http://127.0.0.0:8080/`
+
+Read the [cheatsheet](docs/cheatsheet.md).
+
+## Roadmap
+
+- [ ] Chat integration with other models
+- [ ] Menus / Settings
+- [ ] Reminders
+- [ ] Passwords / encryption at rest
+- [ ] Import data from browser bookmarks
+- [ ] Import data from other PKM apps
+- [ ] Export notes to JSON, Markdown, HTML, PDF
+- [ ] File handling
+- [ ] Access to local file system
+- [ ] Multi-user
+- [ ] Mobile friendly
+- [ ] More unit/integration testing
+
+## License
+
+MetaList is licensed under the MIT license. See [LICENSE](LICENSE) for more information.
+

{MetaList-0.1.6 → MetaList-0.1.7}/MetaList.egg-info/SOURCES.txt

@@ -36,10 +36,9 @@ metalist/static/js/components/items-list.js
 metalist/static/js/components/search-bar.js
 metalist/static/js/components/suggestions-list.js
 metalist/static/js/components/tags-bar.js
-metalist/static/js/misc/LLMs.js
-metalist/static/js/misc/formats.js
 metalist/static/js/misc/item-formatter.js
 metalist/static/js/misc/logging.js
+metalist/static/js/misc/parsing.js
 metalist/static/js/misc/scrolling.js
 metalist/static/js/misc/server-proxy.js
 metalist/static/js/misc/vdom.js
@@ -65,6 +64,7 @@ metalist/static/libs/voca/voca.min.js
 metalist/static/tests/unit-tests/run-tests.html
 metalist/static/tests/unit-tests/test-search-bar.js
 metalist/utils/__init__.py
+metalist/utils/chat_prompts.py
 metalist/utils/crud.py
 metalist/utils/decorate_single_item.py
 metalist/utils/find.py
@@ -73,6 +73,7 @@ metalist/utils/initialize.py
 metalist/utils/ontology.py
 metalist/utils/reporting.py
 metalist/utils/search_filters.py
+metalist/utils/search_index.py
 metalist/utils/search_suggestions.py
 metalist/utils/server.py
 metalist/utils/snapshots.py
@@ -80,5 +81,5 @@ metalist/utils/tags_suggestions.py
 metalist/utils/update_multiple_items.py
 metalist/utils/update_single_item.py
 metalist/utils/misc/__init__.py
-metalist/utils/misc/crypto.py
+metalist/utils/misc/crypto-test.py
 tests/test_utils.py

MetaList-0.1.7/PKG-INFO

@@ -0,0 +1,110 @@
+Metadata-Version: 2.1
+Name: MetaList
+Version: 0.1.7
+Summary: A PKM web app integrated with LLMs
+Home-page: https://github.com/evolvingstuff/MetaList
+Author: Thomas Lahore
+Author-email: tom.lahore@gmail.com
+Classifier: Programming Language :: Python :: 3.8
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: bottle==0.12.25
+Requires-Dist: bottle-sqlite==0.2.0
+Requires-Dist: beautifulsoup4~=4.12.2
+Requires-Dist: tqdm==4.66.1
+
+
+# MetaList
+
+<img src="docs/img/MetaList-Logo.png" width="130" />
+
+[![PyPI](https://img.shields.io/pypi/v/metalist)](https://pypi.org/project/metalist/)
+![PyPI - Python Version](https://img.shields.io/badge/python-3.8+-blue)
+![Runs on Linux | MacOS | Windows](https://img.shields.io/badge/runs%20on-Linux%20%7C%20MacOS%20%7C%20Windows-blue)
+
+
+## What is MetaList?
+MetaList is a personal knowledge management web application, 
+tightly integrated with large language models.
+
+> ⚠️ **MetaList is currently in Development**: Expect breaking changes and bugs
+
+## Features
+
+- [x] Chat feature: LLM integration (currently only [OpenAI API](https://openai.com/blog/openai-api))
+- [x] [Chart.js](https://www.chartjs.org/) integration for LLM-generated visualizations
+- [x] Hierarchical, collapsable note structure
+- [x] Reordering notes in one context survives to other search contexts; notes maintain a [total order](https://en.wikipedia.org/wiki/Total_order)
+- [x] Quick, responsive searching (tags, full-text, or combo of both)
+- [x] Search suggestions
+- [x] Tagging
+- [x] Tag suggestions (uses a [Jaccard index](https://en.wikipedia.org/wiki/Jaccard_index))
+- [x] Tags can imply other tags
+- [x] Tag implication rules are also notes; they can themselves be tagged and searched
+- [x] Nested subitems in notes inherit tags from their parents
+- [x] [Markdown](https://markdown-it.github.io/)/[LaTeX](https://katex.org/) support
+- [x] Infinite undo/redo
+- [x] Infinite scrolling
+
+###  Chat Feature Example
+
+Search filters notes down to a relevant subset (a "search context"), which has the nice 
+effect of acting as a form of human-guided [retrieval augmented generation](https://www.promptingguide.ai/techniques/rag).
+The chat feature is fully integrated with the underlying notes. Mousing over a reference in the chat will highlight 
+the referenced note, if it is within view, and clicking on the reference will navigate to and select the note, 
+even opening up collapsed items in the note hierarchy so that all the relevant info becomes visible.
+
+Some examples of interactions with the chat feature (in this case, recommending new movies):
+
+<img src="docs/img/demo-1.gif" width="800"/>
+<img src="docs/img/chat-example-2.png" width="800"/>
+<!--<img src="docs/img/chat-example-3.png" width="800"/>-->
+
+## Installation
+
+It is recommended that you create a Python virtual environment before installing MetaList.
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+Then, install MetaList using pip:
+
+```bash
+pip install metalist
+```
+
+## Usage
+
+From the installation directory, run:
+```
+metalist
+```
+
+Navigate to `http://127.0.0.0:8080/`
+
+Read the [cheatsheet](docs/cheatsheet.md).
+
+## Roadmap
+
+- [ ] Chat integration with other models
+- [ ] Menus / Settings
+- [ ] Reminders
+- [ ] Passwords / encryption at rest
+- [ ] Import data from browser bookmarks
+- [ ] Import data from other PKM apps
+- [ ] Export notes to JSON, Markdown, HTML, PDF
+- [ ] File handling
+- [ ] Access to local file system
+- [ ] Multi-user
+- [ ] Mobile friendly
+- [ ] More unit/integration testing
+
+## License
+
+MetaList is licensed under the MIT license. See [LICENSE](LICENSE) for more information.
+

MetaList-0.1.7/README.md

@@ -0,0 +1,92 @@
+
+# MetaList
+
+<img src="docs/img/MetaList-Logo.png" width="130" />
+
+[![PyPI](https://img.shields.io/pypi/v/metalist)](https://pypi.org/project/metalist/)
+![PyPI - Python Version](https://img.shields.io/badge/python-3.8+-blue)
+![Runs on Linux | MacOS | Windows](https://img.shields.io/badge/runs%20on-Linux%20%7C%20MacOS%20%7C%20Windows-blue)
+
+
+## What is MetaList?
+MetaList is a personal knowledge management web application, 
+tightly integrated with large language models.
+
+> ⚠️ **MetaList is currently in Development**: Expect breaking changes and bugs
+
+## Features
+
+- [x] Chat feature: LLM integration (currently only [OpenAI API](https://openai.com/blog/openai-api))
+- [x] [Chart.js](https://www.chartjs.org/) integration for LLM-generated visualizations
+- [x] Hierarchical, collapsable note structure
+- [x] Reordering notes in one context survives to other search contexts; notes maintain a [total order](https://en.wikipedia.org/wiki/Total_order)
+- [x] Quick, responsive searching (tags, full-text, or combo of both)
+- [x] Search suggestions
+- [x] Tagging
+- [x] Tag suggestions (uses a [Jaccard index](https://en.wikipedia.org/wiki/Jaccard_index))
+- [x] Tags can imply other tags
+- [x] Tag implication rules are also notes; they can themselves be tagged and searched
+- [x] Nested subitems in notes inherit tags from their parents
+- [x] [Markdown](https://markdown-it.github.io/)/[LaTeX](https://katex.org/) support
+- [x] Infinite undo/redo
+- [x] Infinite scrolling
+
+###  Chat Feature Example
+
+Search filters notes down to a relevant subset (a "search context"), which has the nice 
+effect of acting as a form of human-guided [retrieval augmented generation](https://www.promptingguide.ai/techniques/rag).
+The chat feature is fully integrated with the underlying notes. Mousing over a reference in the chat will highlight 
+the referenced note, if it is within view, and clicking on the reference will navigate to and select the note, 
+even opening up collapsed items in the note hierarchy so that all the relevant info becomes visible.
+
+Some examples of interactions with the chat feature (in this case, recommending new movies):
+
+<img src="docs/img/demo-1.gif" width="800"/>
+<img src="docs/img/chat-example-2.png" width="800"/>
+<!--<img src="docs/img/chat-example-3.png" width="800"/>-->
+
+## Installation
+
+It is recommended that you create a Python virtual environment before installing MetaList.
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+Then, install MetaList using pip:
+
+```bash
+pip install metalist
+```
+
+## Usage
+
+From the installation directory, run:
+```
+metalist
+```
+
+Navigate to `http://127.0.0.0:8080/`
+
+Read the [cheatsheet](docs/cheatsheet.md).
+
+## Roadmap
+
+- [ ] Chat integration with other models
+- [ ] Menus / Settings
+- [ ] Reminders
+- [ ] Passwords / encryption at rest
+- [ ] Import data from browser bookmarks
+- [ ] Import data from other PKM apps
+- [ ] Export notes to JSON, Markdown, HTML, PDF
+- [ ] File handling
+- [ ] Access to local file system
+- [ ] Multi-user
+- [ ] Mobile friendly
+- [ ] More unit/integration testing
+
+## License
+
+MetaList is licensed under the MIT license. See [LICENSE](LICENSE) for more information.
+

{MetaList-0.1.6 → MetaList-0.1.7}/metalist/app.py

@@ -1,12 +1,15 @@
 import os
-from bottle import Bottle, run, static_file, request
+from typing import List
+from bottle import Bottle, run, static_file, request, response
 import bottle_sqlite
-
-from metalist.config import reset_undo_stack_on_search, port
+import requests
+from metalist.config import reset_undo_stack_on_search, port, open_ai_url, open_ai_model, development_mode
+from metalist.utils.chat_prompts import build_initial_prompts, build_selection_prompt
 from metalist.utils.crud import get_database_path
 from metalist.utils.search_suggestions import calculate_search_suggestions
 from metalist.utils.server import get_request_context, \
-    generic_response, noop_response, error_response, filter_items, Context
+    generic_response, noop_response, error_response, filter_and_sort_items, Context, chat_response
+from metalist.utils.search_index import SearchIndex
 from metalist.utils.tags_suggestions import calculate_tags_suggestions
 from metalist.utils.initialize import initialize_cache
 from metalist.utils.snapshots import Snapshots, SnapshotFragment, Snapshot, compress_snapshots
@@ -16,6 +19,7 @@ from metalist.utils import update_single_item, update_multiple_items
 
 cache = {}
 snapshots = Snapshots()
+chat_history: List[dict] = list()
 
 app = Bottle()
 plugin = bottle_sqlite.Plugin(dbfile=get_database_path())
@@ -23,6 +27,8 @@ app.install(plugin)
 
 
 def run_app():
+    if development_mode:
+        print('DEV MODE = True')
     initialize_cache(cache)
     run(app, port=port, debug=False)
 
@@ -91,7 +97,7 @@ def todo(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_single_item.todo(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/todo', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -111,7 +117,7 @@ def done(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_single_item.done(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/done', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -131,7 +137,7 @@ def expand(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_single_item.expand(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/expand', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -151,7 +157,7 @@ def collapse(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_single_item.collapse(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/collapse', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -175,7 +181,7 @@ def delete_subitem(db):
             must_recalculate_ontology = update_multiple_items.remove_item(db, cache, context)
             if must_recalculate_ontology:
                 update_multiple_items.recalculate_ontology(db, cache, context)
-            filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+            filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
             snap_post = SnapshotFragment(cache, None)
             snapshot = Snapshot('/delete-subitem (item)', snap_pre, snap_post, context.item_subitem_id)
             snapshots.push(snapshot)
@@ -186,7 +192,7 @@ def delete_subitem(db):
             must_recalculate_ontology = update_single_item.delete_subitem(db, context, cache)
             if must_recalculate_ontology:
                 update_multiple_items.recalculate_ontology(db, cache, context)
-            filtered_items, reached_scroll_end = filter_items(cache, context)
+            filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
             snap_post = SnapshotFragment(cache, None)
             snapshot = Snapshot('/delete-subitem', snap_pre, snap_post, context.item_subitem_id)
             snapshots.push(snapshot)
@@ -227,7 +233,7 @@ def move_item_up(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_multiple_items.move_item_up(db, cache, context)
-        filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/move-item-up', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -247,7 +253,7 @@ def move_item_down(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_multiple_items.move_item_down(db, cache, context)
-        filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/move-item-down', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -267,7 +273,7 @@ def move_subitem_up(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         new_item_subitem_id = update_single_item.move_subitem_up(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, new_item_subitem_id)
         snapshot = Snapshot('/move-subitem-up', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -287,7 +293,7 @@ def move_subitem_down(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         new_item_subitem_id = update_single_item.move_subitem_down(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, new_item_subitem_id)
         snapshot = Snapshot('/move-subitem-down', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -307,7 +313,7 @@ def indent(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_single_item.indent(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/indent', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -327,7 +333,7 @@ def outdent(db):
         context = get_request_context(request, cache)
         snap_pre = SnapshotFragment(cache, context.item_subitem_id)
         update_single_item.outdent(db, context, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
         snap_post = SnapshotFragment(cache, context.item_subitem_id)
         snapshot = Snapshot('/outdent', snap_pre, snap_post, context.item_subitem_id)
         snapshots.push(snapshot)
@@ -345,8 +351,7 @@ def search(db):
     context = get_request_context(request, cache)
     # TODO why recreate context?
     context = Context(context.app_state, search_text=context.search_text, search_filter=context.search_filter)
-    filtered_items, reached_scroll_end = filter_items(cache, context, updated_search=True, dirty_ranking=True)
-    print(f'debug: reached_scroll_end = {reached_scroll_end}')
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, updated_search=True, dirty_ranking=True)
     if reset_undo_stack_on_search:
         snapshots.reset()
     else:
@@ -385,7 +390,7 @@ def add_item_sibling(db):
     context = get_request_context(request, cache)
     snap_pre = SnapshotFragment(cache, context.item_subitem_id)
     new_item_subitem_id = update_multiple_items.add_item_sibling(db, cache, context)
-    filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
     snap_post = SnapshotFragment(cache, new_item_subitem_id)
     snapshot = Snapshot('/add-item-sibling', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -401,7 +406,7 @@ def add_subitem_sibling(db):
     context = get_request_context(request, cache)
     snap_pre = SnapshotFragment(cache, context.item_subitem_id)
     new_item_subitem_id = update_single_item.add_subitem_sibling(db, context, cache)
-    filtered_items, reached_scroll_end = filter_items(cache, context)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
     snap_post = SnapshotFragment(cache, new_item_subitem_id)
     snapshot = Snapshot('/add-subitem-sibling', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -417,7 +422,7 @@ def add_subitem_child(db):
     context = get_request_context(request, cache)
     snap_pre = SnapshotFragment(cache, context.item_subitem_id)
     new_item_subitem_id = update_single_item.add_subitem_child(db, context, cache)
-    filtered_items, reached_scroll_end = filter_items(cache, context)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
     snap_post = SnapshotFragment(cache, new_item_subitem_id)
     snapshot = Snapshot('/add-subitem-child', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -436,7 +441,7 @@ def add_item_top(db):
     if len(context.search_filter['texts']) > 0 or context.search_filter['partial_text'] is not None:
         return error_response('Cannot add new items when using a text based search filter.')
     new_item_subitem_id = update_multiple_items.add_item_top(db, cache, context)
-    filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
     snap_post = SnapshotFragment(cache, new_item_subitem_id)
     snapshot = Snapshot('/add-item-top', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -453,7 +458,7 @@ def paste_sibling(db):
     snap_pre = SnapshotFragment(cache, context.item_subitem_id)
     assert context.clipboard is not None, 'missing clipboard from request'
     new_item_subitem_id = update_multiple_items.paste_sibling(db, cache, context)
-    filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
     snap_post = SnapshotFragment(cache, new_item_subitem_id)
     snapshot = Snapshot('/paste-sibling', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -470,7 +475,7 @@ def paste_child(db):
     snap_pre = SnapshotFragment(cache, context.item_subitem_id)
     assert context.clipboard is not None, 'missing clipboard from request'
     new_item_subitem_id = update_single_item.paste_child(db, context, cache)
-    filtered_items, reached_scroll_end = filter_items(cache, context)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
     snap_post = SnapshotFragment(cache, new_item_subitem_id)
     snapshot = Snapshot('/paste-child', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -484,7 +489,7 @@ def pagination_update(db):
     global cache, snapshots
     context = get_request_context(request, cache)
     # no snapshot
-    filtered_items, reached_scroll_end = filter_items(cache, context)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
     return generic_response(filtered_items, reached_scroll_end, new_item_subitem_id=context.item_subitem_id)
 
 
@@ -497,7 +502,7 @@ def update_tags(db):
     must_recalculate_ontology = update_single_item.update_tags(db, context, cache)
     if must_recalculate_ontology:
         update_multiple_items.recalculate_ontology(db, cache, context)
-    filtered_items, reached_scroll_end = filter_items(cache, context)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
     snap_post = SnapshotFragment(cache, context.item_subitem_id)
     snapshot = Snapshot('/update-tags', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -513,7 +518,7 @@ def open_to(db):
     context = get_request_context(request, cache)
     snap_pre = SnapshotFragment(cache, context.item_subitem_id)
     update_single_item.open_to(db, context, cache)
-    filtered_items, reached_scroll_end = filter_items(cache, context)
+    filtered_items, reached_scroll_end = filter_and_sort_items(cache, context)
     snap_post = SnapshotFragment(cache, context.item_subitem_id)
     snapshot = Snapshot('/open-to', snap_pre, snap_post, context.item_subitem_id)
     snapshots.push(snapshot)
@@ -530,7 +535,7 @@ def undo(db):
     # no snapshot
     try:
         new_item_subitem_id = update_multiple_items.undo(db, snapshots, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
         crud.commit(db)
         return generic_response(filtered_items, reached_scroll_end, new_item_subitem_id=new_item_subitem_id)
     except Exception as e:
@@ -546,7 +551,7 @@ def redo(db):
     # no snapshot
     try:
         new_item_subitem_id = update_multiple_items.redo(db, snapshots, cache)
-        filtered_items, reached_scroll_end = filter_items(cache, context, dirty_ranking=True)
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
         crud.commit(db)
         return generic_response(filtered_items, reached_scroll_end, new_item_subitem_id=new_item_subitem_id)
     except Exception as e:
@@ -554,5 +559,78 @@ def redo(db):
         return noop_response('nothing to redo')
 
 
+@app.post('/chat-open')
+def chat_open(db):
+    global chat_history
+    chat_history = []  # reset history
+    return noop_response('chat-open')
+
+
+@app.post('/chat-close')
+def chat_close(db):
+    global chat_history
+    chat_history = []  # reset history
+    return noop_response('chat-close')
+
+
+@app.post('/chat-select-reference')
+def chat_select_reference(db):
+    global chat_history, cache
+    context = get_request_context(request, cache)
+    system_message = build_selection_prompt(context)
+    chat_history.append(system_message)
+    return noop_response('chat-select-reference')
+
+
+@app.post('/chat-send-message')
+def chat_send_message(db):
+    global chat_history, cache
+    context = get_request_context(request, cache)
+    assert context.open_ai_api_key is not None, "OpenAI key is None"
+    if len(chat_history) == 0:
+        # get relevant items based on search filter
+        filtered_items, reached_scroll_end = filter_and_sort_items(cache, context, dirty_ranking=True)
+        prompts = build_initial_prompts(context, filtered_items)
+        for prompt in prompts:
+            chat_history.append(prompt)
+    else:
+        assert chat_history[0]['role'] == 'system'
+    user_message = {'role': 'user', 'content': context.chat_user_message}
+    chat_history.append(user_message)
+
+    # web_crawl_system_messages = process_user_message_for_urls(context.chat_user_message)
+    # chat_history.extend(web_crawl_system_messages)
+
+    ########################################################
+
+    # Prepare the payload for the OpenAI API
+    data = {
+        "model": open_ai_model,
+        "messages": chat_history
+    }
+    headers = {
+        "Authorization": f"Bearer {context.open_ai_api_key}",
+        "Content-Type": "application/json"
+    }
+
+    # Make a synchronous POST request to the OpenAI Chat API  # TODO FastAPI
+    openai_response = requests.post(open_ai_url, json=data, headers=headers)
+
+    # Check if the request was successful and process the response
+    if openai_response.status_code == 200:
+        openai_data = openai_response.json()
+        assistant_message = openai_data['choices'][0]['message']
+        msg = {
+            'role': assistant_message['role'],
+            'content': assistant_message['content']
+        }
+        chat_history.append(msg)
+        return chat_response(chat_history)
+    else:
+        return error_response('bad response from open ai')
+
+
 if __name__ == '__main__':
+    global development_mode
+    development_mode = True
     run_app()