bookworm_genai 0.1.0__tar.gz

bookworm_genai-0.1.0/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.1
+Name: bookworm_genai
+Version: 0.1.0
+Summary: Bookworm - A LLM-powered bookmark search engine
+Author: kiran94
+Requires-Python: >=3.9,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: duckdb (>=1.0.0,<2.0.0)
+Requires-Dist: jq (>=1.7.0,<2.0.0)
+Requires-Dist: langchain (>=0.2.12,<0.3.0)
+Requires-Dist: langchain-community (>=0.2.11,<0.3.0)
+Requires-Dist: langchain-openai (>=0.1.20,<0.2.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: platformdirs (>=4.2.2,<5.0.0)
+Requires-Dist: rich (>=13.7.1,<14.0.0)
+Description-Content-Type: text/markdown
+
+# bookworm
+
+[![main](https://github.com/kiran94/bookworm/actions/workflows/main.yml/badge.svg)](https://github.com/kiran94/bookworm/actions/workflows/main.yml)
+
+### Processes 
+
+*`bookworm sync`*
+
+```python
+python -m bookworm sync
+```
+
+```mermaid
+graph LR
+
+subgraph Bookmarks 
+    Chrome(Chrome Bookmarks)
+    Brave(Brave Bookmarks)
+end
+
+Bookworm(bookworm sync)
+
+EmbeddingsService(Embeddings Service e.g OpenAIEmbeddings)
+
+VectorStore(Vector Store e.g DuckDB)
+
+Chrome -->|load bookmarks|Bookworm
+Brave -->|load bookmarks|Bookworm
+
+Bookworm -->|vectorize bookmarks|EmbeddingsService-->|store embeddings|VectorStore
+```
+
+---
+
+*`bookworm ask`*
+
+```python
+python -m bookworm ask
+```
+
+```mermaid
+graph LR
+
+query
+Bookworm(bookworm ask)
+
+subgraph _
+    LLM(LLM e.g OpenAI)
+    VectorStore(Vector Store e.g DuckDB)
+end
+
+query -->|user queries for information|Bookworm
+
+Bookworm -->|simularity search|VectorStore -->|send similar docs + user query|LLM
+LLM -->|send back response|Bookworm
+```
+
+---
+
+
+### Developer Setup 
+
+```bash
+# LLMs
+export OPENAI_API_KEY=
+
+# Langchain (optional, but useful for debugging)
+export LANGCHAIN_API_KEY=
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_PROJECT=bookworm
+
+# Misc (optional)
+export LOGGING_LEVEL=INFO
+```
+
+Recommendations:
+
+- Install [`pyenv`](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation) and ensure [build dependencies are installed](https://github.com/pyenv/pyenv?tab=readme-ov-file#install-python-build-dependencies) for your OS.
+- Install [Poetry](https://python-poetry.org/docs/) we will be using [environment management](https://python-poetry.org/docs/managing-environments/) below.
+
+
+```bash
+poetry env use 3.9 # or path to your 3.9 installation
+
+poetry shell
+poetry install
+
+bookworm --help
+```
+

bookworm_genai-0.1.0/README.md

@@ -0,0 +1,89 @@
+# bookworm
+
+[![main](https://github.com/kiran94/bookworm/actions/workflows/main.yml/badge.svg)](https://github.com/kiran94/bookworm/actions/workflows/main.yml)
+
+### Processes 
+
+*`bookworm sync`*
+
+```python
+python -m bookworm sync
+```
+
+```mermaid
+graph LR
+
+subgraph Bookmarks 
+    Chrome(Chrome Bookmarks)
+    Brave(Brave Bookmarks)
+end
+
+Bookworm(bookworm sync)
+
+EmbeddingsService(Embeddings Service e.g OpenAIEmbeddings)
+
+VectorStore(Vector Store e.g DuckDB)
+
+Chrome -->|load bookmarks|Bookworm
+Brave -->|load bookmarks|Bookworm
+
+Bookworm -->|vectorize bookmarks|EmbeddingsService-->|store embeddings|VectorStore
+```
+
+---
+
+*`bookworm ask`*
+
+```python
+python -m bookworm ask
+```
+
+```mermaid
+graph LR
+
+query
+Bookworm(bookworm ask)
+
+subgraph _
+    LLM(LLM e.g OpenAI)
+    VectorStore(Vector Store e.g DuckDB)
+end
+
+query -->|user queries for information|Bookworm
+
+Bookworm -->|simularity search|VectorStore -->|send similar docs + user query|LLM
+LLM -->|send back response|Bookworm
+```
+
+---
+
+
+### Developer Setup 
+
+```bash
+# LLMs
+export OPENAI_API_KEY=
+
+# Langchain (optional, but useful for debugging)
+export LANGCHAIN_API_KEY=
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_PROJECT=bookworm
+
+# Misc (optional)
+export LOGGING_LEVEL=INFO
+```
+
+Recommendations:
+
+- Install [`pyenv`](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation) and ensure [build dependencies are installed](https://github.com/pyenv/pyenv?tab=readme-ov-file#install-python-build-dependencies) for your OS.
+- Install [Poetry](https://python-poetry.org/docs/) we will be using [environment management](https://python-poetry.org/docs/managing-environments/) below.
+
+
+```bash
+poetry env use 3.9 # or path to your 3.9 installation
+
+poetry shell
+poetry install
+
+bookworm --help
+```

bookworm_genai-0.1.0/bookworm_genai/__init__.py

@@ -0,0 +1,17 @@
+import logging
+import os
+
+from rich.logging import RichHandler
+
+LOGGING_LEVEL = os.environ.get("LOGGING_LEVEL", logging.INFO)
+LOGGING_FORMAT = os.environ.get("LOGGING_FORMAT", "%(message)s")
+
+_is_debug = logging.getLevelName(LOGGING_LEVEL) == logging.DEBUG
+
+logging.basicConfig(
+    level=LOGGING_LEVEL,
+    format=LOGGING_FORMAT,
+    handlers=[RichHandler(markup=True, show_path=_is_debug, show_time=_is_debug, show_level=_is_debug)],
+)
+
+logging.getLogger("httpx").setLevel(logging.WARNING)

bookworm_genai-0.1.0/bookworm_genai/__main__.py

@@ -0,0 +1,70 @@
+import sys
+import logging
+import argparse
+
+from bookworm_genai.commands.sync import sync
+from bookworm_genai.commands.ask import BookmarkChain
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    logger.info("[bold green]Starting Bookworm 📖")
+    logger.debug("Running on platform '%s'", sys.platform)
+
+    argparser = argparse.ArgumentParser(description="Bookworm - A LLM-powered bookmark search engine")
+
+    sub_parsers = argparser.add_subparsers(dest="command", help="Available commands")
+    sub_parsers.add_parser("sync", help="Sync the bookmark database with the latest changes")
+    sub_parsers.add_parser("ask", help="Search for a bookmark")
+
+    args = argparser.parse_args()
+
+    logger.debug("Arguments: %s", args)
+
+    if args.command == "sync":
+        sync()
+
+    elif args.command == "ask":
+        logger.info("What would you like to search for?")
+        query = input("> ")
+
+        logger.debug("query: %s", query)
+
+        with BookmarkChain() as bookmark_chain:
+            if not bookmark_chain.is_valid():
+                logger.debug("bookmark chain is not valid, exiting early.")
+                return
+
+            bookmarks = bookmark_chain.ask(query)
+
+        if not bookmarks.bookmarks:
+            logger.info("""
+            No bookmarks found for the query 🙁. Please ensure you have performed a "bookworm sync" to update the database
+            and the query is relevant to the bookmarks stored.
+            """)
+            return
+
+        for index, bookmark in enumerate(bookmarks.bookmarks):
+            if logger.isEnabledFor(logging.DEBUG):
+                # also shows the source of the bookmark
+                logger.info(f"[green][{index}] [/] {bookmark.title} - [link={bookmark.url}]{bookmark.url}[/link] ({bookmark.source})")
+            else:
+                logger.info(f"[green][{index}] [/] {bookmark.title} - [link={bookmark.url}]{bookmark.url}[/link]")
+
+        logger.info("Press a number to open the bookmark:")
+        while True:
+            try:
+                raw_input = input("> ")
+                selected_index = int(raw_input)
+                bookmarks.bookmarks[selected_index].open()
+
+                break
+            except ValueError:
+                logger.warning(f"Invalid input: '{raw_input}'. Please enter a number.")
+            except IndexError:
+                logger.warning(f"Invalid index: '{selected_index}'. Please select a valid index.")
+
+
+if __name__ == "__main__":
+    main()

bookworm_genai-0.1.0/bookworm_genai/commands/ask.py

@@ -0,0 +1,86 @@
+import os
+import logging
+
+import duckdb
+from langchain_community.vectorstores import DuckDB as DuckDBVectorStore
+from langchain_openai import ChatOpenAI, AzureChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+from langchain_core.language_models.chat_models import BaseChatModel
+
+from bookworm_genai.models import Bookmarks
+from bookworm_genai.storage import full_database_path, _get_embedding_store
+
+logger = logging.getLogger(__name__)
+
+
+_system_message = """
+You have knowledge about all the browser bookmarks stored by an individual.
+When a user asks a question, you should be able to search the bookmarks and return the most relevant bookmark title and URL.
+It could be multiple bookmarks.
+If you don't have anything in the context then return empty list
+
+The bookmarks available are from the context:
+{context}
+"""
+
+
+class BookmarkChain:
+    def __init__(self):
+        logger.debug("Connecting to vector database at: %s", full_database_path)
+        self._duckdb_connection = duckdb.connect(full_database_path, read_only=False)
+        self.vector_store = DuckDBVectorStore(connection=self._duckdb_connection, embedding=_get_embedding_store())
+
+        llm = _get_llm()
+        llm = llm.with_structured_output(Bookmarks)
+
+        prompt = ChatPromptTemplate.from_messages([("system", _system_message), ("human", "{query}")])
+
+        self.chain = {"context": self.vector_store.as_retriever(), "query": RunnablePassthrough()} | prompt | llm
+
+    def ask(self, query: str) -> Bookmarks:
+        logger.debug("Searching for bookmarks with query: %s", query)
+
+        return self.chain.invoke(query)
+
+    def is_valid(self) -> bool:
+        res = self._duckdb_connection.execute("SELECT COUNT(*) FROM embeddings").fetchall()
+
+        try:
+            res = res[0][0]
+        except IndexError:
+            logger.warning("validation check failed due to unexpected response from the database.")
+            logger.debug("Raw DuckDB Response: %s", res)
+
+            return False
+
+        if res == 0:
+            logger.warning("No bookmarks were found in database. Please ensure you run 'bookworm sync' before asking questions")
+            return False
+        else:
+            return True
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        logger.debug("Closing DuckDB connection")
+
+        self._duckdb_connection.close()
+
+
+def _get_llm() -> BaseChatModel:
+    kwargs = {
+        "temperature": 0.0,
+    }
+
+    if os.environ.get("OPENAI_API_KEY"):
+        # https://api.python.langchain.com/en/latest/chat_models/langchain_openai.chat_models.base.ChatOpenAI.html
+        return ChatOpenAI(**kwargs)
+
+    elif os.environ.get("AZURE_OPENAI_API_KEY"):
+        # https://api.python.langchain.com/en/latest/chat_models/langchain_openai.chat_models.azure.AzureChatOpenAI.html
+        return AzureChatOpenAI(**kwargs)
+
+    else:
+        raise ValueError("No OpenAI API key found in environment variables")

bookworm_genai-0.1.0/bookworm_genai/commands/sync.py

@@ -0,0 +1,39 @@
+import os
+import sys
+import logging
+
+from langchain_community.document_loaders import JSONLoader
+
+from bookworm_genai.integrations import browsers
+from bookworm_genai.storage import store_documents
+
+logger = logging.getLogger(__name__)
+
+
+def sync():
+    docs = []
+
+    for browser, config in browsers.items():
+        try:
+            platform_config = config[sys.platform]
+        except KeyError:
+            logger.warning(f"Platform {sys.platform} not supported for browser {browser}")
+            continue
+        else:
+            expanded_path = os.path.expanduser(platform_config["bookmark_file_path"])
+
+            jq_command = """
+              [.roots.bookmark_bar.children, .roots.other.children] |
+              flatten |
+              .. |
+              objects |
+              select(.type == "url")
+            """
+
+            logger.info("Loading bookmarks from %s", expanded_path)
+            bookmark_bar = JSONLoader(expanded_path, jq_command, text_content=False)
+            docs.extend(bookmark_bar.lazy_load())
+
+    logger.debug(f"{len(docs)} Bookmarks loaded")
+
+    store_documents(docs)

bookworm_genai-0.1.0/bookworm_genai/integrations.py

@@ -0,0 +1,26 @@
+from enum import Enum
+
+
+class Browser(str, Enum):
+    BRAVE = "brave"
+    CHROME = "chrome"
+
+
+# Configuration for various browsers and details about them
+# The bookmark_file_path is the path to the bookmarks file for the browsers, in order for it to be used it must be used in conjunction with
+# os.path.expanduser as it may contain environment variables
+#
+# The platform configuration is keyed off the values from https://docs.python.org/3/library/sys.html#sys.platform
+#
+browsers = {
+    Browser.BRAVE: {
+        "linux": {"bookmark_file_path": "~/.config/BraveSoftware/Brave-Browser/Default/Bookmarks"},
+        "win32": {},
+        "darwin": {},
+    },
+    Browser.CHROME: {
+        "linux": {"bookmark_file_path": "~/.config/google-chrome/Default/Bookmarks"},
+        "win32": {},
+        "darwin": {},
+    },
+}

bookworm_genai-0.1.0/bookworm_genai/models.py

@@ -0,0 +1,36 @@
+import sys
+import subprocess
+import logging
+
+from langchain_core.pydantic_v1 import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+class Bookmark(BaseModel):
+    """
+    A bookmark to a website
+    """
+
+    title: str = Field(description="The title of the bookmark")
+    url: str = Field(description="The URL of the bookmark")
+    source: str = Field(description="The source of the bookmark")
+
+    def open(self):
+        if sys.platform == "win32":
+            subprocess.Popen(["start", self.url], shell=True)
+        elif sys.platform == "darwin":
+            subprocess.Popen(["open", self.url])
+        elif sys.platform == "linux":
+            subprocess.Popen(["xdg-open", self.url])
+        else:
+            logger.warning(f'Platform "{sys.platform}" not supported. Printing URL instead')
+            logger.info(self.url)
+
+
+class Bookmarks(BaseModel):
+    """
+    A list of bookmarks
+    """
+
+    bookmarks: list[Bookmark] = Field(description="A list of bookmarks")

bookworm_genai-0.1.0/bookworm_genai/storage.py

@@ -0,0 +1,43 @@
+import os
+import duckdb
+import logging
+
+from platformdirs import PlatformDirs
+from langchain_community.vectorstores import DuckDB as DuckDBVectorStore
+from langchain_core.documents import Document
+from langchain_core.embeddings.embeddings import Embeddings
+from langchain_openai.embeddings import OpenAIEmbeddings, AzureOpenAIEmbeddings
+
+appdirs = PlatformDirs("bookworm", "bookworm")
+database_name = "bookmarks.duckdb"
+full_database_path = os.path.join(appdirs.user_data_dir, database_name)
+
+logger = logging.getLogger(__name__)
+
+
+def store_documents(docs: list[Document]):
+    logger.debug(f"creating folder {appdirs.user_data_dir}")
+    os.makedirs(appdirs.user_data_dir, exist_ok=True)
+
+    embeddings = _get_embedding_store()
+
+    logger.info(f"vectorizing and storing {len(docs)} documents into {full_database_path}")
+    with duckdb.connect(full_database_path) as conn:
+        logger.debug("dropping existing embeddings table if exists")
+        conn.execute("DROP TABLE IF EXISTS embeddings")
+
+        logger.debug("loading documents")
+        DuckDBVectorStore.from_documents(docs, embeddings, connection=conn)
+
+
+def _get_embedding_store() -> Embeddings:
+    if os.environ.get("AZURE_OPENAI_API_KEY"):
+        logger.debug("Using Azure OpenAI Embeddings")
+        return AzureOpenAIEmbeddings()
+
+    elif os.environ.get("OPENAI_API_KEY"):
+        logger.debug("Using OpenAI Embeddings")
+        return OpenAIEmbeddings()
+
+    else:
+        raise ValueError("No OpenAI API key found in environment variables")

bookworm_genai-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[tool.poetry]
+name = "bookworm_genai"
+version = "0.1.0"
+description = "Bookworm - A LLM-powered bookmark search engine"
+authors = ["kiran94"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+langchain = "^0.2.12"
+langchain-community = "^0.2.11"
+langchain-openai = "^0.1.20"
+jq = "^1.7.0"
+duckdb = "^1.0.0"
+rich = "^13.7.1"
+platformdirs = "^4.2.2"
+pandas = "^2.2.2"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.2"
+ruff = "^0.5.6"
+pytest-cov = "^5.0.0"
+pytest-github-actions-annotate-failures = "^0.2.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+bookworm = 'bookworm_genai.__main__:main'
+
+[tool.ruff]
+line-length = 160