logseq-retriever 0.4.1__py3-none-any.whl

logseq_retriever/loaders/__init__.py

@@ -0,0 +1,15 @@
+from logseq_retriever.loaders.journal_document_metadata import (
+    LogseqJournalDocumentMetadata,
+)
+from logseq_retriever.loaders.journal_filesystem_loader import (
+    LogseqJournalFilesystemLoader,
+)
+from logseq_retriever.loaders.journal_loader_input import LogseqJournalLoaderInput
+from logseq_retriever.loaders.journal_loader import LogseqJournalLoader
+
+__all__ = [
+    "LogseqJournalDocumentMetadata",
+    "LogseqJournalFilesystemLoader",
+    "LogseqJournalLoaderInput",
+    "LogseqJournalLoader",
+]

logseq_retriever/loaders/journal_document_metadata.py

@@ -0,0 +1,32 @@
+from typing import Annotated
+
+from pydantic import BaseModel, Field
+
+
+class LogseqJournalDocumentMetadata(BaseModel):
+    """
+    Metadata for a Logseq journal `Document`.
+    """
+
+    journal_date: Annotated[
+        str,
+        Field(
+            description="The date of the journal entry, in YYYY-MM-DD format.",
+            examples=["2023-01-01", "2025-06-09"],
+        ),
+    ]
+
+    journal_tags: Annotated[
+        list[str],
+        Field(
+            description="The tags associated with the journal entry.",
+            examples=[["tag1", "tag2"], ["tag3"]],
+        ),
+    ]
+
+    journal_char_count: Annotated[
+        int,
+        Field(
+            description="The number of characters in the journal entry.",
+        ),
+    ]

logseq_retriever/loaders/journal_filesystem_loader.py

@@ -0,0 +1,157 @@
+from datetime import date, datetime
+from logging import getLogger
+from pathlib import Path
+
+from langchain_core.documents import Document
+from logseq_retriever.loaders.journal_loader import LogseqJournalLoader
+from logseq_retriever.loaders.journal_loader_input import LogseqJournalLoaderInput
+from logseq_retriever.loaders.journal_document_metadata import (
+    LogseqJournalDocumentMetadata,
+)
+import os
+
+
+logger = getLogger(__name__)
+
+
+class LogseqJournalFilesystemLoader(LogseqJournalLoader):
+    """
+    Based on input, load a collection of Logseq journal files from the filesystem, as
+    Langchain `Document`s.
+    """
+
+    def __init__(
+        self,
+        logseq_journal_path: str,
+        **kwargs,
+    ):
+        """
+        Initialize the loader with the path to the Logseq journal directory.
+        `logseq_journal_path` should be contain Logesq journal files, such as `2025_03_27.md`
+        """
+        self.logseq_journal_path = logseq_journal_path
+        self._validate_logseq_journal_path()
+
+    def load(  # type: ignore[override]
+        self,
+        input: LogseqJournalLoaderInput,
+    ) -> list[Document]:
+        """
+        Synchronously load the documents from the Logseq journal directory, according to the input.
+        """
+        # Convert dates to datetime objects once
+        if input.start_date > input.end_date:
+            raise ValueError("journal_end_date must be after journal_start_date")
+
+        documents: list[Document] = []
+        # TODO this glob pattern can be improved by analyzing start_date & end_date to provide fewer matches
+        for path in Path(self.logseq_journal_path).glob("*.md"):
+            filename = path.name
+            if self._match_journal(filename, input.start_date, input.end_date):
+                file_path = os.path.join(self.logseq_journal_path, filename)
+                with open(file_path, "r") as file:
+                    content = file.read()
+                    documents.extend(
+                        self.__class__.parse_journal_markdown_file(
+                            content, filename, input.enable_splitting
+                        )
+                    )
+        return documents
+
+    def _validate_logseq_journal_path(self):
+        """
+        Validate the path to the Logseq journal directory. Check that the directory exists.
+        If the directory is empty, or does not contain files with the expected format, log a warning.
+        """
+        # verify that the path exist, and is a directory
+        if not os.path.exists(self.logseq_journal_path):
+            raise ValueError(
+                f"Logseq journal path does not exist: {self.logseq_journal_path}"
+            )
+        if not os.path.isdir(self.logseq_journal_path):
+            raise ValueError(
+                f"Logseq journal path is not a directory: {self.logseq_journal_path}"
+            )
+
+        # verify that the directory contains files with the expected format
+        files = os.listdir(self.logseq_journal_path)
+        if len(files) == 0:
+            logger.warning(
+                f"Logseq journal directory is empty: {self.logseq_journal_path}"
+            )
+        files = Path(self.logseq_journal_path).glob("*.md")
+        if not len(list(files)) > 0:
+            logger.warning(
+                f"No files with .md extension found in {self.logseq_journal_path}"
+            )
+
+    def _match_journal(self, filename: str, start_date: date, end_date: date) -> bool:
+        """
+        Return `True` if journal date is between `start_date` & `end_date`.
+
+        Args:
+            filename: The journal filename (e.g., "2025_03_27.md")
+            start_date: The start date as a datetime object
+            end_date: The end date as a datetime object
+
+        Returns:
+            bool: True if the file's date is within the range, False otherwise
+        """
+        if not filename.endswith(".md"):
+            return False
+
+        try:
+            # Convert filename to date object
+            file_date = datetime.strptime(filename[:-3], "%Y_%m_%d").date()
+            return start_date <= file_date <= end_date
+        except ValueError:
+            # If there's any issue parsing the date from filename, skip this file
+            return False
+
+    @staticmethod
+    def parse_journal_markdown_file(
+        content: str, filename: str, enable_splitting: bool = True
+    ) -> list[Document]:
+        """
+        Generate `Document`s from a file's contents. If necessary, split content into digestible
+        `Document`s, and attach metadata.
+        This function can potentially be augmented by calling Logseq APIs, rather than simply parsing markdown files.
+        """
+        sections = content.split("\n- ") if enable_splitting else [content]
+        docs = []
+        for section in sections:
+            if section_content := section.strip():
+                # Create a Document
+                # first, check that the content length (char count) is acceptable
+                # if longer than acceptable, then call recursively
+                # TODO: use self.p.max_char_count below instead
+                metadata = (
+                    LogseqJournalFilesystemLoader.parse_journal_markdown_file_metadata(
+                        section_content, filename
+                    )
+                )
+                docs.append(
+                    Document(
+                        page_content=section_content, metadata=metadata.model_dump()
+                    )
+                )
+        return docs
+
+    @staticmethod
+    def parse_journal_markdown_file_metadata(
+        section: str, filename: str
+    ) -> LogseqJournalDocumentMetadata:
+        """
+        Parse metadata from a journal markdown file. Return `LogseqMarkdownDocumentMetadata`.
+        This function can potentially be augmented by calling Logseq APIs, rather than simply parsing markdown files.
+        """
+        # Extract date from filename
+        date_str = filename.replace(".md", "").replace("_", "-")
+        char_count = len(section)
+
+        return LogseqJournalDocumentMetadata(
+            journal_date=date_str,
+            # TODO get tags from Document's contents
+            journal_tags=[],
+            journal_char_count=char_count,
+        )

logseq_retriever/loaders/journal_loader.py

@@ -0,0 +1,13 @@
+from typing import Any
+
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+
+class LogseqJournalLoader(BaseLoader):
+    """
+    Base class for loading Logseq journal files.
+    """
+
+    def load(self, input: Any) -> list[Document]:  # type: ignore[override]  # ty: ignore[invalid-method-override]
+        raise NotImplementedError("This method should be implemented by subclasses.")

logseq_retriever/loaders/journal_loader_input.py

@@ -0,0 +1,108 @@
+from datetime import datetime, date
+from typing import Annotated
+
+from pydantic import (
+    BaseModel,
+    Field,
+    AfterValidator,
+    computed_field,
+    PrivateAttr,
+    model_validator,
+)
+
+
+def _validate_date_format(value: str) -> str:
+    """
+    Normalize date string to ISO format (YYYY-MM-DD).
+    Accepts dates like '2023-3-1' and converts to '2023-03-01'.
+    """
+    try:
+        _parse_date(value)
+        return value
+    except ValueError:
+        raise ValueError(
+            f"Invalid date: '{value}'. Expecting ISO-8601 format: YYYY-MM-DD"
+        )
+
+
+def _parse_date(date_str: str) -> date:
+    """
+    Parse a date string into a datetime.date object.
+    """
+    return datetime.strptime(date_str, "%Y-%m-%d").date()
+
+
+class LogseqJournalLoaderInput(BaseModel):
+    """
+    Input for a Logseq journal `Document` loader, to invoke a load.
+    """
+
+    journal_start_date: Annotated[
+        str,
+        Field(
+            description="The start date of the journal to load, in YYYY-MM-DD format.",
+            examples=["2023-01-01", "2025-06-09"],
+        ),
+        AfterValidator(_validate_date_format),
+    ]
+    journal_end_date: Annotated[
+        str,
+        Field(
+            description="The end date of the journal to load, in YYYY-MM-DD format.",
+            examples=["2023-01-01", "2025-06-09"],
+        ),
+        AfterValidator(_validate_date_format),
+    ]
+    max_char_length: Annotated[
+        int,
+        Field(
+            description="The maximum number of characters to include in a single `Document`.",
+            examples=[8196, 2000],
+            default=1024 * 8,
+        ),
+    ] = 1024 * 8
+    enable_splitting: Annotated[
+        bool,
+        Field(
+            description="Whether to split the journal file into multiple `Document`s.",
+            examples=[True, False],
+            default=True,
+        ),
+    ] = True
+
+    # Private attributes that won't be included in model_dump
+    _start_date: date = PrivateAttr()
+    _end_date: date = PrivateAttr()
+
+    @model_validator(mode="after")
+    def _parse_dates(self) -> "LogseqJournalLoaderInput":
+        """Parse date strings into date objects after validation."""
+        self._start_date = _parse_date(self.journal_start_date)
+        self._end_date = _parse_date(self.journal_end_date)
+        return self
+
+    @computed_field
+    @property
+    def start_date(self) -> date:
+        """Get `journal_start_date` as a date object."""
+        return self._start_date
+
+    @computed_field
+    @property
+    def end_date(self) -> date:
+        """Get `journal_end_date` as a date object."""
+        return self._end_date
+
+
+# debugging only
+if __name__ == "__main__":
+    from pprint import pprint
+
+    pprint(LogseqJournalLoaderInput.model_json_schema())
+
+    example = LogseqJournalLoaderInput(
+        journal_start_date="2023-01-01",
+        journal_end_date="2023-01-02",
+        max_char_length=1024 * 4,
+    )
+    print(example.model_dump())

logseq_retriever/models/__init__.py

@@ -0,0 +1,15 @@
+from logseq_retriever.models.journal_pgvector import (
+    JournalDocument,
+    JournalCorpusMetadata,
+    JournalDocumentMetadata,
+    JournalSearchClientConfig,
+    JournalSearchQuery,
+)
+
+__all__ = [
+    "JournalDocument",
+    "JournalCorpusMetadata",
+    "JournalDocumentMetadata",
+    "JournalSearchClientConfig",
+    "JournalSearchQuery",
+]

logseq_retriever/models/journal_pgvector.py

@@ -0,0 +1,107 @@
+from typing import Type
+
+from pgvector.sqlalchemy import Vector
+from pydantic import Field
+from sqlalchemy import Column, String
+
+from pgvector_template.core import (
+    BaseDocument,
+    BaseDocumentMetadata,
+    BaseSearchClientConfig,
+)
+from pgvector_template.models.search import (
+    SearchQuery,
+    MetadataFilter,
+)
+
+
+class JournalDocument(BaseDocument):
+    """
+    Each `Corpus` is the entire entry for a given date. A corpus may consist of 1 or more chunks of `Document`s.
+    Each `Corpus` has a set of metadata, and each `Document` chunk has all of those, plus more.
+    """
+
+    __abstract__ = False
+    __tablename__ = "logseq_journal"
+
+    corpus_id = Column(String(len("2025-06-09")), index=True)
+    """Length of ISO date string"""
+    embedding = Column(Vector(1024))
+    """Embedding vector"""
+
+
+class JournalCorpusMetadata(BaseDocumentMetadata):
+    """Metadata schema for Logseq journal corpora. Consist of 1-or-more chunks, called `Document`s."""
+
+    # corpus
+    date_str: str = Field(
+        pattern=r"^\d{4}-\d{2}-\d{2}$",
+        description="Date in ISO format, e.g. `2025-04-20`",
+    )
+
+    # defaults
+    document_type: str = Field(default="logseq_journal")
+    schema_version: str = Field(default="2025-07-10")
+
+
+class JournalDocumentMetadata(JournalCorpusMetadata):
+    """Metadata schema for Logseq journal `Document`s. 1-or-more `Document`s make up a corpus."""
+
+    # chunk/document
+    chunk_len: int = Field()
+    """Length of the content in characters"""
+    word_count: int | None = Field()
+    """Length of the content in words"""
+    references: list[str] = Field(default=[])
+    """List of references to other Logseq documents, or journal dates"""
+    anchor_ids: list[str] = Field(default=[])
+    """Blocks in the document can have UUID anchors, which are referenced elsewhere. This is a list of all present"""
+
+
+class JournalSearchClientConfig(BaseSearchClientConfig):
+    """Configuration for the Logseq journal search client."""
+
+    document_cls: Type[BaseDocument] = JournalDocument
+    """The document type to use for the search client."""
+    document_metadata_cls: Type[BaseDocumentMetadata] = JournalDocumentMetadata
+    """The document metadata type to use for the search client."""
+    # embedding_provider
+
+
+class JournalSearchQuery(SearchQuery):
+    """
+    Standardized search query structure, specifically for searching Logseq `JournalDocument`s.
+    At least 1 search criterion is required (text, keywords, metadata_filters), but multiple are allowed.
+    Types are the same as in `SearchQuery`.
+    Descriptions are customized to better suit Logseq `JournalDocument`'s.
+    """
+
+    text: str | None = None
+    """
+    String to match against using in a semantic search, i.e. using vector distance.
+    Instead of passing in a question, rephrase the question to be a string/phrase matching closer
+    to the content expected to be found.
+    """
+
+    keywords: list[str] = []
+    """
+    List of keywords to **exact-match**.
+    If any keywords are provided, at least 1 keyword must appear in the content,
+    so use only if certain that the word will appear.
+    Do not include keywords that can be covered in metadata_filters, e.g. dates, document type.
+    If you are not certain that a word will appear, try using `text` for a semantic search instead.
+    """
+
+    metadata_filters: list[MetadataFilter] = Field(
+        default=[],
+        json_schema_extra={
+            "metadata_schema": JournalDocumentMetadata.model_json_schema()
+        },
+    )
+    """
+    List of metadata conditions that must be matched.
+    Refer to `metadata_schema` for the expected schema, as it exists in the database.
+    """
+
+    limit: int = Field(20, ge=3)
+    """Maximum number of results to return."""

logseq_retriever/retrievers/__init__.py

@@ -0,0 +1,19 @@
+from logseq_retriever.retrievers.contextualizer import (
+    RetrieverContextualizerProps,
+    RetrieverContextualizer,
+)
+from logseq_retriever.retrievers.journal_retriever import LogseqJournalRetriever
+from logseq_retriever.retrievers.journal_date_range_retriever import (
+    LogseqJournalDateRangeRetriever,
+)
+from logseq_retriever.retrievers.pgvector_journal_retriever import (
+    PGVectorJournalRetriever,
+)
+
+__all__ = [
+    "RetrieverContextualizerProps",
+    "RetrieverContextualizer",
+    "LogseqJournalRetriever",
+    "LogseqJournalDateRangeRetriever",
+    "PGVectorJournalRetriever",
+]

logseq_retriever/retrievers/contextualizer.py

@@ -0,0 +1,172 @@
+from datetime import datetime
+from logging import getLogger
+from textwrap import dedent
+from typing import Annotated, Any, Optional, Type
+
+from langchain_core.language_models import BaseLanguageModel
+from langchain_core.runnables import Runnable
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import PydanticOutputParser
+from pydantic import BaseModel, Field
+
+
+logger = getLogger(__name__)
+
+
+RETRIEVER_CONTEXTUALIZER_PROMPT_TEMPLATE = dedent(
+    """\
+    {prompt}
+
+    Latest user input: {user_input}
+
+    {format_instructions}
+    """
+)
+RETRIEVER_CONTEXTUALIZER_PROMPT_TEMPLATE_WITH_CHAT_HISTORY = dedent(
+    """\
+    Realtime context:
+    {realtime_context}
+
+    {prompt}
+
+    Chat History:
+    {chat_history}
+
+    Latest user input: {user_input}
+
+    {format_instructions}
+    """
+)
+
+
+class RetrieverContextualizerProps(BaseModel):
+    """
+    Contextualizers are a component within Langchain `Retriever`s, that transform a natural-language
+    input (and history) into an actionable query, which can in turn be used to fetch relevant
+    `Document`s to answer address the input. The actionable query output can be structured, or
+    simply another string that can be used to query a Vectorstore.
+
+    To do this, the core of the Contextualizer is an LLM. The `prompt` is used by the LLM to perform
+    the transformation task.
+    """
+
+    llm: Annotated[
+        BaseLanguageModel,
+        Field(
+            "The LLM that will be used to transform the input into an actionable query.",
+        ),
+    ]
+
+    prompt: Annotated[
+        str,
+        Field(
+            description="The prompt to use for the LLM to transform the input into an actionable query.",
+            examples=[
+                "Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language.\n\nChat History:\n{chat_history}\nFollow Up Input: {user_input}\nStandalone question:"
+            ],
+            default="Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language.\n\nChat History:\n{chat_history}\nFollow Up Input: {user_input}\nStandalone question:",
+        ),
+    ]
+
+    # TODO impl validation on this schema
+    output_schema: Annotated[
+        Optional[Type[BaseModel]],
+        Field(
+            description="(Optional) Structured output schema, as a Pydantic `BaseModel`. If provided, will be added to the end of the prompt.",
+            default=None,
+        ),
+    ] = None
+
+    enable_chat_history: Annotated[
+        bool,
+        Field(
+            description="Whether to enable chat history in the prompt.",
+            default=True,
+        ),
+    ] = True
+
+
+class RetrieverContextualizer(Runnable):
+    """
+    A Runnable that transforms natural language input into an actionable query
+    for retrievers, based on the provided configuration.
+    """
+
+    def __init__(self, props: RetrieverContextualizerProps):
+        """Initialize with validated props."""
+        self.props = props
+        self.chain = self._generate_chain()
+        self._parser_type = self.parser._type
+        self._output_type = (
+            self.parser.OutputType
+            if not self.props.output_schema
+            else self.props.output_schema
+        )
+
+    def _generate_chain(self) -> Runnable:
+        """
+        Generate and return the appropriate chain based on props.
+
+        Returns:
+            A Runnable chain that processes inputs according to the configuration.
+        """
+        # If output schema is provided, use PydanticOutputParser
+        if self.props.output_schema:
+            self.parser = PydanticOutputParser(pydantic_object=self.props.output_schema)
+            # create a PromptTemplate with partials
+            self.prompt_template = PromptTemplate(
+                input_variables=(
+                    ["chat_history", "user_input"]
+                    if self.props.enable_chat_history
+                    else ["user_input"]
+                ),
+                partial_variables={
+                    "realtime_context": self._get_realtime_context(),
+                    "prompt": self.props.prompt,
+                    "format_instructions": self.parser.get_format_instructions(),
+                },
+                template=(
+                    RETRIEVER_CONTEXTUALIZER_PROMPT_TEMPLATE_WITH_CHAT_HISTORY
+                    if self.props.enable_chat_history
+                    else RETRIEVER_CONTEXTUALIZER_PROMPT_TEMPLATE
+                ),
+            )
+
+        else:
+            # Otherwise, use the LLM and extract the string content
+            # This ensures we get a clean string output rather than an LLM result object
+            from langchain_core.output_parsers import StrOutputParser
+
+            self.parser = StrOutputParser()
+            self.prompt_template = PromptTemplate.from_template(self.props.prompt)
+
+        # can enable for debugging, will not fail
+        return (
+            self.prompt_template
+            | (lambda x: logger.debug(f"Contextualizer prompt: {x}") or x)
+            | self.props.llm
+            | (lambda x: logger.debug(f"Contextualizer LLM output: {x}") or x)
+            | self.parser
+            | (lambda x: logger.info(f"OutputParser output: {x}") or x)
+        )
+
+    def invoke(self, input: dict[str, Any], config=None, **kwargs) -> Any:
+        """
+        Process the input through the chain.
+
+        Args:
+            input: The input to process, typically containing 'question' and 'chat_history'.
+            config: Optional configuration for the chain.
+
+        Returns:
+            The processed output, either a string or a structured object based on the output_schema.
+        """
+        return self.chain.invoke(input, config=config, **kwargs)
+
+    @staticmethod
+    def _get_realtime_context() -> str:
+        """
+        Get a string representing the realtime context of the retriever, with info such as:
+        - current datetime
+        """
+        return f"Current datetime: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n"

logseq_retriever/retrievers/journal_date_range_retriever.py

@@ -0,0 +1,79 @@
+from collections.abc import Sequence
+from logging import getLogger
+
+from langchain_core.documents import Document
+from langchain_core.messages import BaseMessage
+from logseq_retriever.loaders import LogseqJournalLoader
+from logseq_retriever.retrievers.journal_retriever import LogseqJournalRetriever
+from logseq_retriever.retrievers.contextualizer import RetrieverContextualizer
+from logseq_retriever.loaders.journal_loader_input import LogseqJournalLoaderInput
+
+
+logger = getLogger(__name__)
+
+
+class LogseqJournalDateRangeRetriever(LogseqJournalRetriever):
+    """
+    A `Retriever` that retrieves documents from a Logseq journal within a specified date range.
+    """
+
+    def __init__(
+        self,
+        contextualizer: RetrieverContextualizer,
+        loader: LogseqJournalLoader,
+        verbose: bool = True,
+    ):
+        """
+        Initialize the `Retriever` with a contextualizer and a loader.
+
+        Args:
+            contextualizer (`RetrieverContextualizer`)
+            loader (`LogseqJournalLoader`)
+        """
+        super().__init__()
+
+        if not isinstance(contextualizer, RetrieverContextualizer):
+            raise TypeError(
+                "Contextualizer must be an instance of RetrieverContextualizer"
+            )
+        if contextualizer._output_type != LogseqJournalLoaderInput:
+            raise TypeError(
+                "Contextualizer output type must be LogseqJournalLoaderInput"
+            )
+        self._contextualizer = contextualizer
+
+        if not isinstance(loader, LogseqJournalLoader):
+            raise TypeError("Loader must be an instance of LogseqJournalLoader")
+        self._loader = loader
+        self._verbose = verbose
+
+    def _build_loader_input(
+        self,
+        query: str,
+        chat_history: Sequence[BaseMessage] = (),
+    ) -> LogseqJournalLoaderInput:
+        """
+        Based on the natural-language `query`, return an instance of `LogseqJournalLoaderInput`,
+        which can then be used to invoke the `LogseqJournalLoader`.
+        Use the `RetrieverContextualizer` to do this.
+        """
+        contextualizer_input = {
+            "chat_history": chat_history,
+            "user_input": query,
+        }
+        loader_input = self._contextualizer.invoke(contextualizer_input)
+        if self._verbose:
+            logger.info(f"Contextualizer output: {loader_input}")
+        if not isinstance(loader_input, LogseqJournalLoaderInput):
+            raise TypeError(
+                f"Expected LogseqJournalLoaderInput but got {type(loader_input).__name__}"
+            )
+        return loader_input
+
+    def _fetch_documents(
+        self, loader_input: LogseqJournalLoaderInput
+    ) -> list[Document]:
+        docs = self._loader.load(loader_input)
+        if self._verbose:
+            logger.info(f"Retrieved {len(docs)} documents")
+        return docs

logseq_retriever/retrievers/journal_retriever.py

@@ -0,0 +1,105 @@
+from abc import abstractmethod
+from collections.abc import Sequence
+from logging import getLogger
+from typing import Any
+
+from langchain_core.callbacks.manager import CallbackManagerForRetrieverRun
+from langchain_core.documents import Document
+from langchain_core.exceptions import OutputParserException
+from langchain_core.retrievers import BaseRetriever
+from langchain_core.messages import BaseMessage
+from pydantic import ValidationError
+
+
+logger = getLogger(__name__)
+
+
+class LogseqJournalRetriever(BaseRetriever):
+    """
+    A Langchain `Retriever` that is specifically for retrieving Logseq journal `Document`'s,
+    based on a natural-language query. This `Retriever` will, in turn, leverage a Loader or
+    Vectorstore to retrieve relevant documents to the query.
+    """
+
+    document_context: str = "These Documents represent journal entries. "
+
+    def retrieve(
+        self, query: str, chat_history: Sequence[BaseMessage] | None = None
+    ) -> list[Document]:
+        """
+        Directly retrieve documents for a query, bypassing LangChain's `invoke()` machinery.
+
+        Note: unlike `invoke()`, this method does not trigger LangSmith tracing or
+        registered callbacks. Use `invoke()` if those are needed.
+        """
+        return self._execute(query, chat_history or ())
+
+    def _get_relevant_documents(
+        self,
+        query: str | dict[str, Any],
+        *,
+        run_manager: CallbackManagerForRetrieverRun,
+        chat_history: Sequence[BaseMessage] | None = None,
+    ) -> list[Document]:
+        """
+        Called by `invoke`.
+
+        `query` can be provided as a `str` (a natural-language query), or as a dict where
+        `chat_history` can be provided additionally. Format:
+
+        ```python
+        query = {
+            "user_input": "user's latest question",
+            "chat_history": [("AiMessage", )]
+        }
+        ```
+
+        Returns potentially relevant `langchain_core.documents.Document`s to answer the query.
+        """
+        # Handle case where query is passed as a dictionary (e.g., {"user_input": "query", "chat_history": [...]})
+        if isinstance(query, dict):
+            actual_query = (
+                query.get("user_input") or query.get("input") or query.get("query", "")
+            )
+            chat_history = (
+                chat_history or query.get("chat_history") or query.get("history")
+            )
+        else:
+            actual_query = query
+
+        return self._execute(actual_query, chat_history or [])
+
+    def _execute(
+        self, query: str, chat_history: Sequence[BaseMessage]
+    ) -> list[Document]:
+        try:
+            loader_input = self._build_loader_input(query, chat_history)
+        except (TypeError, ValidationError, OutputParserException):
+            logger.exception("Error building loader input")
+            return []
+        return self._fetch_documents(loader_input)
+
+    @abstractmethod
+    def _fetch_documents(
+        self,
+        loader_input: Any,
+    ) -> list[Document]:
+        """
+        Subclasses shall impl this method.
+        Return a list of `langchain_core.documents.Document`s based on the user's query
+        (and chat_history if available).
+        """
+        raise NotImplementedError("This method shall be implemented by subclasses.")
+
+    @abstractmethod
+    def _build_loader_input(
+        self,
+        query: str,
+        chat_history: Sequence[BaseMessage] = (),
+    ) -> Any:
+        """
+        Subclasses shall impl this method.
+        Return a dataclass, based on the user's query and chat_history if available, which shall
+        be used in the subsequent step to load/query for relevant documents.
+        """
+        raise NotImplementedError("This method shall be implemented by subclasses.")

logseq_retriever/retrievers/pgvector_journal_retriever.py

@@ -0,0 +1,95 @@
+from collections.abc import Sequence
+from logging import getLogger
+
+from langchain_core.messages import BaseMessage
+from langchain_core.documents import Document
+from pgvector_template.core.search import SearchQuery
+from pgvector_template.service import DocumentService
+
+from logseq_retriever.retrievers.contextualizer import RetrieverContextualizer
+from logseq_retriever.retrievers.journal_retriever import LogseqJournalRetriever
+from logseq_retriever.models.journal_pgvector import JournalDocument
+
+
+logger = getLogger(__name__)
+
+
+class PGVectorJournalRetriever(LogseqJournalRetriever):
+    """
+    A `Retriever` that relies on a PGVector backend to fetch Logseq journals.
+    """
+
+    def __init__(
+        self,
+        contextualizer: RetrieverContextualizer,
+        document_service: DocumentService,
+        verbose: bool = True,
+        **kwargs,
+    ):
+        """
+        Initialize the `Retriever` with a contextualizer and a loader.
+        """
+        super().__init__()
+
+        if not isinstance(contextualizer, RetrieverContextualizer):
+            raise TypeError(
+                "contextualizer must be an instance of RetrieverContextualizer"
+            )
+        if not issubclass(contextualizer._output_type, SearchQuery):
+            raise TypeError(
+                "contextualizer._output_type must be SearchQuery or a subclass"
+            )
+        self._contextualizer = contextualizer
+
+        if not isinstance(document_service, DocumentService):
+            raise TypeError("document_service must be an instance of DocumentService")
+        self._document_service = document_service
+        self._verbose = verbose
+
+    def _build_loader_input(
+        self,
+        query: str,
+        chat_history: Sequence[BaseMessage] = (),
+    ) -> SearchQuery:
+        """
+        Based on the natural-language `query`, return an instance of `SearchQuery`,
+        which can then be used to invoke the `DocumentService.search_client.search`.
+        Use the `RetrieverContextualizer` to do this.
+        """
+        contextualizer_input = {
+            "chat_history": chat_history,
+            "user_input": query,
+        }
+        db_query = self._contextualizer.invoke(contextualizer_input)
+        if self._verbose:
+            logger.info(f"Contextualizer output: {db_query}")
+        if not isinstance(db_query, SearchQuery):
+            raise TypeError(
+                f"Expected SearchQuery or subclass but got {type(db_query).__name__}"
+            )
+        return db_query
+
+    def _fetch_documents(self, loader_input: SearchQuery) -> list[Document]:
+        """
+        Return a list of `langchain_core.documents.Document`s based on the user's query
+        (and chat_history if available).
+        `load_input` shall be an instance of `SearchQuery` or a subclass, in this context.
+        """
+        db_results = self._document_service.search_client.search(loader_input)
+        if self._verbose:
+            logger.info(f"Retrieved {len(db_results)} documents from PGVector.")
+        return [
+            self._build_langchain_document_from_pgvector_document(result.document)
+            for result in db_results
+        ]
+
+    def _build_langchain_document_from_pgvector_document(
+        self, pgvector_document: JournalDocument
+    ) -> Document:
+        """
+        Build a LangChain document from a PGVector document.
+        """
+        return Document(
+            page_content=str(pgvector_document.content),
+            metadata=pgvector_document.document_metadata,
+        )

logseq_retriever/uploaders/pgvector/__init__.py

@@ -0,0 +1,9 @@
+from logseq_retriever.uploaders.pgvector.journal_corpus_manager import (
+    JournalCorpusManagerConfig,
+    JournalCorpusManager,
+)
+
+__all__ = [
+    "JournalCorpusManagerConfig",
+    "JournalCorpusManager",
+]

logseq_retriever/uploaders/pgvector/journal_corpus_manager.py

@@ -0,0 +1,72 @@
+import re
+from typing import Any, Type
+
+from pgvector_template.core import (
+    BaseCorpusManager,
+    BaseCorpusManagerConfig,
+    BaseDocument,
+    BaseDocumentMetadata,
+)
+
+from logseq_retriever.models.journal_pgvector import (
+    JournalDocument,
+    JournalDocumentMetadata,
+)
+
+
+class JournalCorpusManagerConfig(BaseCorpusManagerConfig):
+    """Configuration for Logseq journal `JournalCorpusManager`."""
+
+    schema_name: str = "logseq_journal"
+    """Name of the schema to use for the corpus manager"""
+    document_cls: Type[BaseDocument] = JournalDocument
+    """Class to use for the document model"""
+    document_metadata_cls: Type[BaseDocumentMetadata] = JournalDocumentMetadata
+    """Class to use for the document metadata model"""
+    # embedding_provider: BaseEmbeddingProvider # is still required
+
+
+class JournalCorpusManager(BaseCorpusManager):
+    """
+    CorpusManager declaration for Logseq journals. Each `Corpus` is the entire entry for a given date.
+    """
+
+    def _split_corpus(self, content: str, **kwargs) -> list[str]:
+        """Split the journal file on root-level bullet points"""
+        split_content = content.split("\n-")
+        return [
+            cleaned_chunk
+            for chunk in split_content
+            if (cleaned_chunk := chunk.strip().removeprefix("-").removeprefix(" "))
+        ]
+
+    def _extract_chunk_metadata(self, content: str, **kwargs) -> dict[str, Any]:
+        """Extract metadata from chunk content"""
+        # Add some basic metadata about the chunk
+        split_content = content.split()
+        return {
+            "chunk_len": len(content),
+            "word_count": len(split_content),
+            "references": self._extract_chunk_references(split_content),
+            "anchor_ids": self._extract_anchor_ids(content),
+        }
+
+    def _extract_chunk_references(self, split_content: list[str]) -> list[str]:
+        """
+        Extract references to other Logseq corpora, including other journals.
+        Expected to start with `#`, e.g. `#2025-07-07`, `#cookout`.
+        Special chars `!?,:'"\\` break references. `\\` is ignored.
+        """
+        references = []
+        for word in split_content:
+            if word.startswith("#"):
+                ref = word.lstrip("#").rstrip("#").replace("\\", "")
+                for char in "!?,:'\"":
+                    ref = ref.split(char)[0]
+                if ref:
+                    references.append(ref)
+        return references
+
+    def _extract_anchor_ids(self, content: str) -> list[str]:
+        """Extract Logseq anchor IDs from content (id:: <uuid>)"""
+        return re.findall(r"id:: ([a-f0-9-]{36})", content)

logseq_retriever-0.4.1.dist-info/METADATA

@@ -0,0 +1,102 @@
+Metadata-Version: 2.4
+Name: logseq-retriever
+Version: 0.4.1
+Summary: Python library for loading and retrieving Logseq documents
+Author-email: DL <v49t9zpqd@mozmail.com>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: langchain<0.4.0,>=0.3.24
+Requires-Dist: pydantic<3.0,>=2.11
+Requires-Dist: pgvector-template>=0.3.4
+Provides-Extra: scripts
+Requires-Dist: python-dotenv; extra == "scripts"
+Requires-Dist: psycopg2-binary>=2.9.0; extra == "scripts"
+Requires-Dist: langchain-aws>=0.2.0; extra == "scripts"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: python-dotenv; extra == "test"
+Requires-Dist: boto3; extra == "test"
+Requires-Dist: boto3-stubs; extra == "test"
+Requires-Dist: langchain-aws>=0.2.0; extra == "test"
+Requires-Dist: psycopg[binary]>=3.0.0; extra == "test"
+Requires-Dist: ty; extra == "test"
+Requires-Dist: ruff; extra == "test"
+Provides-Extra: dist
+Requires-Dist: build>=1.2.2; extra == "dist"
+Requires-Dist: twine>=6.1.0; extra == "dist"
+Dynamic: license-file
+
+# Logseq Retriever
+Python library for loading and retrieving Logseq documents.
+
+---
+## Components
+This section provides an overview of the components provided, listed by type
+
+
+### Retrievers
+Retrievers inject context into a conversation. Works in tandem with a Contextualizer and `Document` Loader.
+- **Input**:
+  - natural-language user-input, usually query-like
+  - (optional) chat history
+- **Output**:
+  - list of `Document`s to provide context for an LLM to answer the user-input
+
+#### Implementations
+- `LogseqJournalDateRangeRetriever`
+  - retrieve Logseq journal `Document`s, intended for queries that require context from a date range
+  - required to set up:
+    - `RetrieverContextualizer`
+    - `LogseqJournalLoader`
+  - examples:
+    - "What did I do over Christmas break 2024?"
+    - "How did I spend the last Independence Day?"
+
+
+### Contextualizers
+Contextualizers serve as the bridge between natural-language input and a downstream component that
+handles fetching of relevant `Document`s.
+- **Input**:
+  - natural-language user-input, usually query-like
+  - (optional) chat history
+- **Output**:
+  - structured downstream query, based on
+
+In this library, an instance of `RetrieverContextualizer` is provided directly to 
+`Retriever`s during the latter's instantiation. To set up the `RetrieverContextualizer`, provide
+`RetrieverContextualizerProps`, which includes:
+- `llm` - this is the backbone of the contextualizer
+- `prompt` - instructions provided to the LLM
+- `output_schema` - (optional) structured schema used to fetch relevant `Document`s
+  - if no schema is provided, a string shall be returned instead
+- other flags and settings
+
+
+### Loaders
+Loaders are one type of component that can fetch relevant `Document`s. Loaders are typically specific to
+a corresponding `Retriever` component.
+- **Input**:
+  - each loader specifies its own schema
+    - the Contextualizer is usually responsible for creating an instance of the query obj to act upon
+- **Output**:
+  - `list[Document]`
+
+#### Implementations
+- `LogseqJournalFilesystemLoader`
+  - loads from the filesystem, where journal files are expected to be present at specified path
+
+
+---
+## Scripts
+
+### PGVector
+
+#### `upload_journal`
+
+usage: `python scripts/upload_journal_to_pgvector.py [-h] [-p PATH] from_date to_date`

logseq_retriever-0.4.1.dist-info/RECORD

@@ -0,0 +1,21 @@
+logseq_retriever/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+logseq_retriever/loaders/__init__.py,sha256=L5qqUEjx9A5sWdzHG-4c04z_KTxPnihJDh15gVmtWcs,507
+logseq_retriever/loaders/journal_document_metadata.py,sha256=q5XDHX95gnOePP9v-Fwv79sYBrqP8p9Qh4ZyFoddPRk,744
+logseq_retriever/loaders/journal_filesystem_loader.py,sha256=w0Z5Z8YLe8TLWTKbakRTPJ7dRr9qUMJZUFik9IHb9RE,6244
+logseq_retriever/loaders/journal_loader.py,sha256=9e2bK_LDrTYHlUlHamD7MgGzFq8DBCAiBVweXAUtnzM,433
+logseq_retriever/loaders/journal_loader_input.py,sha256=A88-NueZs6h9fNRs5Vm6Y00PgRZImYji2p6Wum-2edA,2963
+logseq_retriever/models/__init__.py,sha256=nkbyo3WbzF4Z4XQAM-IT_SDNFytRo6UGEc_L84j4K-o,346
+logseq_retriever/models/journal_pgvector.py,sha256=wtn2SM_GKqK45QZnTH8tEoDlBPWOlGfcRWGl65YvcF8,3759
+logseq_retriever/retrievers/__init__.py,sha256=6MOLyW01yHK_AYpi_L7fKD9nHhJ6XYds_pGi5bAiQBM,597
+logseq_retriever/retrievers/contextualizer.py,sha256=61YYXVanrHdgL_96bRMtob7Rzs0LuhEJpVb6lOwo__A,6141
+logseq_retriever/retrievers/journal_date_range_retriever.py,sha256=dB0jDY5JaA_YjZf2G1tGC2uy_e0gVV0PSX61rb7pe4E,2850
+logseq_retriever/retrievers/journal_retriever.py,sha256=e7MK2GRVDqo56B-xLEpF0HElZxNWVm50O1zrLx7uLio,3768
+logseq_retriever/retrievers/pgvector_journal_retriever.py,sha256=vYcnicWLdqBIgSgjb_xnNMKvrSyCDGKz6rmGwr9RTdw,3525
+logseq_retriever/uploaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+logseq_retriever/uploaders/pgvector/__init__.py,sha256=Y5wQzHtrEXaUwP05JZ8RqP5en_udUUV-f8ptbGl4MQE,210
+logseq_retriever/uploaders/pgvector/journal_corpus_manager.py,sha256=7xCV5cgI4mJHPxTG92WMH8ylXOjB0PP7obDKwXxNuOU,2655
+logseq_retriever-0.4.1.dist-info/licenses/LICENSE,sha256=4chADZoF7TXixgJtj6FYx2PiAjCMreSUMHevGcgdSG4,1069
+logseq_retriever-0.4.1.dist-info/METADATA,sha256=ZwfjrzO5REJpb_cJ4Qh_rg8zXL_6DqOMd5waNBEgeUU,3563
+logseq_retriever-0.4.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+logseq_retriever-0.4.1.dist-info/top_level.txt,sha256=1WZc9D05rwFgELxnbrEddvzcscLyUXD1ODwLpVx0r90,17
+logseq_retriever-0.4.1.dist-info/RECORD,,

logseq_retriever-0.4.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

logseq_retriever-0.4.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 David Ge Liu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

logseq_retriever-0.4.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+logseq_retriever