omop-emb 0.2.1__tar.gz

omop_emb-0.2.1/.github/workflows/docs.yml

@@ -0,0 +1,31 @@
+name: docs 
+on:
+  push:
+    branches:
+      - master 
+      - main
+    workflow_dispatch:
+
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
+      - uses: actions/cache@v4
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: ~/.cache 
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install mkdocs mkdocs-material mkdocstrings-python mkdocs-mermaid2-plugin mkdocs-table-reader-plugin
+      - run: mkdocs gh-deploy --force

omop_emb-0.2.1/.github/workflows/lint-pr.yml

@@ -0,0 +1,17 @@
+# This workflow runs a linter on pull requests to ensure the right prefix is used for semantic-versioning
+name: "Lint PR"
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

omop_emb-0.2.1/.github/workflows/release.yml

@@ -0,0 +1,99 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16  # Have the pgvector extension pre-installed
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d postgres"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    env:
+      TEST_DB_HOST: localhost
+      TEST_DB_PORT: 5432
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      TEST_DATABASE_NAME: test_omop_emb
+      TEST_DB_USERNAME: test
+      TEST_DB_PASSWORD: test
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Run tests
+        run: uv run pytest
+
+  release:
+    needs: test
+    runs-on: ubuntu-latest
+    concurrency: release
+    permissions:
+      id-token: write # Required for Trusted Publishing to PyPI
+      contents: write # Required for Semantic Release to push tags/labels
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/omop-emb
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          # Use GITHUB_TOKEN for standard releases
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Semantic Release
+        id: semantic
+        uses: cycjimmy/semantic-release-action@v6
+        with:
+          extra_plugins: |
+            @semantic-release/changelog
+            @semantic-release/git
+            @semantic-release/exec
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install uv
+        if: steps.semantic.outputs.new_release_published == 'true'
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+
+      - name: Build and publish to PyPI
+        if: steps.semantic.outputs.new_release_published == 'true'
+        run: |
+          uv build
+          # uv uses the id-token permission for OIDC "Trusted Publishing"
+          uv publish
+
+      - name: Trigger docs update
+        if: steps.semantic.outputs.new_release_published == 'true'
+        run: gh workflow run docs.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

omop_emb-0.2.1/.gitignore

@@ -0,0 +1,207 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+.vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

omop_emb-0.2.1/.releaserc.json

@@ -0,0 +1,22 @@
+{
+  "branches": ["main"],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    "@semantic-release/changelog",
+    [
+      "@semantic-release/exec",
+      {
+        "prepareCmd": "sed -i 's/version = \".*\"/version = \"${nextRelease.version}\"/' pyproject.toml"
+      }
+    ],
+    [
+      "@semantic-release/git",
+      {
+        "assets": ["CHANGELOG.md", "pyproject.toml"],
+        "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+      }
+    ],
+    "@semantic-release/github"
+  ]
+}

omop_emb-0.2.1/CHANGELOG.md

@@ -0,0 +1,20 @@
+## [0.2.1](https://github.com/AustralianCancerDataNetwork/omop-emb/compare/v0.2.0...v0.2.1) (2026-04-01)
+
+
+### Bug Fixes
+
+* trigger PyPI publish after OIDC config ([2fb4b40](https://github.com/AustralianCancerDataNetwork/omop-emb/commit/2fb4b40c4d9221ceac1ae1f3a25f7059380b53bd))
+
+# [0.2.0](https://github.com/AustralianCancerDataNetwork/omop-emb/compare/v0.1.0...v0.2.0) (2026-04-01)
+
+
+### Bug Fixes
+
+* pull newest omop-llm ([c3fb805](https://github.com/AustralianCancerDataNetwork/omop-emb/commit/c3fb8050d84804f48a44966e5d4271465485652d))
+* remove dupblicat optional-dep ([c7a58c1](https://github.com/AustralianCancerDataNetwork/omop-emb/commit/c7a58c16078615a42e5dc6787e0abb44449ab273))
+* Remove duplicate "scripts" key after PR ([9d8369d](https://github.com/AustralianCancerDataNetwork/omop-emb/commit/9d8369d9c54aecddb39efcefada186748eb6ff0f))
+
+
+### Features
+
+* diverse interface for embedding backends ([3d696dc](https://github.com/AustralianCancerDataNetwork/omop-emb/commit/3d696dc906ac7a94ee7464b82459b0a9b9db3ed2))

omop_emb-0.2.1/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: omop-emb
+Version: 0.2.1
+Summary: Embedding extension to omop-graph
+Author-email: Nico Loesch <n.loesch@unsw.edu.au>
+License-Expression: Apache-2.0
+Keywords: LLM-grounding,OHDSI,OMOP,clinical-data,health-informatics,knowledge-graph,sqlalchemy
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Requires-Python: >=3.12
+Requires-Dist: numpy>=1.26
+Requires-Dist: omop-alchemy>=0.5.7
+Requires-Dist: omop-llm
+Requires-Dist: orm-loader>=0.3.15
+Requires-Dist: psycopg2-binary>=2.9.11
+Requires-Dist: sqlalchemy>=2.0.45
+Requires-Dist: typing-extensions>=4.15.0
+Provides-Extra: all
+Requires-Dist: faiss-cpu>=1.8.0; extra == 'all'
+Requires-Dist: h5py; extra == 'all'
+Requires-Dist: pgvector; extra == 'all'
+Provides-Extra: faiss
+Requires-Dist: faiss-cpu>=1.8.0; extra == 'faiss'
+Requires-Dist: h5py; extra == 'faiss'
+Provides-Extra: pgvector
+Requires-Dist: pgvector; extra == 'pgvector'
+Description-Content-Type: text/markdown
+
+# omop-emb
+Embedding layer for OMOP CDM.
+
+## Installation
+
+`omop-emb` now exposes backend-specific optional dependencies so installation
+can match the embedding backend you actually intend to use.
+
+```bash
+pip install "omop-emb[postgres]"
+pip install "omop-emb[faiss]"
+pip install "omop-emb[all]"
+```
+
+Notes:
+
+- `postgres` installs the PostgreSQL/pgvector dependencies.
+- `faiss` installs the FAISS-based backend dependencies. This currently only includes CPU support
+- `all` installs both backend stacks for development or mixed environments.
+- A plain `pip install omop-emb` installs the shared core package only.
+- PostgreSQL-specific embedding dependencies are now optional, but `omop-emb`
+  still requires some database backend for OMOP access and model registration.
+- Non-PostgreSQL database backends have not yet been tested.
+
+Extended documentation can be found [here](https://AustralianCancerDataNetwork.github.io/omop-emb).
+
+# Project Roadmap
+
+- [x] Interface for postgres storage of vectors
+- [x] Interface for FAISS storage of embeddings
+- [ ] Extensive unit testing
+    - [ ] Backend testing
+    - [ ] Corruption and restoration of DB testing
+- [ ] Support non-Flat indices for each backend
+- [ ] `faiss` GPU support
+- [ ] [`pgvectorscale`](https://github.com/timescale/pgvectorscale) support
+- [ ] Vector-quantisation for more efficient storage

omop_emb-0.2.1/README.md

@@ -0,0 +1,37 @@
+# omop-emb
+Embedding layer for OMOP CDM.
+
+## Installation
+
+`omop-emb` now exposes backend-specific optional dependencies so installation
+can match the embedding backend you actually intend to use.
+
+```bash
+pip install "omop-emb[postgres]"
+pip install "omop-emb[faiss]"
+pip install "omop-emb[all]"
+```
+
+Notes:
+
+- `postgres` installs the PostgreSQL/pgvector dependencies.
+- `faiss` installs the FAISS-based backend dependencies. This currently only includes CPU support
+- `all` installs both backend stacks for development or mixed environments.
+- A plain `pip install omop-emb` installs the shared core package only.
+- PostgreSQL-specific embedding dependencies are now optional, but `omop-emb`
+  still requires some database backend for OMOP access and model registration.
+- Non-PostgreSQL database backends have not yet been tested.
+
+Extended documentation can be found [here](https://AustralianCancerDataNetwork.github.io/omop-emb).
+
+# Project Roadmap
+
+- [x] Interface for postgres storage of vectors
+- [x] Interface for FAISS storage of embeddings
+- [ ] Extensive unit testing
+    - [ ] Backend testing
+    - [ ] Corruption and restoration of DB testing
+- [ ] Support non-Flat indices for each backend
+- [ ] `faiss` GPU support
+- [ ] [`pgvectorscale`](https://github.com/timescale/pgvectorscale) support
+- [ ] Vector-quantisation for more efficient storage

omop_emb-0.2.1/docs/index.md

@@ -0,0 +1,42 @@
+# OMOP Embeddings
+
+`omop-emb` is an optional package to super-charge [`omop-graph`](https://AustralianCancerDataNetwork.github.io/omop-graph) and provide additional graph reasoning tools for information retrieval and RAG-based knowledge extraction.
+
+The package currently supports:
+
+- dynamic embedding model registration
+  - multiple embedding models can be stored in the respective backend
+- embedding and lookup for OMOP concepts
+- supports various backends with a PostgreSQL linker
+  - [pgvector](https://github.com/pgvector/pgvector): storage in the original OMOP database
+  - [FAISS](https://github.com/facebookresearch/faiss): efficient storage on disk for low-RAM applications 
+- Extension to [`omop-alchemy`](https://AustralianCancerDataNetwork.github.io/OMOP_Alchemy/) to support new tables
+- CLI scripts to add embeddings to an already existing OMOP CDM
+
+## Installation
+
+Install the backend you actually want to use:
+
+```bash
+pip install "omop-emb[postgres]"
+pip install "omop-emb[faiss]"
+pip install "omop-emb[all]"
+```
+
+A plain `pip install omop-emb` installs only the shared core package.
+
+At runtime, backend choice should also be explicit. The intended direction is:
+
+- install-time choice via extras
+- runtime choice via config such as `OMOP_EMB_BACKEND=postgres` or `OMOP_EMB_BACKEND=faiss` or passing it as an argument to the respective interface (e.g. see [CLI reference](usage/cli.md))
+
+
+!!! info "Important caveats"
+
+  - `omop-emb` depends on an OMOP PostgreSQL database for storage of embeddings (pgvector) or to keep track of already embedded concepts.
+
+
+## Documentation overview
+- [Installation](usage/installation.md)
+- [Backend Selection](usage/backend-selection.md)
+- [CLI Reference](usage/cli.md)

omop_emb-0.2.1/docs/usage/backend-selection.md

@@ -0,0 +1,89 @@
+# Backend Selection
+
+`omop-emb` now has a backend abstraction layer so embedding storage and
+retrieval can be selected explicitly instead of being inferred implicitly from
+whatever happens to be installed.
+
+## Supported backend names
+
+The current backend factory recognizes:
+
+- `pgvector`: The [pgvector](https://github.com/pgvector/pgvector) extension to a standard postgres database to store embeddings directly in the database.
+- `faiss`: The [FAISS](https://github.com/facebookresearch/faiss) storage solution for on-disk storage.
+
+The default backend name is currently `postgres`.
+
+## Runtime selection
+
+The intended pattern is:
+
+1. choose the backend at install time with package extras
+2. choose the backend again at runtime explicitly
+
+Examples:
+
+```bash
+export OMOP_EMB_BACKEND=postgres
+export OMOP_EMB_BACKEND=faiss
+```
+
+You can also pass the backend name directly in Python.
+
+## Python factory
+
+The backend factory lives in `omop_emb.backends`:
+
+```python
+from omop_emb.backends import get_embedding_backend
+
+backend = get_embedding_backend("postgres")
+backend = get_embedding_backend("faiss")
+```
+
+The factory currently exposes:
+
+- `get_embedding_backend(...)`
+- `normalize_backend_name(...)`
+
+## Why explicit selection is necessary
+
+Explicit backend selection improves clarity in a multi-backend world:
+
+- users can see which backend they intended to use
+- missing optional dependencies fail clearly
+- the system avoids silent fallback between incompatible storage implementations
+
+This is especially important when embeddings affect retrieval behavior, because
+silent fallback can make users think semantic retrieval is active when it is
+not.
+
+## Dependency errors
+
+If a backend is requested but its optional dependencies are missing, the
+factory raises an explicit backend dependency error rather than falling back to
+another backend.
+
+This is the intended behavior.
+
+Examples of the error classes exposed by the backend layer:
+
+- `EmbeddingBackendDependencyError`
+- `UnknownEmbeddingBackendError`
+- `EmbeddingBackendConfigurationError`
+
+## Current scope
+
+At the moment:
+
+- the backend abstraction and backend factory exist
+- PostgreSQL and FAISS backend classes exist
+- the production CLI path still targets the PostgreSQL embedding workflow
+- PostgreSQL-specific embedding dependencies are optional, but a database
+  backend is still required for OMOP access and model registration
+- model registration is intended to remain shared and database-backed even when
+  FAISS is used for vector storage and retrieval
+- database backends other than PostgreSQL have not yet been tested
+
+So this page documents the selection model and Python interface shape now, even
+before every runtime path has been migrated to delegate through the backend
+factory.

omop_emb-0.2.1/docs/usage/cli.md

@@ -0,0 +1,55 @@
+# Embedding Generation CLI
+
+This tool generates vector embeddings for OMOP CDM concepts and stores them in the configured embedding backend.
+
+At present, the production CLI path is PostgreSQL-oriented and stores embeddings in Postgres/pgvector-backed model tables. It specifically targets concepts that do not yet have embeddings and processes them in batches.
+
+!!! note "Supported Models"
+
+    Currently supported are only Ollama models
+---
+
+## Prerequisites
+
+- **Installation**: install the PostgreSQL backend dependencies:
+
+  ```bash
+  pip install "omop-emb[postgres]"
+  ```
+
+- **Database**: Postgres implementation of OMOP CDM. See [`omop-graph` documentation](reference-missing) for information how to setup.
+- **Environment**: `OMOP_DATABASE_URL` must be exported or existing in the .env file  (e.g., `postgresql://user:pass@localhost:5432/omop`).
+- **Connectivity**: Access to an OpenAI-compatible embeddings endpoint. *Currently only Ollama supported*.
+
+!!! note "Backend Scope"
+
+    `omop-emb` now defines a backend abstraction layer for both PostgreSQL and FAISS-style storage.
+    The current `add-embeddings` CLI still targets the PostgreSQL backend path.
+
+---
+
+## `add-embeddings`
+
+### Usage
+```bash
+omop-emb add-embeddings --api-base <URL> --api-key <KEY> [OPTIONS]
+```
+where `[OPTIONS]` are optional arguments that can be specified as described below.
+
+
+### Command Options
+
+### Command Options
+
+| Option | Short | Type | Default | Description |
+| :--- | :--- | :--- | :--- | :--- |
+| **`--api-base`** | | `String` | **Required** | Base URL for the embedding API service. |
+| **`--api-key`** | | `String` | **Required** | API key for the embedding API provider. |
+| **`--index-type`** | | `IndexType` | `FLAT` | The storage index for the embeddings for retrieval. Currently supported: `FLAT`. |
+| **`--batch-size`** | `-b` | `Integer` | `100` | Number of concepts to process in each chunk. |
+| **`--model`** | `-m` | `String` | `text-embedding-3-small` | Name of the embedding model to use for generating vectors. |
+| **`--backend`** | | `Literal['pgvector', 'faiss']` | `None` | Embedding backend to use (can be replaced by `OMOP_EMB_BACKEND` env var). Requires the respective backend installed using `pip install omop-emb[pgvector or faiss]` |
+| **`--faiss-base-dir`** | | `String` | `None` | Optional base directory for FAISS backend storage. |
+| **`--standard-only`** | | `Boolean` | `False` | If set, only generate embeddings for OMOP standard concepts (`standard_concept = 'S'`). |
+| **`--vocabulary`** | | `List[String]` | `None` | Filter to embed concepts only from specific OMOP vocabularies. |
+| **`--num-embeddings`** | `-n` | `Integer` | `None` | Limit the number of concepts processed (useful for testing). |