microrag 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. microrag-0.1.0/.github/workflows/ci.yml +55 -0
  2. microrag-0.1.0/.github/workflows/publish.yml +93 -0
  3. microrag-0.1.0/.gitignore +77 -0
  4. microrag-0.1.0/LICENSE +21 -0
  5. microrag-0.1.0/Makefile +224 -0
  6. microrag-0.1.0/PKG-INFO +341 -0
  7. microrag-0.1.0/README.md +313 -0
  8. microrag-0.1.0/examples/advanced_config.py +110 -0
  9. microrag-0.1.0/examples/basic_usage.py +81 -0
  10. microrag-0.1.0/examples/faq_search.py +110 -0
  11. microrag-0.1.0/pyproject.toml +102 -0
  12. microrag-0.1.0/src/microrag/__init__.py +57 -0
  13. microrag-0.1.0/src/microrag/config.py +86 -0
  14. microrag-0.1.0/src/microrag/core.py +293 -0
  15. microrag-0.1.0/src/microrag/embedding/__init__.py +5 -0
  16. microrag-0.1.0/src/microrag/embedding/model.py +118 -0
  17. microrag-0.1.0/src/microrag/exceptions.py +29 -0
  18. microrag-0.1.0/src/microrag/models.py +45 -0
  19. microrag-0.1.0/src/microrag/query_processor.py +128 -0
  20. microrag-0.1.0/src/microrag/search/__init__.py +6 -0
  21. microrag-0.1.0/src/microrag/search/bm25.py +79 -0
  22. microrag-0.1.0/src/microrag/search/hybrid.py +234 -0
  23. microrag-0.1.0/src/microrag/stopwords.py +114 -0
  24. microrag-0.1.0/src/microrag/storage/__init__.py +6 -0
  25. microrag-0.1.0/src/microrag/storage/base.py +122 -0
  26. microrag-0.1.0/src/microrag/storage/duckdb.py +296 -0
  27. microrag-0.1.0/src/microrag/utils.py +151 -0
  28. microrag-0.1.0/tests/__init__.py +1 -0
  29. microrag-0.1.0/tests/conftest.py +133 -0
  30. microrag-0.1.0/tests/test_bm25.py +65 -0
  31. microrag-0.1.0/tests/test_config.py +61 -0
  32. microrag-0.1.0/tests/test_integration.py +108 -0
  33. microrag-0.1.0/tests/test_models.py +28 -0
  34. microrag-0.1.0/tests/test_query_processor.py +52 -0
  35. microrag-0.1.0/tests/test_storage.py +76 -0
  36. microrag-0.1.0/tests/test_utils.py +91 -0
  37. microrag-0.1.0/uv.lock +1414 -0
@@ -0,0 +1,55 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [master, main]
6
+ pull_request:
7
+ branches: [master, main]
8
+
9
+ jobs:
10
+ lint:
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - uses: actions/checkout@v4
14
+
15
+ - name: Set up Python
16
+ uses: actions/setup-python@v5
17
+ with:
18
+ python-version: "3.12"
19
+
20
+ - name: Install uv
21
+ uses: astral-sh/setup-uv@v4
22
+
23
+ - name: Install dependencies
24
+ run: uv sync --all-groups
25
+
26
+ - name: Run ruff format
27
+ run: uv run ruff format --check src/microrag tests
28
+
29
+ - name: Run ruff lint
30
+ run: uv run ruff check src/microrag tests
31
+
32
+ - name: Run mypy
33
+ run: uv run mypy src/microrag
34
+
35
+ test:
36
+ runs-on: ubuntu-latest
37
+ strategy:
38
+ matrix:
39
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
40
+ steps:
41
+ - uses: actions/checkout@v4
42
+
43
+ - name: Set up Python
44
+ uses: actions/setup-python@v5
45
+ with:
46
+ python-version: ${{ matrix.python-version }}
47
+
48
+ - name: Install uv
49
+ uses: astral-sh/setup-uv@v4
50
+
51
+ - name: Install dependencies
52
+ run: uv sync --all-groups
53
+
54
+ - name: Run tests
55
+ run: uv run pytest --cov=microrag --cov-report=term-missing
@@ -0,0 +1,93 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*"
7
+
8
+ jobs:
9
+ lint:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v4
13
+
14
+ - name: Set up Python
15
+ uses: actions/setup-python@v5
16
+ with:
17
+ python-version: "3.12"
18
+
19
+ - name: Install uv
20
+ uses: astral-sh/setup-uv@v4
21
+
22
+ - name: Install dependencies
23
+ run: uv sync --all-groups
24
+
25
+ - name: Run ruff format
26
+ run: uv run ruff format --check src/microrag tests
27
+
28
+ - name: Run ruff lint
29
+ run: uv run ruff check src/microrag tests
30
+
31
+ - name: Run mypy
32
+ run: uv run mypy src/microrag
33
+
34
+ test:
35
+ runs-on: ubuntu-latest
36
+ strategy:
37
+ matrix:
38
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
39
+ steps:
40
+ - uses: actions/checkout@v4
41
+
42
+ - name: Set up Python
43
+ uses: actions/setup-python@v5
44
+ with:
45
+ python-version: ${{ matrix.python-version }}
46
+
47
+ - name: Install uv
48
+ uses: astral-sh/setup-uv@v4
49
+
50
+ - name: Install dependencies
51
+ run: uv sync --all-groups
52
+
53
+ - name: Run tests
54
+ run: uv run pytest --cov=microrag --cov-report=term-missing
55
+
56
+ build:
57
+ needs: [lint, test]
58
+ runs-on: ubuntu-latest
59
+ steps:
60
+ - uses: actions/checkout@v4
61
+
62
+ - name: Set up Python
63
+ uses: actions/setup-python@v5
64
+ with:
65
+ python-version: "3.12"
66
+
67
+ - name: Install build tools
68
+ run: pip install build
69
+
70
+ - name: Build package
71
+ run: python -m build
72
+
73
+ - name: Upload artifacts
74
+ uses: actions/upload-artifact@v4
75
+ with:
76
+ name: dist
77
+ path: dist/
78
+
79
+ publish:
80
+ needs: build
81
+ runs-on: ubuntu-latest
82
+ environment: pypi
83
+ permissions:
84
+ id-token: write
85
+ steps:
86
+ - name: Download artifacts
87
+ uses: actions/download-artifact@v4
88
+ with:
89
+ name: dist
90
+ path: dist/
91
+
92
+ - name: Publish to PyPI
93
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,77 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ .idea
6
+ media
7
+
8
+ # C extensions
9
+ *.so
10
+
11
+ # Distribution / packaging
12
+ .Python
13
+ env/
14
+ build/
15
+ develop-eggs/
16
+ dist/
17
+ downloads/
18
+ eggs/
19
+ .eggs/
20
+ parts/
21
+ sdist/
22
+ var/
23
+ *.egg-info/
24
+ .installed.cfg
25
+ *.egg
26
+
27
+ configs/local_settings.py
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .coverage
43
+ .coverage.*
44
+ .cache
45
+ nosetests.xml
46
+ coverage.xml
47
+ *,cover
48
+ .hypothesis/
49
+
50
+ # Translations
51
+ *.pot
52
+
53
+ # Django stuff:
54
+ *.log
55
+
56
+ # Sphinx documentation
57
+ docs/_build/
58
+
59
+ # PyBuilder
60
+ target/
61
+
62
+ #Ipython Notebook
63
+ .ipynb_checkpoints
64
+ venv/*
65
+ env/*
66
+ .venv/*
67
+ .env/*
68
+
69
+ # Idea modules
70
+ *.iml
71
+
72
+ .ruff_cache
73
+ .pytest_cache
74
+ .mypy_cache
75
+
76
+ .claude/
77
+ CLAUDE.md
microrag-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Dave Allie
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,224 @@
1
+ .PHONY: help
2
+
3
+ #################################################################################
4
+ # GLOBALS #
5
+ #################################################################################
6
+
7
+ PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
8
+ PROJECT_NAME = microrag
9
+
10
+ SRC_PATH = src/microrag
11
+ LINT_PATHS = src tests
12
+
13
+ VENV := $(or ${VIRTUAL_ENV},${VIRTUAL_ENV},.venv)
14
+ PYTHON = $(VENV)/bin/python
15
+ PIP = $(VENV)/bin/pip
16
+ UV = $(VENV)/bin/uv
17
+
18
+ PYTHON_VERSION = 3.12
19
+ PYTHONPATH := $(or ${PYTHONPATH},${PYTHONPATH},.)
20
+
21
+ export ${PYTHONPATH}
22
+
23
+ #######################
24
+ ### System commands
25
+ #######################
26
+ .PHONY: sys/changelog
27
+ ## Generating changelog file
28
+ sys/changelog:
29
+ @echo "Generating CHANGELOG.md..."
30
+ @echo "" > CHANGELOG.md;
31
+ @previous_tag=0; \
32
+ for current_tag in $$(git tag --sort=-creatordate | grep '^v'); do \
33
+ if [ "$$previous_tag" != 0 ]; then \
34
+ tag_date=$$(git log -1 --pretty=format:'%ad' --date=short $${previous_tag}); \
35
+ printf "\n## $${previous_tag} ($${tag_date})\n\n" >> CHANGELOG.md; \
36
+ git log $${current_tag}...$${previous_tag} --pretty=format:'* %s [%an]' --reverse | grep -v Merge >> CHANGELOG.md; \
37
+ printf "\n" >> CHANGELOG.md; \
38
+ fi; \
39
+ previous_tag=$${current_tag}; \
40
+ done
41
+ @echo "CHANGELOG.md generated successfully."
42
+
43
+ .PHONY: sys/tag
44
+ ## Create and push tag
45
+ sys/tag:
46
+ @read -p "Enter tag version (e.g., 1.0.0): " TAG; \
47
+ if [[ $$TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$$ ]]; then \
48
+ git tag -a v$$TAG -m v$$TAG; \
49
+ git push origin v$$TAG; \
50
+ echo "Tag v$$TAG created and pushed successfully."; \
51
+ else \
52
+ echo "Invalid tag format. Please use X.Y.Z (e.g., 1.0.0)"; \
53
+ exit 1; \
54
+ fi
55
+
56
+ #######################
57
+ ### Virtual environment
58
+ #######################
59
+
60
+ .PHONY: venv/create
61
+ venv/create: ## Create virtual environment
62
+ @echo "create virtual environment..."
63
+ python -m venv ${VENV}
64
+ @echo "done"
65
+ @echo
66
+
67
+ .PHONY: venv/install/main
68
+ ## Install main dependencies
69
+ venv/install/main:
70
+ @echo "install virtual environment..."
71
+ $(PIP) install uv
72
+ $(UV) sync --no-group dev
73
+
74
+ .PHONY: venv/install/all
75
+ ## Install all dependencies
76
+ venv/install/all:
77
+ @echo "install virtual environment..."
78
+ $(PIP) install uv
79
+ $(UV) sync --all-groups
80
+
81
+ #################################################################################
82
+ # COMMANDS #
83
+ #################################################################################
84
+
85
+ ########################################
86
+ ### Code style & formatting tools
87
+ ########################################
88
+
89
+ .PHONY: lint/ruff
90
+ lint/ruff:
91
+ @echo "linting using ruff..."
92
+ uv run ruff format --check $(LINT_PATHS)
93
+ uv run ruff check $(LINT_PATHS)
94
+ @echo "done"
95
+ @echo
96
+
97
+ .PHONY: lint/mypy
98
+ lint/mypy:
99
+ @echo "type checking using mypy..."
100
+ uv run mypy $(SRC_PATH)
101
+ @echo "done"
102
+ @echo
103
+
104
+ .PHONY: lint
105
+ ## Run all linters
106
+ lint: lint/ruff lint/mypy
107
+
108
+ .PHONY: format
109
+ ## Format source code
110
+ format:
111
+ @echo "formatting using ruff..."
112
+ uv run ruff format $(LINT_PATHS)
113
+ uv run ruff check --fix $(LINT_PATHS)
114
+ @echo "done"
115
+ @echo
116
+
117
+ .PHONY: clean
118
+ ## Delete all compiled Python files and caches
119
+ clean:
120
+ @echo "Clear cache directories"
121
+ rm -rf .mypy_cache .pytest_cache .coverage
122
+ @rm -rf `find . -name __pycache__`
123
+ @rm -rf `find . -type f -name '*.py[co]' `
124
+ @rm -rf `find . -type f -name '*~' `
125
+ @rm -rf `find . -type f -name '.*~' `
126
+ @rm -rf `find . -type f -name '@*' `
127
+ @rm -rf `find . -type f -name '#*#' `
128
+ @rm -rf `find . -type f -name '*.orig' `
129
+ @rm -rf `find . -type f -name '*.rej' `
130
+ @rm -rf .coverage
131
+ @rm -rf coverage.html
132
+ @rm -rf coverage.xml
133
+ @rm -rf htmlcov
134
+ @rm -rf build
135
+ @rm -rf cover
136
+ @rm -rf .develop
137
+ @rm -rf .flake
138
+ @rm -rf .install-deps
139
+ @rm -rf *.egg-info
140
+ @rm -rf .pytest_cache
141
+ @rm -rf .ruff_cache
142
+ @rm -rf .mypy_cache
143
+ @rm -rf dist
144
+ @rm -rf test-reports
145
+
146
+
147
+ ####################
148
+ ### Tests
149
+ ####################
150
+
151
+ .PHONY: test
152
+ ## Run all tests with coverage
153
+ test:
154
+ uv run pytest --cov=$(SRC_PATH) --cov-report=term-missing
155
+
156
+ ####################
157
+ ### Examples
158
+ ####################
159
+
160
+ .PHONY: example
161
+ ## Run example: make example name=basic_usage
162
+ example:
163
+ uv run python examples/$(name).py
164
+
165
+ #################################################################################
166
+ # Self Documenting Commands #
167
+ #################################################################################
168
+
169
+ .DEFAULT_GOAL := help
170
+
171
+ # Inspired by <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
172
+ # sed script explained:
173
+ # /^##/:
174
+ # * save line in hold space
175
+ # * purge line
176
+ # * Loop:
177
+ # * append newline + line to hold space
178
+ # * go to next line
179
+ # * if line starts with doc comment, strip comment character off and loop
180
+ # * remove target prerequisites
181
+ # * append hold space (+ newline) to line
182
+ # * replace newline plus comments by `---`
183
+ # * print line
184
+ # Separate expressions are necessary because labels cannot be delimited by
185
+ # semicolon; see <http://stackoverflow.com/a/11799865/1968>
186
+ .PHONY: help
187
+ help:
188
+ @echo "$$(tput bold)Available rules:$$(tput sgr0)"
189
+ @echo
190
+ @sed -n -e "/^## / { \
191
+ h; \
192
+ s/.*//; \
193
+ :doc" \
194
+ -e "H; \
195
+ n; \
196
+ s/^## //; \
197
+ t doc" \
198
+ -e "s/:.*//; \
199
+ G; \
200
+ s/\\n## /---/; \
201
+ s/\\n/ /g; \
202
+ p; \
203
+ }" ${MAKEFILE_LIST} \
204
+ | LC_ALL='C' sort --ignore-case \
205
+ | awk -F '---' \
206
+ -v ncol=$$(tput cols) \
207
+ -v indent=19 \
208
+ -v col_on="$$(tput setaf 6)" \
209
+ -v col_off="$$(tput sgr0)" \
210
+ '{ \
211
+ printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
212
+ n = split($$2, words, " "); \
213
+ line_length = ncol - indent; \
214
+ for (i = 1; i <= n; i++) { \
215
+ line_length -= length(words[i]) + 1; \
216
+ if (line_length <= 0) { \
217
+ line_length = ncol - indent - length(words[i]) - 1; \
218
+ printf "\n%*s ", -indent, " "; \
219
+ } \
220
+ printf "%s ", words[i]; \
221
+ } \
222
+ printf "\n"; \
223
+ }' \
224
+ | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')