agent-library 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. agent_library-0.7.0/.editorconfig +14 -0
  2. agent_library-0.7.0/.env.example +20 -0
  3. agent_library-0.7.0/.github/dependabot.yml +17 -0
  4. agent_library-0.7.0/.github/workflows/ci.yml +98 -0
  5. agent_library-0.7.0/.github/workflows/release.yml +124 -0
  6. agent_library-0.7.0/.gitignore +166 -0
  7. agent_library-0.7.0/.prettierrc.toml +15 -0
  8. agent_library-0.7.0/.ruff.toml +51 -0
  9. agent_library-0.7.0/LICENSE +21 -0
  10. agent_library-0.7.0/Makefile +83 -0
  11. agent_library-0.7.0/PKG-INFO +201 -0
  12. agent_library-0.7.0/README.md +161 -0
  13. agent_library-0.7.0/eval_context_tools.py +590 -0
  14. agent_library-0.7.0/librarian/.env.example +11 -0
  15. agent_library-0.7.0/librarian/__init__.py +52 -0
  16. agent_library-0.7.0/librarian/cli.py +1265 -0
  17. agent_library-0.7.0/librarian/config.py +113 -0
  18. agent_library-0.7.0/librarian/indexing.py +146 -0
  19. agent_library-0.7.0/librarian/processing/__init__.py +19 -0
  20. agent_library-0.7.0/librarian/processing/embed/__init__.py +171 -0
  21. agent_library-0.7.0/librarian/processing/embed/base.py +98 -0
  22. agent_library-0.7.0/librarian/processing/embed/local.py +187 -0
  23. agent_library-0.7.0/librarian/processing/embed/openai.py +254 -0
  24. agent_library-0.7.0/librarian/processing/manager.py +241 -0
  25. agent_library-0.7.0/librarian/processing/parsers/__init__.py +18 -0
  26. agent_library-0.7.0/librarian/processing/parsers/base.py +78 -0
  27. agent_library-0.7.0/librarian/processing/parsers/md.py +243 -0
  28. agent_library-0.7.0/librarian/processing/parsers/obsidian.py +172 -0
  29. agent_library-0.7.0/librarian/processing/transform/__init__.py +12 -0
  30. agent_library-0.7.0/librarian/processing/transform/chunker.py +392 -0
  31. agent_library-0.7.0/librarian/providers/__init__.py +29 -0
  32. agent_library-0.7.0/librarian/providers/anthropic.py +207 -0
  33. agent_library-0.7.0/librarian/providers/base.py +127 -0
  34. agent_library-0.7.0/librarian/providers/oai.py +185 -0
  35. agent_library-0.7.0/librarian/retrieval/__init__.py +13 -0
  36. agent_library-0.7.0/librarian/retrieval/search.py +356 -0
  37. agent_library-0.7.0/librarian/server.py +1180 -0
  38. agent_library-0.7.0/librarian/storage/__init__.py +24 -0
  39. agent_library-0.7.0/librarian/storage/database.py +636 -0
  40. agent_library-0.7.0/librarian/storage/fts_store.py +231 -0
  41. agent_library-0.7.0/librarian/storage/vector_store.py +193 -0
  42. agent_library-0.7.0/librarian/types.py +139 -0
  43. agent_library-0.7.0/librarian/utils/__init__.py +17 -0
  44. agent_library-0.7.0/librarian/utils/timeframe.py +170 -0
  45. agent_library-0.7.0/pyproject.toml +90 -0
  46. agent_library-0.7.0/setup.sh +151 -0
  47. agent_library-0.7.0/tests/__init__.py +11 -0
  48. agent_library-0.7.0/tests/conftest.py +211 -0
  49. agent_library-0.7.0/tests/data/biologynotes.md +122 -0
  50. agent_library-0.7.0/tests/data/essays.md +108 -0
  51. agent_library-0.7.0/tests/test_chunker.py +147 -0
  52. agent_library-0.7.0/tests/test_parser.py +131 -0
  53. agent_library-0.7.0/tests/test_tools.py +323 -0
@@ -0,0 +1,14 @@
1
+ # Stop the editor from looking for .editorconfig files in the parent directories
2
+ root = true
3
+
4
+ [*]
5
+ charset = utf-8
6
+ insert_final_newline = true
7
+ end_of_line = lf
8
+ indent_style = space
9
+ indent_size = 4
10
+ max_line_length = 100 # This is also set in .ruff.toml for ruff
11
+
12
+ [*.{json,jsonc,yml,yaml}]
13
+ indent_style = space
14
+ indent_size = 2 # This is also set in .prettierrc.toml
@@ -0,0 +1,20 @@
1
+ # Librarian Configuration
2
+
3
+ # Document paths
4
+ DOCUMENTS_PATH=./documents
5
+ DATABASE_PATH=~/.librarian/index.db
6
+
7
+ # Embedding model (sentence-transformers)
8
+ EMBEDDING_MODEL=all-MiniLM-L6-v2
9
+
10
+ # Chunking settings
11
+ CHUNK_SIZE=512
12
+ CHUNK_OVERLAP=50
13
+
14
+ # Search settings
15
+ SEARCH_LIMIT=10
16
+ MMR_LAMBDA=0.5
17
+ HYBRID_ALPHA=0.7
18
+
19
+ # For running evaluations
20
+ OPENAI_API_KEY=your-openai-api-key
@@ -0,0 +1,17 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "pip"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "weekly"
7
+ open-pull-requests-limit: 5
8
+ labels:
9
+ - "dependencies"
10
+
11
+ - package-ecosystem: "github-actions"
12
+ directory: "/"
13
+ schedule:
14
+ interval: "weekly"
15
+ labels:
16
+ - "ci"
17
+
@@ -0,0 +1,98 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main, master]
6
+ pull_request:
7
+ branches: [main, master]
8
+
9
+ jobs:
10
+ check:
11
+ name: Code Quality
12
+ runs-on: ubuntu-latest
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+
16
+ - name: Install uv
17
+ uses: astral-sh/setup-uv@v4
18
+ with:
19
+ version: "latest"
20
+
21
+ - name: Set up Python
22
+ uses: actions/setup-python@v5
23
+ with:
24
+ python-version: "3.11"
25
+
26
+ - name: Install dependencies
27
+ run: uv sync --extra dev
28
+
29
+ - name: Run linting
30
+ run: uv run ruff check librarian tests
31
+
32
+ - name: Check formatting
33
+ run: uv run ruff format --check librarian tests
34
+
35
+ - name: Run type checking
36
+ run: uv run mypy librarian
37
+
38
+ test:
39
+ name: Tests
40
+ runs-on: ubuntu-latest
41
+ strategy:
42
+ matrix:
43
+ python-version: ["3.10", "3.11", "3.12"]
44
+ steps:
45
+ - uses: actions/checkout@v4
46
+
47
+ - name: Install uv
48
+ uses: astral-sh/setup-uv@v4
49
+ with:
50
+ version: "latest"
51
+
52
+ - name: Set up Python ${{ matrix.python-version }}
53
+ uses: actions/setup-python@v5
54
+ with:
55
+ python-version: ${{ matrix.python-version }}
56
+
57
+ - name: Install dependencies
58
+ run: uv sync --extra dev
59
+
60
+ - name: Run tests
61
+ run: uv run pytest -v --cov --cov-config=pyproject.toml --cov-report=xml
62
+ env:
63
+ EMBEDDING_PROVIDER: local
64
+ EMBEDDING_DIMENSION: "384"
65
+
66
+ - name: Upload coverage to Codecov
67
+ uses: codecov/codecov-action@v4
68
+ if: matrix.python-version == '3.11'
69
+ with:
70
+ files: ./coverage.xml
71
+ fail_ci_if_error: false
72
+
73
+ build:
74
+ name: Build Package
75
+ runs-on: ubuntu-latest
76
+ needs: [check, test]
77
+ steps:
78
+ - uses: actions/checkout@v4
79
+
80
+ - name: Install uv
81
+ uses: astral-sh/setup-uv@v4
82
+ with:
83
+ version: "latest"
84
+
85
+ - name: Set up Python
86
+ uses: actions/setup-python@v5
87
+ with:
88
+ python-version: "3.11"
89
+
90
+ - name: Build package
91
+ run: uv build
92
+
93
+ - name: Upload artifacts
94
+ uses: actions/upload-artifact@v4
95
+ with:
96
+ name: dist
97
+ path: dist/
98
+
@@ -0,0 +1,124 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*.*.*"
7
+ workflow_dispatch:
8
+ inputs:
9
+ version:
10
+ description: "Version to release (e.g., 0.5.0)"
11
+ required: true
12
+ type: string
13
+
14
+ permissions:
15
+ contents: write
16
+ id-token: write
17
+
18
+ jobs:
19
+ build:
20
+ name: Build Package
21
+ runs-on: ubuntu-latest
22
+ steps:
23
+ - uses: actions/checkout@v4
24
+ with:
25
+ fetch-depth: 0
26
+
27
+ - name: Install uv
28
+ uses: astral-sh/setup-uv@v4
29
+ with:
30
+ version: "latest"
31
+
32
+ - name: Set up Python
33
+ uses: actions/setup-python@v5
34
+ with:
35
+ python-version: "3.11"
36
+
37
+ - name: Install dependencies
38
+ run: uv sync --extra dev
39
+
40
+ - name: Run checks
41
+ run: |
42
+ uv run ruff check librarian tests
43
+ uv run ruff format --check librarian tests
44
+ uv run mypy librarian
45
+
46
+ - name: Run tests
47
+ run: uv run pytest -v
48
+ env:
49
+ EMBEDDING_PROVIDER: local
50
+ EMBEDDING_DIMENSION: "384"
51
+
52
+ - name: Build package
53
+ run: uv build
54
+
55
+ - name: Upload artifacts
56
+ uses: actions/upload-artifact@v4
57
+ with:
58
+ name: dist
59
+ path: dist/
60
+
61
+ publish-pypi:
62
+ name: Publish to PyPI
63
+ runs-on: ubuntu-latest
64
+ needs: build
65
+ environment:
66
+ name: pypi
67
+ url: https://pypi.org/project/agent-library/
68
+ steps:
69
+ - name: Download artifacts
70
+ uses: actions/download-artifact@v4
71
+ with:
72
+ name: dist
73
+ path: dist/
74
+
75
+ - name: Publish to PyPI
76
+ uses: pypa/gh-action-pypi-publish@release/v1
77
+ with:
78
+ skip-existing: true
79
+
80
+ create-release:
81
+ name: Create GitHub Release
82
+ runs-on: ubuntu-latest
83
+ needs: build
84
+ if: startsWith(github.ref, 'refs/tags/')
85
+ steps:
86
+ - uses: actions/checkout@v4
87
+
88
+ - name: Download artifacts
89
+ uses: actions/download-artifact@v4
90
+ with:
91
+ name: dist
92
+ path: dist/
93
+
94
+ - name: Get version from tag
95
+ id: get_version
96
+ run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
97
+
98
+ - name: Create Release
99
+ uses: softprops/action-gh-release@v1
100
+ with:
101
+ name: Release v${{ steps.get_version.outputs.VERSION }}
102
+ body: |
103
+ ## Changes in v${{ steps.get_version.outputs.VERSION }}
104
+
105
+ See [CHANGELOG.md](CHANGELOG.md) for details.
106
+
107
+ ## Installation
108
+
109
+ ```bash
110
+ pip install agent-library==${{ steps.get_version.outputs.VERSION }}
111
+ ```
112
+
113
+ Or from source:
114
+
115
+ ```bash
116
+ pip install git+https://github.com/arcadeai-labs/librarian.git@v${{ steps.get_version.outputs.VERSION }}
117
+ ```
118
+ files: |
119
+ dist/*
120
+ draft: false
121
+ prerelease: false
122
+ env:
123
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
124
+
@@ -0,0 +1,166 @@
1
+ .DS_Store
2
+
3
+ *.lock
4
+
5
+ # From https://raw.githubusercontent.com/github/gitignore/main/Python.gitignore
6
+
7
+ # Byte-compiled / optimized / DLL files
8
+ __pycache__/
9
+ *.py[cod]
10
+ *$py.class
11
+
12
+ # C extensions
13
+ *.so
14
+
15
+ # Distribution / packaging
16
+ .Python
17
+ build/
18
+ develop-eggs/
19
+ dist/
20
+ downloads/
21
+ eggs/
22
+ .eggs/
23
+ lib/
24
+ lib64/
25
+ parts/
26
+ sdist/
27
+ var/
28
+ wheels/
29
+ share/python-wheels/
30
+ *.egg-info/
31
+ .installed.cfg
32
+ *.egg
33
+ MANIFEST
34
+
35
+ # PyInstaller
36
+ # Usually these files are written by a python script from a template
37
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
38
+ *.manifest
39
+ *.spec
40
+
41
+ # Installer logs
42
+ pip-log.txt
43
+ pip-delete-this-directory.txt
44
+
45
+ # Unit test / coverage reports
46
+ htmlcov/
47
+ .tox/
48
+ .nox/
49
+ .coverage
50
+ .coverage.*
51
+ .cache
52
+ nosetests.xml
53
+ coverage.xml
54
+ *.cover
55
+ *.py,cover
56
+ .hypothesis/
57
+ .pytest_cache/
58
+ cover/
59
+
60
+ # Translations
61
+ *.mo
62
+ *.pot
63
+
64
+ # Django stuff:
65
+ *.log
66
+ local_settings.py
67
+ db.sqlite3
68
+ db.sqlite3-journal
69
+
70
+ # Flask stuff:
71
+ instance/
72
+ .webassets-cache
73
+
74
+ # Scrapy stuff:
75
+ .scrapy
76
+
77
+ # Sphinx documentation
78
+ docs/_build/
79
+
80
+ # PyBuilder
81
+ .pybuilder/
82
+ target/
83
+
84
+ # Jupyter Notebook
85
+ .ipynb_checkpoints
86
+
87
+ # IPython
88
+ profile_default/
89
+ ipython_config.py
90
+
91
+ # pyenv
92
+ # For a library or package, you might want to ignore these files since the code is
93
+ # intended to run in multiple environments; otherwise, check them in:
94
+ # .python-version
95
+
96
+ # pipenv
97
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
98
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
99
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
100
+ # install all needed dependencies.
101
+ #Pipfile.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ poetry.lock
109
+
110
+ # pdm
111
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
112
+ #pdm.lock
113
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
114
+ # in version control.
115
+ # https://pdm.fming.dev/#use-with-ide
116
+ .pdm.toml
117
+
118
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
119
+ __pypackages__/
120
+
121
+ # Celery stuff
122
+ celerybeat-schedule
123
+ celerybeat.pid
124
+
125
+ # SageMath parsed files
126
+ *.sage.py
127
+
128
+ # Environments
129
+ .env
130
+ .venv
131
+ env/
132
+ venv/
133
+ ENV/
134
+ env.bak/
135
+ venv.bak/
136
+
137
+ # Spyder project settings
138
+ .spyderproject
139
+ .spyproject
140
+
141
+ # Rope project settings
142
+ .ropeproject
143
+
144
+ # mkdocs documentation
145
+ /site
146
+
147
+ # mypy
148
+ .mypy_cache/
149
+ .dmypy.json
150
+ dmypy.json
151
+
152
+ # Pyre type checker
153
+ .pyre/
154
+
155
+ # pytype static type analyzer
156
+ .pytype/
157
+
158
+ # Cython debug symbols
159
+ cython_debug/
160
+
161
+ # PyCharm
162
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
163
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
164
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
165
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
166
+ #.idea/
@@ -0,0 +1,15 @@
1
+ # See https://prettier.io/docs/en/configuration
2
+
3
+ # Note: This prettier config is only for the non-python files in this repo.
4
+ # Python files are formatted with ruff and ignored in .prettierignore
5
+
6
+ trailingComma = "es5"
7
+ tabWidth = 4
8
+ semi = false
9
+ singleQuote = false
10
+
11
+ [[overrides]]
12
+ files = [ "*.json", "*.jsonc", "*.yml", "*.yaml" ]
13
+
14
+ [overrides.options]
15
+ tabWidth = 2
@@ -0,0 +1,51 @@
1
+ target-version = "py310"
2
+ line-length = 100
3
+ fix = true
4
+
5
+ [lint]
6
+ select = [
7
+ # flake8-2020
8
+ "YTT",
9
+ # flake8-bandit
10
+ "S",
11
+ # flake8-bugbear
12
+ "B",
13
+ # flake8-builtins
14
+ "A",
15
+ # flake8-comprehensions
16
+ "C4",
17
+ # flake8-debugger
18
+ "T10",
19
+ # flake8-simplify
20
+ "SIM",
21
+ # isort
22
+ "I",
23
+ # pycodestyle
24
+ "E", "W",
25
+ # pyflakes
26
+ "F",
27
+ # pygrep-hooks
28
+ "PGH",
29
+ # pyupgrade
30
+ "UP",
31
+ # ruff
32
+ "RUF",
33
+ # tryceratops
34
+ "TRY",
35
+ ]
36
+
37
+ ignore = [
38
+ "E501", # Line too long (handled by formatter)
39
+ "TRY003", # Avoid specifying messages outside exception class
40
+ "PGH003", # Blanket type ignores (we use them for decorated functions)
41
+ "UP007", # Use X | Y instead of Union[X, Y] - disabled for typer CLI compatibility
42
+ "UP045", # Use X | None instead of Optional[X] - disabled for typer CLI compatibility
43
+ ]
44
+
45
+ [lint.per-file-ignores]
46
+ "**/tests/*" = ["S101"]
47
+ "librarian/cli.py" = ["S603", "S607"] # CLI uses controlled subprocess calls
48
+
49
+ [format]
50
+ preview = true
51
+ skip-magic-trailing-comma = false
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2024, spartee
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,83 @@
1
+ .PHONY: help
2
+ help: ## Show this help message
3
+ @echo "Librarian Development Commands:\n"
4
+ @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
5
+
6
+ .PHONY: setup
7
+ setup: ## Run the setup script to install uv and create environment
8
+ @./setup.sh
9
+
10
+ .PHONY: install
11
+ install: ## Install the package in development mode
12
+ @if ! command -v uv &> /dev/null; then \
13
+ echo "Installing uv"; \
14
+ curl -LsSf https://astral.sh/uv/install.sh | sh; \
15
+ fi
16
+ @uv pip install -e ".[dev]"
17
+
18
+ .PHONY: sync
19
+ sync: ## Sync dependencies from pyproject.toml
20
+ @uv pip install -e ".[dev]"
21
+
22
+ .PHONY: build
23
+ build: clean-build ## Build wheel file
24
+ @uv build
25
+
26
+ .PHONY: clean-build
27
+ clean-build: ## Clean build artifacts
28
+ @rm -rf dist build *.egg-info
29
+
30
+ .PHONY: clean
31
+ clean: clean-build ## Clean all generated files
32
+ @rm -rf .pytest_cache .mypy_cache .coverage htmlcov .ruff_cache
33
+ @find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
34
+ @find . -type f -name "*.pyc" -delete 2>/dev/null || true
35
+
36
+ .PHONY: test
37
+ test: ## Run tests with pytest
38
+ @uv run pytest -W ignore -v --cov --cov-config=pyproject.toml --cov-report=xml
39
+
40
+ .PHONY: test-fast
41
+ test-fast: ## Run tests without coverage
42
+ @uv run pytest -W ignore -v
43
+
44
+ .PHONY: coverage
45
+ coverage: ## Generate coverage report
46
+ @uv run coverage report
47
+ @uv run coverage html
48
+
49
+ .PHONY: lint
50
+ lint: ## Run linting with ruff
51
+ @uv run ruff check librarian tests
52
+
53
+ .PHONY: lint-fix
54
+ lint-fix: ## Run linting with auto-fix
55
+ @uv run ruff check --fix librarian tests
56
+
57
+ .PHONY: format
58
+ format: ## Format code with ruff
59
+ @uv run ruff format librarian tests
60
+
61
+ .PHONY: format-check
62
+ format-check: ## Check code formatting
63
+ @uv run ruff format --check librarian tests
64
+
65
+ .PHONY: typecheck
66
+ typecheck: ## Run type checking with mypy
67
+ @uv run mypy librarian
68
+
69
+ .PHONY: check
70
+ check: lint format-check typecheck ## Run all code quality checks
71
+
72
+ .PHONY: pre-commit
73
+ pre-commit: ## Run pre-commit hooks on all files
74
+ @uv run pre-commit run -a
75
+
76
+ .PHONY: pre-commit-install
77
+ pre-commit-install: ## Install pre-commit hooks
78
+ @uv run pre-commit install
79
+
80
+ .PHONY: evals
81
+ evals: ## Run Arcade tool evaluations
82
+ @uv pip install -e ".[evals]"
83
+ @uv run arcade evals . -p openai