qdrant-loader 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qdrant_loader-0.1.1/.cursor/rules/project-specifications.mdc +12 -0
- qdrant_loader-0.1.1/.env.template +23 -0
- qdrant_loader-0.1.1/.github/workflows/publish.yml +30 -0
- qdrant_loader-0.1.1/.github/workflows/test.yml +75 -0
- qdrant_loader-0.1.1/.gitignore +52 -0
- qdrant_loader-0.1.1/LICENSE +674 -0
- qdrant_loader-0.1.1/PKG-INFO +385 -0
- qdrant_loader-0.1.1/README.md +369 -0
- qdrant_loader-0.1.1/config.template.yaml +134 -0
- qdrant_loader-0.1.1/docs/CodingStandards.md +231 -0
- qdrant_loader-0.1.1/docs/PRD.md +122 -0
- qdrant_loader-0.1.1/docs/Plan.md +251 -0
- qdrant_loader-0.1.1/pyproject.toml +79 -0
- qdrant_loader-0.1.1/requirements-test.txt +13 -0
- qdrant_loader-0.1.1/requirements.txt +21 -0
- qdrant_loader-0.1.1/src/__init__.py +3 -0
- qdrant_loader-0.1.1/src/main.py +8 -0
- qdrant_loader-0.1.1/src/qdrant_loader/__init__.py +16 -0
- qdrant_loader-0.1.1/src/qdrant_loader/chunking_service.py +61 -0
- qdrant_loader-0.1.1/src/qdrant_loader/cli.py +166 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/__init__.py +247 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/base.py +78 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/chunking.py +27 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/embedding.py +16 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/global_.py +54 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/settings.py +46 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/sources.py +72 -0
- qdrant_loader-0.1.1/src/qdrant_loader/config/types.py +65 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/__init__.py +1 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/confluence/__init__.py +268 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/confluence/config.py +49 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/__init__.py +6 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/config.py +74 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/connector.py +545 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/metadata_extractor.py +381 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/__init__.py +6 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/config.py +125 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/jira_connector.py +259 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/models.py +53 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/public_docs/__init__.py +5 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/public_docs/config.py +56 -0
- qdrant_loader-0.1.1/src/qdrant_loader/connectors/public_docs/connector.py +147 -0
- qdrant_loader-0.1.1/src/qdrant_loader/core/__init__.py +8 -0
- qdrant_loader-0.1.1/src/qdrant_loader/core/chunking.py +139 -0
- qdrant_loader-0.1.1/src/qdrant_loader/core/document.py +70 -0
- qdrant_loader-0.1.1/src/qdrant_loader/embedding_service.py +71 -0
- qdrant_loader-0.1.1/src/qdrant_loader/ingestion_pipeline.py +290 -0
- qdrant_loader-0.1.1/src/qdrant_loader/init_collection.py +26 -0
- qdrant_loader-0.1.1/src/qdrant_loader/qdrant_manager.py +141 -0
- qdrant_loader-0.1.1/src/qdrant_loader/utils/__init__.py +0 -0
- qdrant_loader-0.1.1/src/qdrant_loader/utils/logger.py +58 -0
- qdrant_loader-0.1.1/tests/.env.test.template +26 -0
- qdrant_loader-0.1.1/tests/__init__.py +3 -0
- qdrant_loader-0.1.1/tests/config.test.template.yaml +60 -0
- qdrant_loader-0.1.1/tests/conftest.py +69 -0
- qdrant_loader-0.1.1/tests/integration/conftest.py +50 -0
- qdrant_loader-0.1.1/tests/integration/connectors/confluence/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/integration/connectors/confluence/test_confluence_integration.py +249 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/conftest.py +47 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_authentication.py +209 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_cleanup.py +96 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_directory_structure.py +219 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_file_access.py +73 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_file_sizes.py +80 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_file_types.py +70 -0
- qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_integration.py +159 -0
- qdrant_loader-0.1.1/tests/integration/connectors/jira/test_jira_integration.py +111 -0
- qdrant_loader-0.1.1/tests/integration/core/test_openai_integration.py +125 -0
- qdrant_loader-0.1.1/tests/integration/core/test_qdrant_integration.py +136 -0
- qdrant_loader-0.1.1/tests/unit/cli/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/unit/cli/test_cli.py +442 -0
- qdrant_loader-0.1.1/tests/unit/config/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/unit/config/test_config.py +134 -0
- qdrant_loader-0.1.1/tests/unit/conftest.py +10 -0
- qdrant_loader-0.1.1/tests/unit/connectors/git/test_git_python_adapter.py +172 -0
- qdrant_loader-0.1.1/tests/unit/connectors/git/test_metadata_extractor.py +174 -0
- qdrant_loader-0.1.1/tests/unit/connectors/jira/test_config.py +91 -0
- qdrant_loader-0.1.1/tests/unit/connectors/jira/test_jira_connector.py +298 -0
- qdrant_loader-0.1.1/tests/unit/connectors/jira/test_models.py +133 -0
- qdrant_loader-0.1.1/tests/unit/connectors/public_docs/test_public_docs_connector.py +127 -0
- qdrant_loader-0.1.1/tests/unit/core/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/unit/core/test_chunking.py +202 -0
- qdrant_loader-0.1.1/tests/unit/core/test_document.py +57 -0
- qdrant_loader-0.1.1/tests/unit/management/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/unit/management/test_init_collection.py +59 -0
- qdrant_loader-0.1.1/tests/unit/services/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/unit/services/test_chunking_service.py +110 -0
- qdrant_loader-0.1.1/tests/unit/services/test_ingestion_pipeline.py +423 -0
- qdrant_loader-0.1.1/tests/unit/utils/__init__.py +0 -0
- qdrant_loader-0.1.1/tests/unit/utils/test_logger.py +76 -0
- qdrant_loader-0.1.1/tests/utils.py +12 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
---
|
|
2
|
+
description:
|
|
3
|
+
globs:
|
|
4
|
+
alwaysApply: true
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# You must always read in details all the following documents at the beginning of any new conversation. Confirm that you read everything by starting your answer by "I have read the documentation"
|
|
8
|
+
|
|
9
|
+
- [PRD.md](mdc:docs/PRD.md) will help you understant the goal of the application
|
|
10
|
+
- [Plan.md](mdc:docs/Plan.md) will help you understand the progress we made in our project
|
|
11
|
+
- [CodingStandards.md](mdc:docs/CodingStandards.md) are all the tules you need to stricly follow
|
|
12
|
+
- [README.md](mdc:README.md) for the general documentation of the project
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# OpenAI Configuration
|
|
2
|
+
OPENAI_API_KEY=your_openai_api_key_here
|
|
3
|
+
|
|
4
|
+
# qDrant Configuration
|
|
5
|
+
QDRANT_URL=your_qdrant_url_here
|
|
6
|
+
QDRANT_API_KEY=your_qdrant_api_key_here
|
|
7
|
+
QDRANT_COLLECTION_NAME=your_collection_name_here
|
|
8
|
+
|
|
9
|
+
# Git Authentication Configuration
|
|
10
|
+
# GitHub Personal Access Token
|
|
11
|
+
GITHUB_TOKEN=your_github_token_here
|
|
12
|
+
|
|
13
|
+
# GitLab Personal Access Token
|
|
14
|
+
GITLAB_TOKEN=your_gitlab_token_here
|
|
15
|
+
|
|
16
|
+
# Bitbucket Personal Access Token
|
|
17
|
+
BITBUCKET_TOKEN=your_bitbucket_token_here
|
|
18
|
+
|
|
19
|
+
# Confluence Configuration
|
|
20
|
+
# Confluence Personal Access Token
|
|
21
|
+
# Generate at: https://id.atlassian.com/manage/api-tokens
|
|
22
|
+
CONFLUENCE_TOKEN=your_confluence_token_here
|
|
23
|
+
CONFLUENCE_EMAIL=your_confluence_email_here
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: Upload Python Package to PyPI when a Release is Created
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [created]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
pypi-publish:
|
|
9
|
+
name: Publish release to PyPI
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
environment:
|
|
12
|
+
name: pypi-publish
|
|
13
|
+
url: https://pypi.org/p/qdrant-loader
|
|
14
|
+
permissions:
|
|
15
|
+
id-token: write
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v4
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.12.2"
|
|
22
|
+
- name: Install dependencies
|
|
23
|
+
run: |
|
|
24
|
+
python -m pip install --upgrade pip
|
|
25
|
+
pip install build twine
|
|
26
|
+
- name: Build package
|
|
27
|
+
run: |
|
|
28
|
+
python -m build
|
|
29
|
+
- name: Publish package distributions to PyPI
|
|
30
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
name: Test and Coverage
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ main ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ main ]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
pages: write
|
|
12
|
+
id-token: write
|
|
13
|
+
actions: read
|
|
14
|
+
|
|
15
|
+
concurrency:
|
|
16
|
+
group: "test-and-coverage"
|
|
17
|
+
cancel-in-progress: false
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
test:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- name: Set up Python
|
|
26
|
+
uses: actions/setup-python@v5
|
|
27
|
+
with:
|
|
28
|
+
python-version: '3.13.2'
|
|
29
|
+
|
|
30
|
+
- name: Install dependencies
|
|
31
|
+
run: |
|
|
32
|
+
python -m pip install --upgrade pip
|
|
33
|
+
pip install -r requirements.txt -r requirements-test.txt
|
|
34
|
+
pip install -e .
|
|
35
|
+
|
|
36
|
+
- name: Create .env.test file
|
|
37
|
+
run: |
|
|
38
|
+
cp tests/.env.test.template tests/.env.test
|
|
39
|
+
sed -i "s|QDRANT_URL=.*|QDRANT_URL=${{ secrets.QDRANT_URL }}|g" tests/.env.test
|
|
40
|
+
sed -i "s|QDRANT_API_KEY=.*|QDRANT_API_KEY=${{ secrets.QDRANT_API_KEY }}|g" tests/.env.test
|
|
41
|
+
sed -i "s|QDRANT_COLLECTION_NAME=.*|QDRANT_COLLECTION_NAME=${{ secrets.QDRANT_COLLECTION_NAME }}|g" tests/.env.test
|
|
42
|
+
sed -i "s|OPENAI_API_KEY=.*|OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}|g" tests/.env.test
|
|
43
|
+
sed -i "s|REPO_TOKEN=.*|REPO_TOKEN=${{ secrets.REPO_TOKEN }}|g" tests/.env.test
|
|
44
|
+
sed -i "s|REPO_URL=.*|REPO_URL=${{ secrets.REPO_URL }}|g" tests/.env.test
|
|
45
|
+
sed -i "s|CONFLUENCE_TOKEN=.*|CONFLUENCE_TOKEN=${{ secrets.CONFLUENCE_TOKEN }}|g" tests/.env.test
|
|
46
|
+
sed -i "s|CONFLUENCE_EMAIL=.*|CONFLUENCE_EMAIL=${{ secrets.CONFLUENCE_EMAIL }}|g" tests/.env.test
|
|
47
|
+
sed -i "s|CONFLUENCE_URL=.*|CONFLUENCE_URL=${{ secrets.CONFLUENCE_URL }}|g" tests/.env.test
|
|
48
|
+
sed -i "s|CONFLUENCE_SPACE_KEY=.*|CONFLUENCE_SPACE_KEY=${{ secrets.CONFLUENCE_SPACE_KEY }}|g" tests/.env.test
|
|
49
|
+
sed -i "s|JIRA_TOKEN=.*|JIRA_TOKEN=${{ secrets.JIRA_TOKEN }}|g" tests/.env.test
|
|
50
|
+
sed -i "s|JIRA_EMAIL=.*|JIRA_EMAIL=${{ secrets.JIRA_EMAIL }}|g" tests/.env.test
|
|
51
|
+
sed -i "s|JIRA_URL=.*|JIRA_URL=${{ secrets.JIRA_URL }}|g" tests/.env.test
|
|
52
|
+
sed -i "s|JIRA_PROJECT_KEY=.*|JIRA_PROJECT_KEY=${{ secrets.JIRA_PROJECT_KEY }}|g" tests/.env.test
|
|
53
|
+
|
|
54
|
+
- name: Create config.test.yaml file
|
|
55
|
+
run: |
|
|
56
|
+
cp tests/config.test.template.yaml tests/config.test.yaml
|
|
57
|
+
|
|
58
|
+
- name: Run tests and generate coverage reports
|
|
59
|
+
run: |
|
|
60
|
+
python -m pytest tests/ --cov=qdrant_loader --cov-report=html -v
|
|
61
|
+
|
|
62
|
+
- name: Setup Pages
|
|
63
|
+
uses: actions/configure-pages@v4
|
|
64
|
+
|
|
65
|
+
- name: Upload coverage artifact
|
|
66
|
+
uses: actions/upload-pages-artifact@v3
|
|
67
|
+
with:
|
|
68
|
+
path: htmlcov
|
|
69
|
+
|
|
70
|
+
deploy:
|
|
71
|
+
runs-on: ubuntu-latest
|
|
72
|
+
needs: test
|
|
73
|
+
steps:
|
|
74
|
+
- name: Deploy to GitHub Pages
|
|
75
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
develop-eggs/
|
|
9
|
+
dist/
|
|
10
|
+
downloads/
|
|
11
|
+
eggs/
|
|
12
|
+
.eggs/
|
|
13
|
+
lib/
|
|
14
|
+
lib64/
|
|
15
|
+
parts/
|
|
16
|
+
sdist/
|
|
17
|
+
var/
|
|
18
|
+
wheels/
|
|
19
|
+
*.egg-info/
|
|
20
|
+
.installed.cfg
|
|
21
|
+
*.egg
|
|
22
|
+
|
|
23
|
+
# Virtual Environment
|
|
24
|
+
venv/
|
|
25
|
+
env/
|
|
26
|
+
ENV/
|
|
27
|
+
|
|
28
|
+
# IDE
|
|
29
|
+
.idea/
|
|
30
|
+
.vscode/
|
|
31
|
+
*.swp
|
|
32
|
+
*.swo
|
|
33
|
+
|
|
34
|
+
# Environment variables
|
|
35
|
+
.env
|
|
36
|
+
.env.local
|
|
37
|
+
.env.test
|
|
38
|
+
|
|
39
|
+
# Logs
|
|
40
|
+
*.log
|
|
41
|
+
logs/
|
|
42
|
+
|
|
43
|
+
# Test coverage
|
|
44
|
+
.coverage
|
|
45
|
+
htmlcov/
|
|
46
|
+
.pytest_cache/
|
|
47
|
+
|
|
48
|
+
# Distribution
|
|
49
|
+
dist/
|
|
50
|
+
build/
|
|
51
|
+
config.yaml
|
|
52
|
+
tests/config.test.yaml
|