qdrant-loader 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. qdrant_loader-0.1.1/.cursor/rules/project-specifications.mdc +12 -0
  2. qdrant_loader-0.1.1/.env.template +23 -0
  3. qdrant_loader-0.1.1/.github/workflows/publish.yml +30 -0
  4. qdrant_loader-0.1.1/.github/workflows/test.yml +75 -0
  5. qdrant_loader-0.1.1/.gitignore +52 -0
  6. qdrant_loader-0.1.1/LICENSE +674 -0
  7. qdrant_loader-0.1.1/PKG-INFO +385 -0
  8. qdrant_loader-0.1.1/README.md +369 -0
  9. qdrant_loader-0.1.1/config.template.yaml +134 -0
  10. qdrant_loader-0.1.1/docs/CodingStandards.md +231 -0
  11. qdrant_loader-0.1.1/docs/PRD.md +122 -0
  12. qdrant_loader-0.1.1/docs/Plan.md +251 -0
  13. qdrant_loader-0.1.1/pyproject.toml +79 -0
  14. qdrant_loader-0.1.1/requirements-test.txt +13 -0
  15. qdrant_loader-0.1.1/requirements.txt +21 -0
  16. qdrant_loader-0.1.1/src/__init__.py +3 -0
  17. qdrant_loader-0.1.1/src/main.py +8 -0
  18. qdrant_loader-0.1.1/src/qdrant_loader/__init__.py +16 -0
  19. qdrant_loader-0.1.1/src/qdrant_loader/chunking_service.py +61 -0
  20. qdrant_loader-0.1.1/src/qdrant_loader/cli.py +166 -0
  21. qdrant_loader-0.1.1/src/qdrant_loader/config/__init__.py +247 -0
  22. qdrant_loader-0.1.1/src/qdrant_loader/config/base.py +78 -0
  23. qdrant_loader-0.1.1/src/qdrant_loader/config/chunking.py +27 -0
  24. qdrant_loader-0.1.1/src/qdrant_loader/config/embedding.py +16 -0
  25. qdrant_loader-0.1.1/src/qdrant_loader/config/global_.py +54 -0
  26. qdrant_loader-0.1.1/src/qdrant_loader/config/settings.py +46 -0
  27. qdrant_loader-0.1.1/src/qdrant_loader/config/sources.py +72 -0
  28. qdrant_loader-0.1.1/src/qdrant_loader/config/types.py +65 -0
  29. qdrant_loader-0.1.1/src/qdrant_loader/connectors/__init__.py +1 -0
  30. qdrant_loader-0.1.1/src/qdrant_loader/connectors/confluence/__init__.py +268 -0
  31. qdrant_loader-0.1.1/src/qdrant_loader/connectors/confluence/config.py +49 -0
  32. qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/__init__.py +6 -0
  33. qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/config.py +74 -0
  34. qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/connector.py +545 -0
  35. qdrant_loader-0.1.1/src/qdrant_loader/connectors/git/metadata_extractor.py +381 -0
  36. qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/__init__.py +6 -0
  37. qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/config.py +125 -0
  38. qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/jira_connector.py +259 -0
  39. qdrant_loader-0.1.1/src/qdrant_loader/connectors/jira/models.py +53 -0
  40. qdrant_loader-0.1.1/src/qdrant_loader/connectors/public_docs/__init__.py +5 -0
  41. qdrant_loader-0.1.1/src/qdrant_loader/connectors/public_docs/config.py +56 -0
  42. qdrant_loader-0.1.1/src/qdrant_loader/connectors/public_docs/connector.py +147 -0
  43. qdrant_loader-0.1.1/src/qdrant_loader/core/__init__.py +8 -0
  44. qdrant_loader-0.1.1/src/qdrant_loader/core/chunking.py +139 -0
  45. qdrant_loader-0.1.1/src/qdrant_loader/core/document.py +70 -0
  46. qdrant_loader-0.1.1/src/qdrant_loader/embedding_service.py +71 -0
  47. qdrant_loader-0.1.1/src/qdrant_loader/ingestion_pipeline.py +290 -0
  48. qdrant_loader-0.1.1/src/qdrant_loader/init_collection.py +26 -0
  49. qdrant_loader-0.1.1/src/qdrant_loader/qdrant_manager.py +141 -0
  50. qdrant_loader-0.1.1/src/qdrant_loader/utils/__init__.py +0 -0
  51. qdrant_loader-0.1.1/src/qdrant_loader/utils/logger.py +58 -0
  52. qdrant_loader-0.1.1/tests/.env.test.template +26 -0
  53. qdrant_loader-0.1.1/tests/__init__.py +3 -0
  54. qdrant_loader-0.1.1/tests/config.test.template.yaml +60 -0
  55. qdrant_loader-0.1.1/tests/conftest.py +69 -0
  56. qdrant_loader-0.1.1/tests/integration/conftest.py +50 -0
  57. qdrant_loader-0.1.1/tests/integration/connectors/confluence/__init__.py +0 -0
  58. qdrant_loader-0.1.1/tests/integration/connectors/confluence/test_confluence_integration.py +249 -0
  59. qdrant_loader-0.1.1/tests/integration/connectors/github/__init__.py +0 -0
  60. qdrant_loader-0.1.1/tests/integration/connectors/github/conftest.py +47 -0
  61. qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_authentication.py +209 -0
  62. qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_cleanup.py +96 -0
  63. qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_directory_structure.py +219 -0
  64. qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_file_access.py +73 -0
  65. qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_file_sizes.py +80 -0
  66. qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_file_types.py +70 -0
  67. qdrant_loader-0.1.1/tests/integration/connectors/github/test_git_integration.py +159 -0
  68. qdrant_loader-0.1.1/tests/integration/connectors/jira/test_jira_integration.py +111 -0
  69. qdrant_loader-0.1.1/tests/integration/core/test_openai_integration.py +125 -0
  70. qdrant_loader-0.1.1/tests/integration/core/test_qdrant_integration.py +136 -0
  71. qdrant_loader-0.1.1/tests/unit/cli/__init__.py +0 -0
  72. qdrant_loader-0.1.1/tests/unit/cli/test_cli.py +442 -0
  73. qdrant_loader-0.1.1/tests/unit/config/__init__.py +0 -0
  74. qdrant_loader-0.1.1/tests/unit/config/test_config.py +134 -0
  75. qdrant_loader-0.1.1/tests/unit/conftest.py +10 -0
  76. qdrant_loader-0.1.1/tests/unit/connectors/git/test_git_python_adapter.py +172 -0
  77. qdrant_loader-0.1.1/tests/unit/connectors/git/test_metadata_extractor.py +174 -0
  78. qdrant_loader-0.1.1/tests/unit/connectors/jira/test_config.py +91 -0
  79. qdrant_loader-0.1.1/tests/unit/connectors/jira/test_jira_connector.py +298 -0
  80. qdrant_loader-0.1.1/tests/unit/connectors/jira/test_models.py +133 -0
  81. qdrant_loader-0.1.1/tests/unit/connectors/public_docs/test_public_docs_connector.py +127 -0
  82. qdrant_loader-0.1.1/tests/unit/core/__init__.py +0 -0
  83. qdrant_loader-0.1.1/tests/unit/core/test_chunking.py +202 -0
  84. qdrant_loader-0.1.1/tests/unit/core/test_document.py +57 -0
  85. qdrant_loader-0.1.1/tests/unit/management/__init__.py +0 -0
  86. qdrant_loader-0.1.1/tests/unit/management/test_init_collection.py +59 -0
  87. qdrant_loader-0.1.1/tests/unit/services/__init__.py +0 -0
  88. qdrant_loader-0.1.1/tests/unit/services/test_chunking_service.py +110 -0
  89. qdrant_loader-0.1.1/tests/unit/services/test_ingestion_pipeline.py +423 -0
  90. qdrant_loader-0.1.1/tests/unit/utils/__init__.py +0 -0
  91. qdrant_loader-0.1.1/tests/unit/utils/test_logger.py +76 -0
  92. qdrant_loader-0.1.1/tests/utils.py +12 -0
@@ -0,0 +1,12 @@
1
+ ---
2
+ description:
3
+ globs:
4
+ alwaysApply: true
5
+ ---
6
+
7
+ # You must always read in details all the following documents at the beginning of any new conversation. Confirm that you read everything by starting your answer by "I have read the documentation"
8
+
9
+ - [PRD.md](mdc:docs/PRD.md) will help you understant the goal of the application
10
+ - [Plan.md](mdc:docs/Plan.md) will help you understand the progress we made in our project
11
+ - [CodingStandards.md](mdc:docs/CodingStandards.md) are all the tules you need to stricly follow
12
+ - [README.md](mdc:README.md) for the general documentation of the project
@@ -0,0 +1,23 @@
1
+ # OpenAI Configuration
2
+ OPENAI_API_KEY=your_openai_api_key_here
3
+
4
+ # qDrant Configuration
5
+ QDRANT_URL=your_qdrant_url_here
6
+ QDRANT_API_KEY=your_qdrant_api_key_here
7
+ QDRANT_COLLECTION_NAME=your_collection_name_here
8
+
9
+ # Git Authentication Configuration
10
+ # GitHub Personal Access Token
11
+ GITHUB_TOKEN=your_github_token_here
12
+
13
+ # GitLab Personal Access Token
14
+ GITLAB_TOKEN=your_gitlab_token_here
15
+
16
+ # Bitbucket Personal Access Token
17
+ BITBUCKET_TOKEN=your_bitbucket_token_here
18
+
19
+ # Confluence Configuration
20
+ # Confluence Personal Access Token
21
+ # Generate at: https://id.atlassian.com/manage/api-tokens
22
+ CONFLUENCE_TOKEN=your_confluence_token_here
23
+ CONFLUENCE_EMAIL=your_confluence_email_here
@@ -0,0 +1,30 @@
1
+ name: Upload Python Package to PyPI when a Release is Created
2
+
3
+ on:
4
+ release:
5
+ types: [created]
6
+
7
+ jobs:
8
+ pypi-publish:
9
+ name: Publish release to PyPI
10
+ runs-on: ubuntu-latest
11
+ environment:
12
+ name: pypi-publish
13
+ url: https://pypi.org/p/qdrant-loader
14
+ permissions:
15
+ id-token: write
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - name: Set up Python
19
+ uses: actions/setup-python@v4
20
+ with:
21
+ python-version: "3.12.2"
22
+ - name: Install dependencies
23
+ run: |
24
+ python -m pip install --upgrade pip
25
+ pip install build twine
26
+ - name: Build package
27
+ run: |
28
+ python -m build
29
+ - name: Publish package distributions to PyPI
30
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,75 @@
1
+ name: Test and Coverage
2
+
3
+ on:
4
+ push:
5
+ branches: [ main ]
6
+ pull_request:
7
+ branches: [ main ]
8
+
9
+ permissions:
10
+ contents: read
11
+ pages: write
12
+ id-token: write
13
+ actions: read
14
+
15
+ concurrency:
16
+ group: "test-and-coverage"
17
+ cancel-in-progress: false
18
+
19
+ jobs:
20
+ test:
21
+ runs-on: ubuntu-latest
22
+ steps:
23
+ - uses: actions/checkout@v4
24
+
25
+ - name: Set up Python
26
+ uses: actions/setup-python@v5
27
+ with:
28
+ python-version: '3.13.2'
29
+
30
+ - name: Install dependencies
31
+ run: |
32
+ python -m pip install --upgrade pip
33
+ pip install -r requirements.txt -r requirements-test.txt
34
+ pip install -e .
35
+
36
+ - name: Create .env.test file
37
+ run: |
38
+ cp tests/.env.test.template tests/.env.test
39
+ sed -i "s|QDRANT_URL=.*|QDRANT_URL=${{ secrets.QDRANT_URL }}|g" tests/.env.test
40
+ sed -i "s|QDRANT_API_KEY=.*|QDRANT_API_KEY=${{ secrets.QDRANT_API_KEY }}|g" tests/.env.test
41
+ sed -i "s|QDRANT_COLLECTION_NAME=.*|QDRANT_COLLECTION_NAME=${{ secrets.QDRANT_COLLECTION_NAME }}|g" tests/.env.test
42
+ sed -i "s|OPENAI_API_KEY=.*|OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}|g" tests/.env.test
43
+ sed -i "s|REPO_TOKEN=.*|REPO_TOKEN=${{ secrets.REPO_TOKEN }}|g" tests/.env.test
44
+ sed -i "s|REPO_URL=.*|REPO_URL=${{ secrets.REPO_URL }}|g" tests/.env.test
45
+ sed -i "s|CONFLUENCE_TOKEN=.*|CONFLUENCE_TOKEN=${{ secrets.CONFLUENCE_TOKEN }}|g" tests/.env.test
46
+ sed -i "s|CONFLUENCE_EMAIL=.*|CONFLUENCE_EMAIL=${{ secrets.CONFLUENCE_EMAIL }}|g" tests/.env.test
47
+ sed -i "s|CONFLUENCE_URL=.*|CONFLUENCE_URL=${{ secrets.CONFLUENCE_URL }}|g" tests/.env.test
48
+ sed -i "s|CONFLUENCE_SPACE_KEY=.*|CONFLUENCE_SPACE_KEY=${{ secrets.CONFLUENCE_SPACE_KEY }}|g" tests/.env.test
49
+ sed -i "s|JIRA_TOKEN=.*|JIRA_TOKEN=${{ secrets.JIRA_TOKEN }}|g" tests/.env.test
50
+ sed -i "s|JIRA_EMAIL=.*|JIRA_EMAIL=${{ secrets.JIRA_EMAIL }}|g" tests/.env.test
51
+ sed -i "s|JIRA_URL=.*|JIRA_URL=${{ secrets.JIRA_URL }}|g" tests/.env.test
52
+ sed -i "s|JIRA_PROJECT_KEY=.*|JIRA_PROJECT_KEY=${{ secrets.JIRA_PROJECT_KEY }}|g" tests/.env.test
53
+
54
+ - name: Create config.test.yaml file
55
+ run: |
56
+ cp tests/config.test.template.yaml tests/config.test.yaml
57
+
58
+ - name: Run tests and generate coverage reports
59
+ run: |
60
+ python -m pytest tests/ --cov=qdrant_loader --cov-report=html -v
61
+
62
+ - name: Setup Pages
63
+ uses: actions/configure-pages@v4
64
+
65
+ - name: Upload coverage artifact
66
+ uses: actions/upload-pages-artifact@v3
67
+ with:
68
+ path: htmlcov
69
+
70
+ deploy:
71
+ runs-on: ubuntu-latest
72
+ needs: test
73
+ steps:
74
+ - name: Deploy to GitHub Pages
75
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,52 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ *.egg-info/
20
+ .installed.cfg
21
+ *.egg
22
+
23
+ # Virtual Environment
24
+ venv/
25
+ env/
26
+ ENV/
27
+
28
+ # IDE
29
+ .idea/
30
+ .vscode/
31
+ *.swp
32
+ *.swo
33
+
34
+ # Environment variables
35
+ .env
36
+ .env.local
37
+ .env.test
38
+
39
+ # Logs
40
+ *.log
41
+ logs/
42
+
43
+ # Test coverage
44
+ .coverage
45
+ htmlcov/
46
+ .pytest_cache/
47
+
48
+ # Distribution
49
+ dist/
50
+ build/
51
+ config.yaml
52
+ tests/config.test.yaml