glean-indexing-sdk 0.0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. glean_indexing_sdk-0.0.3/.cz.toml +5 -0
  2. glean_indexing_sdk-0.0.3/.env.template +3 -0
  3. glean_indexing_sdk-0.0.3/.github/CODEOWNERS +42 -0
  4. glean_indexing_sdk-0.0.3/.github/workflows/ci.yml +66 -0
  5. glean_indexing_sdk-0.0.3/.github/workflows/publish.yml +58 -0
  6. glean_indexing_sdk-0.0.3/.gitignore +179 -0
  7. glean_indexing_sdk-0.0.3/.markdown-coderc.json +6 -0
  8. glean_indexing_sdk-0.0.3/.python-version +1 -0
  9. glean_indexing_sdk-0.0.3/.ruff.toml +3 -0
  10. glean_indexing_sdk-0.0.3/.vscode/settings.json +4 -0
  11. glean_indexing_sdk-0.0.3/CHANGELOG.md +5 -0
  12. glean_indexing_sdk-0.0.3/CONTRIBUTING.md +71 -0
  13. glean_indexing_sdk-0.0.3/LICENSE +21 -0
  14. glean_indexing_sdk-0.0.3/PKG-INFO +482 -0
  15. glean_indexing_sdk-0.0.3/README.md +452 -0
  16. glean_indexing_sdk-0.0.3/RELEASE.md +87 -0
  17. glean_indexing_sdk-0.0.3/env.template +45 -0
  18. glean_indexing_sdk-0.0.3/mise.toml +4 -0
  19. glean_indexing_sdk-0.0.3/pyproject.toml +131 -0
  20. glean_indexing_sdk-0.0.3/snippets/non_streaming/complete.py +92 -0
  21. glean_indexing_sdk-0.0.3/snippets/non_streaming/run_connector.py +14 -0
  22. glean_indexing_sdk-0.0.3/snippets/non_streaming/wiki_connector.py +48 -0
  23. glean_indexing_sdk-0.0.3/snippets/non_streaming/wiki_data_client.py +18 -0
  24. glean_indexing_sdk-0.0.3/snippets/non_streaming/wiki_page_data.py +14 -0
  25. glean_indexing_sdk-0.0.3/snippets/streaming/article_connector.py +42 -0
  26. glean_indexing_sdk-0.0.3/snippets/streaming/article_data.py +12 -0
  27. glean_indexing_sdk-0.0.3/snippets/streaming/article_data_client.py +46 -0
  28. glean_indexing_sdk-0.0.3/snippets/streaming/run_connector.py +12 -0
  29. glean_indexing_sdk-0.0.3/src/glean/indexing/__init__.py +56 -0
  30. glean_indexing_sdk-0.0.3/src/glean/indexing/common/__init__.py +15 -0
  31. glean_indexing_sdk-0.0.3/src/glean/indexing/common/batch_processor.py +31 -0
  32. glean_indexing_sdk-0.0.3/src/glean/indexing/common/content_formatter.py +46 -0
  33. glean_indexing_sdk-0.0.3/src/glean/indexing/common/glean_client.py +18 -0
  34. glean_indexing_sdk-0.0.3/src/glean/indexing/common/metrics.py +54 -0
  35. glean_indexing_sdk-0.0.3/src/glean/indexing/common/mocks.py +20 -0
  36. glean_indexing_sdk-0.0.3/src/glean/indexing/connectors/__init__.py +21 -0
  37. glean_indexing_sdk-0.0.3/src/glean/indexing/connectors/base_connector.py +60 -0
  38. glean_indexing_sdk-0.0.3/src/glean/indexing/connectors/base_data_client.py +35 -0
  39. glean_indexing_sdk-0.0.3/src/glean/indexing/connectors/base_datasource_connector.py +314 -0
  40. glean_indexing_sdk-0.0.3/src/glean/indexing/connectors/base_people_connector.py +154 -0
  41. glean_indexing_sdk-0.0.3/src/glean/indexing/connectors/base_streaming_data_client.py +39 -0
  42. glean_indexing_sdk-0.0.3/src/glean/indexing/connectors/base_streaming_datasource_connector.py +184 -0
  43. glean_indexing_sdk-0.0.3/src/glean/indexing/models.py +45 -0
  44. glean_indexing_sdk-0.0.3/src/glean/indexing/observability/__init__.py +19 -0
  45. glean_indexing_sdk-0.0.3/src/glean/indexing/observability/observability.py +262 -0
  46. glean_indexing_sdk-0.0.3/src/glean/indexing/py.typed +1 -0
  47. glean_indexing_sdk-0.0.3/src/glean/indexing/testing/__init__.py +13 -0
  48. glean_indexing_sdk-0.0.3/src/glean/indexing/testing/connector_test_harness.py +53 -0
  49. glean_indexing_sdk-0.0.3/src/glean/indexing/testing/mock_data_source.py +47 -0
  50. glean_indexing_sdk-0.0.3/src/glean/indexing/testing/mock_glean_client.py +69 -0
  51. glean_indexing_sdk-0.0.3/src/glean/indexing/testing/response_validator.py +52 -0
  52. glean_indexing_sdk-0.0.3/taskfile.yml +260 -0
  53. glean_indexing_sdk-0.0.3/tests/__init__.py +1 -0
  54. glean_indexing_sdk-0.0.3/tests/integration_tests/__init__.py +1 -0
  55. glean_indexing_sdk-0.0.3/tests/unit_tests/__init__.py +1 -0
  56. glean_indexing_sdk-0.0.3/tests/unit_tests/test_base_connector.py +32 -0
  57. glean_indexing_sdk-0.0.3/tests/unit_tests/test_base_data_client.py +41 -0
  58. glean_indexing_sdk-0.0.3/tests/unit_tests/test_base_datasource_connector.py +130 -0
  59. glean_indexing_sdk-0.0.3/tests/unit_tests/test_base_people_connector.py +80 -0
  60. glean_indexing_sdk-0.0.3/tests/unit_tests/test_base_streaming_datasource_connector.py +106 -0
  61. glean_indexing_sdk-0.0.3/tests/unit_tests/test_custom_connector_integration.py +84 -0
  62. glean_indexing_sdk-0.0.3/tests/unit_tests/utils/__init__.py +1 -0
  63. glean_indexing_sdk-0.0.3/tests/unit_tests/utils/mock_clients.py +72 -0
  64. glean_indexing_sdk-0.0.3/tests/unit_tests/utils/test_batch_processor.py +39 -0
  65. glean_indexing_sdk-0.0.3/tests/unit_tests/utils/test_content_formatter.py +55 -0
  66. glean_indexing_sdk-0.0.3/tests/unit_tests/utils/test_metrics.py +66 -0
  67. glean_indexing_sdk-0.0.3/uv.lock +1516 -0
@@ -0,0 +1,5 @@
1
+ [tool.commitizen]
2
+ name = "cz_conventional_commits"
3
+ version = "0.0.3"
4
+ tag_format = "v$version"
5
+ version_files = ["pyproject.toml:version", "src/glean/indexing/__init__.py:__version__"]
@@ -0,0 +1,3 @@
1
+ # Environment variables for development
2
+ GLEAN_INSTANCE="acme"
3
+ GLEAN_INDEXING_API_TOKEN="your-indexing-api-token"
@@ -0,0 +1,42 @@
1
+ # =============================================================================
2
+ # Code Owners for glean-indexing-sdk
3
+ # =============================================================================
4
+ #
5
+ # These owners will be the default owners for everything in the repo.
6
+ # Unless a later match takes precedence, these users will be requested
7
+ # for review when someone opens a pull request.
8
+ #
9
+ # See: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
10
+
11
+ # Global owners - fallback for any files not explicitly defined
12
+ * @glean-io/indexing-sdk-maintainers
13
+
14
+ # Core SDK components
15
+ /src/glean/indexing/ @glean-io/indexing-sdk-maintainers
16
+
17
+ # Configuration and build files
18
+ /pyproject.toml @glean-io/indexing-sdk-maintainers
19
+ /taskfile.yml @glean-io/indexing-sdk-maintainers
20
+ /.github/ @glean-io/indexing-sdk-maintainers
21
+ /mise.toml @glean-io/indexing-sdk-maintainers
22
+ /.pre-commit-config.yaml @glean-io/indexing-sdk-maintainers
23
+
24
+ # Documentation
25
+ /README.md @glean-io/indexing-sdk-maintainers @glean-io/docs-team
26
+ /docs/ @glean-io/indexing-sdk-maintainers @glean-io/docs-team
27
+ *.md @glean-io/indexing-sdk-maintainers
28
+
29
+ # Testing infrastructure
30
+ /tests/ @glean-io/indexing-sdk-maintainers
31
+ /.github/workflows/ @glean-io/indexing-sdk-maintainers @glean-io/platform-team
32
+
33
+ # Connector examples and templates
34
+ /src/glean/indexing/connectors/examples/ @glean-io/indexing-sdk-maintainers @glean-io/connector-experts
35
+
36
+ # API client integrations
37
+ /src/glean/indexing/clients/ @glean-io/indexing-sdk-maintainers @glean-io/api-team
38
+
39
+ # Release and package management
40
+ /CHANGELOG.md @glean-io/indexing-sdk-maintainers
41
+ /RELEASE.md @glean-io/indexing-sdk-maintainers
42
+ /.cz.toml @glean-io/indexing-sdk-maintainers
@@ -0,0 +1,66 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+ workflow_dispatch:
9
+
10
+ jobs:
11
+ lint:
12
+ name: Lint and Type Check
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+
17
+ - name: Set up mise
18
+ uses: jdx/mise-action@v2
19
+ with:
20
+ cache: true
21
+ env:
22
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
23
+
24
+ - name: Setup development environment
25
+ run: task setup
26
+
27
+ - name: Run linters
28
+ run: task lint
29
+
30
+ - name: Build package
31
+ run: task build
32
+
33
+ test:
34
+ name: Test Python ${{ matrix.python-version }}
35
+ runs-on: ubuntu-latest
36
+ strategy:
37
+ matrix:
38
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
39
+
40
+ steps:
41
+ - uses: actions/checkout@v4
42
+
43
+ - name: Set up mise
44
+ uses: jdx/mise-action@v2
45
+ with:
46
+ cache: true
47
+ mise_toml: |
48
+ [tools]
49
+ python = "${{ matrix.python-version }}"
50
+ task = "latest"
51
+ uv = "latest"
52
+
53
+ env:
54
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
55
+
56
+ - name: Setup development environment
57
+ run: task setup
58
+
59
+ - name: Run tests
60
+ run: task test
61
+
62
+ - name: Test package installation
63
+ run: |
64
+ task build
65
+ uv run pip install dist/*.whl
66
+ uv run python -c "import glean.indexing; print('Package installed successfully')"
@@ -0,0 +1,58 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - 'v*.*.*'
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+ deploy:
11
+ runs-on: ubuntu-latest
12
+ environment: pypi
13
+ permissions:
14
+ id-token: write
15
+ contents: write # Needed for creating GitHub releases
16
+
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ with:
20
+ fetch-depth: 0 # Needed for changelog generation
21
+
22
+ - name: Set up mise
23
+ uses: jdx/mise-action@v2
24
+ with:
25
+ cache: true
26
+ env:
27
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
28
+
29
+ - name: Build package
30
+ run: uv build
31
+
32
+ - name: Get Tag and Changelog
33
+ id: CHANGELOG
34
+ run: |
35
+ # Get latest tag for both push and manual triggers
36
+ LATEST_TAG=$(git describe --tags --abbrev=0)
37
+
38
+ echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_OUTPUT
39
+ echo "CHANGELOG<<EOF" >> $GITHUB_OUTPUT
40
+
41
+ # Try to get previous tag
42
+ if PREVIOUS_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null); then
43
+ # If we have a previous tag, show changes between tags
44
+ uvx --from commitizen cz changelog "${PREVIOUS_TAG}..${LATEST_TAG}" --dry-run >> $GITHUB_OUTPUT
45
+ else
46
+ # For first tag, show all changes up to this tag
47
+ uvx --from commitizen cz changelog --dry-run >> $GITHUB_OUTPUT
48
+ fi
49
+ echo "EOF" >> $GITHUB_OUTPUT
50
+
51
+ - name: Create GitHub Release
52
+ uses: softprops/action-gh-release@v1
53
+ with:
54
+ tag_name: ${{ steps.CHANGELOG.outputs.LATEST_TAG }}
55
+ body: ${{ steps.CHANGELOG.outputs.CHANGELOG }}
56
+
57
+ - name: Publish package to PyPI
58
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,179 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ #uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ #poetry.lock
109
+
110
+ # pdm
111
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
112
+ #pdm.lock
113
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
114
+ # in version control.
115
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
116
+ .pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
121
+ __pypackages__/
122
+
123
+ # Celery stuff
124
+ celerybeat-schedule
125
+ celerybeat.pid
126
+
127
+ # SageMath parsed files
128
+ *.sage.py
129
+
130
+ # Environments
131
+ .env
132
+ .venv
133
+ env/
134
+ venv/
135
+ ENV/
136
+ env.bak/
137
+ venv.bak/
138
+
139
+ # Spyder project settings
140
+ .spyderproject
141
+ .spyproject
142
+
143
+ # Rope project settings
144
+ .ropeproject
145
+
146
+ # mkdocs documentation
147
+ /site
148
+
149
+ # mypy
150
+ .mypy_cache/
151
+ .dmypy.json
152
+ dmypy.json
153
+
154
+ # Pyre type checker
155
+ .pyre/
156
+
157
+ # pytype static type analyzer
158
+ .pytype/
159
+
160
+ # Cython debug symbols
161
+ cython_debug/
162
+
163
+ # PyCharm
164
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
165
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
166
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
167
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
168
+ #.idea/
169
+
170
+ # PyPI configuration file
171
+ .pypirc
172
+
173
+ # UV cache
174
+ .uv/
175
+ .task/
176
+ .llm/
177
+
178
+ # Project specific
179
+ .ruff_cache/
@@ -0,0 +1,6 @@
1
+ {
2
+ "snippetRoot": "./snippets",
3
+ "markdownGlob": "README.md",
4
+ "includeExtensions": [".ts", ".js", ".py"]
5
+ }
6
+
@@ -0,0 +1 @@
1
+ 3.10.0
@@ -0,0 +1,3 @@
1
+ line-length = 100
2
+ target-version = "py310"
3
+ exclude = ["__init__.py"]
@@ -0,0 +1,4 @@
1
+ {
2
+ "python.analysis.extraPaths": ["./src"],
3
+ "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python"
4
+ }
@@ -0,0 +1,5 @@
1
+ ## 0.0.2 (2025-06-20)
2
+
3
+ ### Fix
4
+
5
+ - Adds addition model for re-export
@@ -0,0 +1,71 @@
1
+ # Contributing to the Glean Connector SDK
2
+
3
+ Thank you for your interest in contributing to the Glean Connector SDK! This document provides guidelines and instructions for contributing to this project.
4
+
5
+ ## Setup
6
+
7
+ 1. Clone the repository
8
+ 2. Set up your environment:
9
+ ```bash
10
+ # Install go-task if not already installed
11
+ brew install go-task
12
+
13
+ # Set up development environment
14
+ task setup
15
+ ```
16
+
17
+ ## Development Workflow
18
+
19
+ We use the following workflow for development:
20
+
21
+ 1. Create a new branch for your feature or bugfix:
22
+ ```bash
23
+ git checkout -b feature/your-feature-name
24
+ ```
25
+
26
+ 2. Make your changes
27
+
28
+ 3. Run linting and tests:
29
+ ```bash
30
+ task lint:fix
31
+ task test:all
32
+ ```
33
+
34
+ 4. Commit your changes using commitizen:
35
+ ```bash
36
+ uv run python -m commitizen commit
37
+ ```
38
+
39
+ 5. Push your branch and create a pull request
40
+
41
+ ## Code Style
42
+
43
+ We follow standard Python code styles:
44
+
45
+ - Use [Ruff](https://github.com/astral-sh/ruff) for linting and formatting
46
+ - Use [MyPy](https://mypy.readthedocs.io) for type checking
47
+ - Follow [type hints](https://docs.python.org/3/library/typing.html) in all code
48
+
49
+ ## Testing
50
+
51
+ - Write unit tests for all new functionality
52
+ - Ensure all tests pass before submitting a PR
53
+
54
+ ## Release Process
55
+
56
+ We use [commitizen](https://commitizen-tools.github.io/commitizen/) for versioning:
57
+
58
+ ```bash
59
+ # Perform a dry run
60
+ task release DRY_RUN=true
61
+
62
+ # Create a new release
63
+ task release
64
+ ```
65
+
66
+ ## Documentation
67
+
68
+ - Update documentation for any changes to public APIs
69
+ - Include docstrings for all public classes and methods
70
+
71
+ Thank you for your contributions!
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Glean
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.