flowlit-integration 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowlit_integration-0.1.0/.github/workflows/python-publish.yml +91 -0
- flowlit_integration-0.1.0/.gitignore +36 -0
- flowlit_integration-0.1.0/CHANGELOG.md +44 -0
- flowlit_integration-0.1.0/LICENSE +21 -0
- flowlit_integration-0.1.0/PKG-INFO +107 -0
- flowlit_integration-0.1.0/README.md +61 -0
- flowlit_integration-0.1.0/docs/architecture.md +78 -0
- flowlit_integration-0.1.0/docs/credentials.md +107 -0
- flowlit_integration-0.1.0/pyproject.toml +98 -0
- flowlit_integration-0.1.0/src/flowlit_integration/__init__.py +26 -0
- flowlit_integration-0.1.0/src/flowlit_integration/credentials/__init__.py +10 -0
- flowlit_integration-0.1.0/src/flowlit_integration/credentials/provider.py +419 -0
- flowlit_integration-0.1.0/src/flowlit_integration/executors/__init__.py +10 -0
- flowlit_integration-0.1.0/src/flowlit_integration/executors/gmail_executor.py +220 -0
- flowlit_integration-0.1.0/src/flowlit_integration/executors/google_sheets_executor.py +309 -0
- flowlit_integration-0.1.0/src/flowlit_integration/executors/harvest_executor.py +200 -0
- flowlit_integration-0.1.0/src/flowlit_integration/executors/slack_executor.py +150 -0
- flowlit_integration-0.1.0/src/flowlit_integration/executors/spreadsheet_executor.py +395 -0
- flowlit_integration-0.1.0/src/flowlit_integration/py.typed +0 -0
- flowlit_integration-0.1.0/tests/__init__.py +0 -0
- flowlit_integration-0.1.0/tests/unit/__init__.py +0 -0
- flowlit_integration-0.1.0/tests/unit/credentials/__init__.py +0 -0
- flowlit_integration-0.1.0/tests/unit/credentials/test_provider.py +237 -0
- flowlit_integration-0.1.0/tests/unit/executors/__init__.py +0 -0
- flowlit_integration-0.1.0/tests/unit/executors/conftest.py +93 -0
- flowlit_integration-0.1.0/tests/unit/executors/test_gmail_executor.py +233 -0
- flowlit_integration-0.1.0/tests/unit/executors/test_google_sheets_executor.py +356 -0
- flowlit_integration-0.1.0/tests/unit/executors/test_harvest_executor.py +195 -0
- flowlit_integration-0.1.0/tests/unit/executors/test_slack_executor.py +122 -0
- flowlit_integration-0.1.0/tests/unit/executors/test_spreadsheet_executor.py +694 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# This workflow will upload a Python Package to PyPI when a release is created
|
|
2
|
+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
|
3
|
+
|
|
4
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
5
|
+
# They are provided by a third-party and are governed by
|
|
6
|
+
# separate terms of service, privacy policy, and support
|
|
7
|
+
# documentation.
|
|
8
|
+
|
|
9
|
+
name: Upload Python Package
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
test:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
strategy:
|
|
22
|
+
matrix:
|
|
23
|
+
python-version: ["3.11", "3.12"]
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
|
|
28
|
+
- uses: actions/setup-python@v5
|
|
29
|
+
with:
|
|
30
|
+
python-version: ${{ matrix.python-version }}
|
|
31
|
+
|
|
32
|
+
- name: Install
|
|
33
|
+
run: python -m pip install -e ".[dev,all]"
|
|
34
|
+
|
|
35
|
+
- name: Test
|
|
36
|
+
run: python -m pytest -q
|
|
37
|
+
|
|
38
|
+
- name: Type check
|
|
39
|
+
run: python -m mypy src
|
|
40
|
+
|
|
41
|
+
- name: Lint
|
|
42
|
+
run: python -m ruff check src tests
|
|
43
|
+
|
|
44
|
+
release-build:
|
|
45
|
+
needs: test
|
|
46
|
+
runs-on: ubuntu-latest
|
|
47
|
+
|
|
48
|
+
steps:
|
|
49
|
+
- uses: actions/checkout@v4
|
|
50
|
+
|
|
51
|
+
- uses: actions/setup-python@v5
|
|
52
|
+
with:
|
|
53
|
+
python-version: "3.x"
|
|
54
|
+
|
|
55
|
+
- name: Build release distributions
|
|
56
|
+
run: |
|
|
57
|
+
# NOTE: put your own distribution build steps here.
|
|
58
|
+
python -m pip install build
|
|
59
|
+
python -m build
|
|
60
|
+
|
|
61
|
+
- name: Upload distributions
|
|
62
|
+
uses: actions/upload-artifact@v4
|
|
63
|
+
with:
|
|
64
|
+
name: release-dists
|
|
65
|
+
path: dist/
|
|
66
|
+
|
|
67
|
+
pypi-publish:
|
|
68
|
+
runs-on: ubuntu-latest
|
|
69
|
+
needs:
|
|
70
|
+
- release-build
|
|
71
|
+
permissions:
|
|
72
|
+
# IMPORTANT: this permission is mandatory for trusted publishing
|
|
73
|
+
id-token: write
|
|
74
|
+
|
|
75
|
+
# Dedicated environments with protections for publishing are strongly recommended.
|
|
76
|
+
# For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
|
|
77
|
+
environment:
|
|
78
|
+
name: pypi
|
|
79
|
+
url: https://pypi.org/p/flowlit-integration
|
|
80
|
+
|
|
81
|
+
steps:
|
|
82
|
+
- name: Retrieve release distributions
|
|
83
|
+
uses: actions/download-artifact@v4
|
|
84
|
+
with:
|
|
85
|
+
name: release-dists
|
|
86
|
+
path: dist/
|
|
87
|
+
|
|
88
|
+
- name: Publish release distributions to PyPI
|
|
89
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
90
|
+
with:
|
|
91
|
+
packages-dir: dist/
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.egg-info/
|
|
6
|
+
.eggs/
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
|
|
10
|
+
# Virtual environments
|
|
11
|
+
.venv/
|
|
12
|
+
venv/
|
|
13
|
+
env/
|
|
14
|
+
|
|
15
|
+
# Test / coverage / tooling caches
|
|
16
|
+
.pytest_cache/
|
|
17
|
+
.mypy_cache/
|
|
18
|
+
.ruff_cache/
|
|
19
|
+
.coverage
|
|
20
|
+
.coverage.*
|
|
21
|
+
htmlcov/
|
|
22
|
+
coverage.xml
|
|
23
|
+
|
|
24
|
+
# Credentials -- this library never ships a real one, but its own tests
|
|
25
|
+
# and any local smoke-testing might create one; never let it get committed.
|
|
26
|
+
credentials.json
|
|
27
|
+
*.credentials.json
|
|
28
|
+
service-account*.json
|
|
29
|
+
.env
|
|
30
|
+
.env.*
|
|
31
|
+
!.env.example
|
|
32
|
+
|
|
33
|
+
# Editors / OS
|
|
34
|
+
.vscode/
|
|
35
|
+
.idea/
|
|
36
|
+
.DS_Store
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `flowlit-integration` are documented here. Format
|
|
4
|
+
follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this
|
|
5
|
+
project uses [SemVer](https://semver.org/), though pre-1.0 minor bumps
|
|
6
|
+
(`0.x.0`) may include breaking changes, called out explicitly below.
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] - 2026-09-13
|
|
11
|
+
|
|
12
|
+
Initial public release. Extracted from `ccq-automation`, where these
|
|
13
|
+
executors originated as project-specific code, into a standalone,
|
|
14
|
+
reusable library — the middle tier of `flowlit` -> `flowlit-integration`
|
|
15
|
+
-> `<client application>`.
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- Five `flowlit` executors, one per file, each independently installable
|
|
20
|
+
via its own extra:
|
|
21
|
+
- `harvest` — Harvest API v2 reads (`users`, `time_entries`, or any
|
|
22
|
+
other collection endpoint), with automatic `links.next` pagination.
|
|
23
|
+
- `google-sheets` — Google Sheets read/write/append via `gspread`.
|
|
24
|
+
- `gmail` — outbound email with attachments, via SMTP + an app
|
|
25
|
+
password.
|
|
26
|
+
- `slack` — outbound Slack messages (channel or DM) via
|
|
27
|
+
`chat.postMessage`.
|
|
28
|
+
- `spreadsheet` — generic "any rows + columns in, one `.xlsx` out"
|
|
29
|
+
executor, with multi-key row sorting (`sort_by`/`sort_direction`);
|
|
30
|
+
no dependency on any of the other four.
|
|
31
|
+
- `credentials.provider`: a shared `CredentialsProvider` abstraction
|
|
32
|
+
(`Env`/`File`/`Chained`, `default_credentials_provider()`) and one
|
|
33
|
+
typed resolver per integration (`resolve_harvest_credentials`,
|
|
34
|
+
`resolve_google_sheets_credentials`, `resolve_gmail_credentials`,
|
|
35
|
+
`resolve_slack_credentials`) — explicit value > environment variable >
|
|
36
|
+
`FLOWLIT_INTEGRATION_CREDENTIALS_FILE` > a documented, not-yet-
|
|
37
|
+
implemented secret-manager extension point.
|
|
38
|
+
- Registration stays the client's own explicit job — this package never
|
|
39
|
+
calls `flowlit.register_executor(...)` itself, and importing it never
|
|
40
|
+
performs I/O or touches credentials.
|
|
41
|
+
- PEP 561 `py.typed` marker.
|
|
42
|
+
|
|
43
|
+
[Unreleased]: https://github.com/shadiwazir/flowlit-integration/compare/v0.1.0...HEAD
|
|
44
|
+
[0.1.0]: https://github.com/shadiwazir/flowlit-integration/releases/tag/v0.1.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 shadiwazir
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: flowlit-integration
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reusable flowlit executors (Harvest, Google Sheets, Gmail, Slack, spreadsheets) and their shared credentials abstraction, for any flowlit-based client.
|
|
5
|
+
Project-URL: Homepage, https://github.com/shadiwazir/flowlit-integration
|
|
6
|
+
Project-URL: Repository, https://github.com/shadiwazir/flowlit-integration
|
|
7
|
+
Project-URL: Issues, https://github.com/shadiwazir/flowlit-integration/issues
|
|
8
|
+
Author: shadiwazir
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: flowlit>=0.4.0
|
|
21
|
+
Requires-Dist: pydantic>=2.0
|
|
22
|
+
Provides-Extra: all
|
|
23
|
+
Requires-Dist: aiohttp>=3.9; extra == 'all'
|
|
24
|
+
Requires-Dist: google-auth>=2.0; extra == 'all'
|
|
25
|
+
Requires-Dist: gspread>=6.0; extra == 'all'
|
|
26
|
+
Requires-Dist: openpyxl>=3.1; extra == 'all'
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest-mock>=3.14; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
34
|
+
Requires-Dist: types-openpyxl; extra == 'dev'
|
|
35
|
+
Provides-Extra: gmail
|
|
36
|
+
Provides-Extra: google-sheets
|
|
37
|
+
Requires-Dist: google-auth>=2.0; extra == 'google-sheets'
|
|
38
|
+
Requires-Dist: gspread>=6.0; extra == 'google-sheets'
|
|
39
|
+
Provides-Extra: harvest
|
|
40
|
+
Requires-Dist: aiohttp>=3.9; extra == 'harvest'
|
|
41
|
+
Provides-Extra: slack
|
|
42
|
+
Requires-Dist: aiohttp>=3.9; extra == 'slack'
|
|
43
|
+
Provides-Extra: spreadsheet
|
|
44
|
+
Requires-Dist: openpyxl>=3.1; extra == 'spreadsheet'
|
|
45
|
+
Description-Content-Type: text/markdown
|
|
46
|
+
|
|
47
|
+
# flowlit-integration
|
|
48
|
+
|
|
49
|
+
Reusable [flowlit](https://github.com/shadiwazir/flowlit) executors for a
|
|
50
|
+
handful of common external systems -- Harvest, Google Sheets, Gmail,
|
|
51
|
+
Slack -- plus a generic "any data → spreadsheet" executor, and the
|
|
52
|
+
shared `CredentialsProvider` abstraction they all resolve secrets
|
|
53
|
+
through.
|
|
54
|
+
|
|
55
|
+
## Where this sits
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
flowlit the execution engine: DAG plans, the Executor
|
|
59
|
+
port, the EventBus. Knows nothing about any
|
|
60
|
+
specific external system.
|
|
61
|
+
|
|
62
|
+
flowlit-integration (this package) knows how to talk to a handful
|
|
63
|
+
of common external systems. Knows nothing about
|
|
64
|
+
any particular business's use of them.
|
|
65
|
+
|
|
66
|
+
<client application> e.g. ccq-automation -- business-specific DAGs,
|
|
67
|
+
business rules, transport. Imports the
|
|
68
|
+
executors it needs from here and registers them
|
|
69
|
+
with flowlit itself.
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Installing
|
|
73
|
+
|
|
74
|
+
Pick only the extras you need -- each pulls in just the third-party SDK(s)
|
|
75
|
+
that one executor requires:
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
pip install "flowlit-integration[harvest]"
|
|
79
|
+
pip install "flowlit-integration[harvest,gmail,spreadsheet]"
|
|
80
|
+
pip install "flowlit-integration[all]" # everything
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Using an executor
|
|
84
|
+
|
|
85
|
+
Registration is always the client's own explicit job -- importing this
|
|
86
|
+
package never registers anything with flowlit, and never performs I/O:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from flowlit import register_executor
|
|
90
|
+
from flowlit_integration.executors.harvest_executor import HarvestExecutor
|
|
91
|
+
|
|
92
|
+
register_executor("harvest", HarvestExecutor())
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Each executor resolves its own credentials lazily, the first time a step
|
|
96
|
+
of that type actually runs, via `flowlit_integration.credentials`'s
|
|
97
|
+
`default_credentials_provider()` chain (explicit value > environment
|
|
98
|
+
variable > credentials file > a documented, not-yet-implemented secret-
|
|
99
|
+
manager extension point) -- see `docs/architecture.md` and each
|
|
100
|
+
executor's own module docstring for its specific env-var/spec-field
|
|
101
|
+
names.
|
|
102
|
+
|
|
103
|
+
## Status
|
|
104
|
+
|
|
105
|
+
Alpha (pre-1.0: minor version bumps may include breaking changes).
|
|
106
|
+
Published on PyPI. See [CHANGELOG.md](CHANGELOG.md) for what changed in
|
|
107
|
+
each release.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# flowlit-integration
|
|
2
|
+
|
|
3
|
+
Reusable [flowlit](https://github.com/shadiwazir/flowlit) executors for a
|
|
4
|
+
handful of common external systems -- Harvest, Google Sheets, Gmail,
|
|
5
|
+
Slack -- plus a generic "any data → spreadsheet" executor, and the
|
|
6
|
+
shared `CredentialsProvider` abstraction they all resolve secrets
|
|
7
|
+
through.
|
|
8
|
+
|
|
9
|
+
## Where this sits
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
flowlit the execution engine: DAG plans, the Executor
|
|
13
|
+
port, the EventBus. Knows nothing about any
|
|
14
|
+
specific external system.
|
|
15
|
+
|
|
16
|
+
flowlit-integration (this package) knows how to talk to a handful
|
|
17
|
+
of common external systems. Knows nothing about
|
|
18
|
+
any particular business's use of them.
|
|
19
|
+
|
|
20
|
+
<client application> e.g. ccq-automation -- business-specific DAGs,
|
|
21
|
+
business rules, transport. Imports the
|
|
22
|
+
executors it needs from here and registers them
|
|
23
|
+
with flowlit itself.
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Installing
|
|
27
|
+
|
|
28
|
+
Pick only the extras you need -- each pulls in just the third-party SDK(s)
|
|
29
|
+
that one executor requires:
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
pip install "flowlit-integration[harvest]"
|
|
33
|
+
pip install "flowlit-integration[harvest,gmail,spreadsheet]"
|
|
34
|
+
pip install "flowlit-integration[all]" # everything
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Using an executor
|
|
38
|
+
|
|
39
|
+
Registration is always the client's own explicit job -- importing this
|
|
40
|
+
package never registers anything with flowlit, and never performs I/O:
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from flowlit import register_executor
|
|
44
|
+
from flowlit_integration.executors.harvest_executor import HarvestExecutor
|
|
45
|
+
|
|
46
|
+
register_executor("harvest", HarvestExecutor())
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Each executor resolves its own credentials lazily, the first time a step
|
|
50
|
+
of that type actually runs, via `flowlit_integration.credentials`'s
|
|
51
|
+
`default_credentials_provider()` chain (explicit value > environment
|
|
52
|
+
variable > credentials file > a documented, not-yet-implemented secret-
|
|
53
|
+
manager extension point) -- see `docs/architecture.md` and each
|
|
54
|
+
executor's own module docstring for its specific env-var/spec-field
|
|
55
|
+
names.
|
|
56
|
+
|
|
57
|
+
## Status
|
|
58
|
+
|
|
59
|
+
Alpha (pre-1.0: minor version bumps may include breaking changes).
|
|
60
|
+
Published on PyPI. See [CHANGELOG.md](CHANGELOG.md) for what changed in
|
|
61
|
+
each release.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
## Package layout
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
src/flowlit_integration/
|
|
7
|
+
executors/ one flowlit Executor per file, one per external
|
|
8
|
+
system (or generic capability):
|
|
9
|
+
harvest_executor.py Harvest API v2 (users, time entries)
|
|
10
|
+
google_sheets_executor.py Google Sheets (read/write/append)
|
|
11
|
+
gmail_executor.py outbound email via SMTP
|
|
12
|
+
slack_executor.py outbound Slack messages
|
|
13
|
+
spreadsheet_executor.py generic rows+columns -> .xlsx
|
|
14
|
+
credentials/ the CredentialsProvider abstraction shared by
|
|
15
|
+
every executor above, plus one typed resolver
|
|
16
|
+
per integration (resolve_harvest_credentials,
|
|
17
|
+
resolve_google_sheets_credentials,
|
|
18
|
+
resolve_gmail_credentials,
|
|
19
|
+
resolve_slack_credentials).
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Not included: a Slack *signing-secret* resolver (inbound webhook
|
|
23
|
+
verification). None of the executors above need it -- see
|
|
24
|
+
`credentials/provider.py`'s own note on why that stays a client concern.
|
|
25
|
+
|
|
26
|
+
Each executor lives in exactly one file and has zero knowledge of any
|
|
27
|
+
other executor. None of them import anything from a client application --
|
|
28
|
+
this package's own test suite proves that by never importing
|
|
29
|
+
`ccq_automation` (or any other client) anywhere.
|
|
30
|
+
|
|
31
|
+
## Credentials
|
|
32
|
+
|
|
33
|
+
`credentials.provider.CredentialsProvider` is a small ABC with `get`/
|
|
34
|
+
`require` (the latter raising `MissingCredentialError`). Three
|
|
35
|
+
implementations chain together via `default_credentials_provider()`:
|
|
36
|
+
|
|
37
|
+
1. An explicit value passed directly to a resolver call (highest
|
|
38
|
+
precedence -- lets a caller override everything for a single call,
|
|
39
|
+
e.g. in a test).
|
|
40
|
+
2. `EnvCredentialsProvider` -- plain environment variables.
|
|
41
|
+
3. `FileCredentialsProvider` -- a flat JSON credentials file (path from
|
|
42
|
+
`CCQ_CREDENTIALS_FILE`... see the module docstring for how a client
|
|
43
|
+
points this elsewhere via its own env var naming).
|
|
44
|
+
4. A documented, not-yet-implemented extension point for a cloud secret
|
|
45
|
+
manager.
|
|
46
|
+
|
|
47
|
+
Each executor's constructor accepts an optional `CredentialsProvider`
|
|
48
|
+
and defaults to `default_credentials_provider()` -- so constructing an
|
|
49
|
+
executor, and even registering it with flowlit, never touches
|
|
50
|
+
credentials or performs I/O. Resolution happens lazily, inside each
|
|
51
|
+
executor's own `_run()`, the first time a step of that type is actually
|
|
52
|
+
dispatched.
|
|
53
|
+
|
|
54
|
+
## Registration is the client's job
|
|
55
|
+
|
|
56
|
+
This package never calls `flowlit.register_executor(...)` itself, and
|
|
57
|
+
never imports `flowlit.plugins`. A client wires an executor into its own
|
|
58
|
+
flowlit DAGs explicitly:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from flowlit import register_executor
|
|
62
|
+
from flowlit_integration.executors.gmail_executor import GmailExecutor
|
|
63
|
+
|
|
64
|
+
register_executor("gmail", GmailExecutor())
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
This mirrors flowlit's own plugin philosophy (explicit call, not
|
|
68
|
+
automatic discovery) and keeps this package a pure library with no
|
|
69
|
+
import-time side effects -- a client can import individual executor
|
|
70
|
+
modules (e.g. for type-checking a spec) without registering anything.
|
|
71
|
+
|
|
72
|
+
## What does *not* live here
|
|
73
|
+
|
|
74
|
+
Anything specific to one business's use of these executors -- validation
|
|
75
|
+
rules, data-transformation/formatting rules, DAG wiring, settings that
|
|
76
|
+
aren't secrets (e.g. which Google Sheet id/tab to read) -- stays in the
|
|
77
|
+
client application. This package only ever grows genuinely reusable
|
|
78
|
+
"talk to external system X" code and the credential plumbing it needs.
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# Credentials
|
|
2
|
+
|
|
3
|
+
Every executor in this package needs at least one secret (an API token,
|
|
4
|
+
a service-account key, an SMTP password) to make its outbound call. They
|
|
5
|
+
all resolve it through one shared abstraction —
|
|
6
|
+
[`flowlit_integration.credentials.provider`](../src/flowlit_integration/credentials/provider.py)
|
|
7
|
+
— rather than each executor reading `os.environ` (or a config file) on
|
|
8
|
+
its own. See that module's own docstring for the full design rationale;
|
|
9
|
+
this document is the user-facing reference: what sources are supported,
|
|
10
|
+
in what order, and exactly which keys each executor needs.
|
|
11
|
+
|
|
12
|
+
## Why more than one source
|
|
13
|
+
|
|
14
|
+
Different environments call for different ways to hand over a secret:
|
|
15
|
+
|
|
16
|
+
- A **local dev shell** usually just exports environment variables.
|
|
17
|
+
- A **container** (Docker Compose, a Kubernetes Pod, an Azure App
|
|
18
|
+
Service "Configuration" file mount) more often gets secrets as a
|
|
19
|
+
mounted file — one JSON document, not a pile of separate env vars.
|
|
20
|
+
- A **cloud deployment** eventually wants a managed secret store, not
|
|
21
|
+
files or plain env vars at all.
|
|
22
|
+
|
|
23
|
+
Rather than hardcode one of these, `CredentialsProvider` is a small,
|
|
24
|
+
pluggable abstraction with a defined precedence, so a client adding a
|
|
25
|
+
new source later is a one-place change (see
|
|
26
|
+
[Extension point](#extension-point-a-cloud-secret-manager) below) — no
|
|
27
|
+
executor code changes.
|
|
28
|
+
|
|
29
|
+
## Supported sources, and precedence
|
|
30
|
+
|
|
31
|
+
For any single credential lookup (say, `HARVEST_TOKEN`), sources are
|
|
32
|
+
tried in this order — first hit wins:
|
|
33
|
+
|
|
34
|
+
1. **Explicit / inline value.** Not a `CredentialsProvider` source at
|
|
35
|
+
all — a resolver function (e.g. `resolve_harvest_credentials`) or a
|
|
36
|
+
step's own `spec` can hand in an already-resolved value directly,
|
|
37
|
+
which is used as-is, before any provider is even consulted. This is
|
|
38
|
+
how Google Sheets' `credentials_json`/`credentials_path` spec fields
|
|
39
|
+
work.
|
|
40
|
+
2. **Environment variable.** `EnvCredentialsProvider` — plain
|
|
41
|
+
`os.environ` lookup by key.
|
|
42
|
+
3. **Credentials file.** `FileCredentialsProvider` — a flat JSON file:
|
|
43
|
+
|
|
44
|
+
```json
|
|
45
|
+
{
|
|
46
|
+
"HARVEST_TOKEN": "example-harvest-token",
|
|
47
|
+
"HARVEST_ACCOUNT_ID": "1234567",
|
|
48
|
+
"SLACK_BOT_TOKEN": "xoxb-example-0000000000-0000000000000-abcdefghijklmnopqrstuvwx",
|
|
49
|
+
"GMAIL_SMTP_USERNAME": "reports@example.com",
|
|
50
|
+
"GMAIL_SMTP_APP_PASSWORD": "example-app-password",
|
|
51
|
+
"GOOGLE_APPLICATION_CREDENTIALS": "/run/secrets/google-service-account.json"
|
|
52
|
+
}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
(Placeholder values only — never commit a real one; see
|
|
56
|
+
[Keeping secrets out of git](#keeping-secrets-out-of-git).) Enabled by
|
|
57
|
+
pointing the `FLOWLIT_INTEGRATION_CREDENTIALS_FILE` environment
|
|
58
|
+
variable at this file's path. A missing file is *not* an error — it
|
|
59
|
+
just means this source contributes nothing, and resolution falls
|
|
60
|
+
through to the next source (or fails with `MissingCredentialError` if
|
|
61
|
+
nothing else has the key either). A file that exists but isn't valid
|
|
62
|
+
JSON, or isn't a flat object of strings, *is* treated as a real
|
|
63
|
+
misconfiguration and raises.
|
|
64
|
+
4. **(Not yet implemented) a cloud secret manager** — see
|
|
65
|
+
[Extension point](#extension-point-a-cloud-secret-manager).
|
|
66
|
+
|
|
67
|
+
The same key names are used both as environment variable names and as
|
|
68
|
+
credentials-file JSON keys — moving a given secret from one source to
|
|
69
|
+
the other never means renaming it.
|
|
70
|
+
|
|
71
|
+
`default_credentials_provider()` is the one factory that builds this
|
|
72
|
+
chain (today: env, then the credentials file if
|
|
73
|
+
`FLOWLIT_INTEGRATION_CREDENTIALS_FILE` is set and points at a real file)
|
|
74
|
+
— every executor's constructor defaults to it, but also accepts an
|
|
75
|
+
explicit `CredentialsProvider` for tests or a non-default setup.
|
|
76
|
+
|
|
77
|
+
## Keys each executor needs
|
|
78
|
+
|
|
79
|
+
| Executor | Keys | Notes |
|
|
80
|
+
|---|---|---|
|
|
81
|
+
| `harvest` | `HARVEST_TOKEN`, `HARVEST_ACCOUNT_ID` | Personal access token + the account it scopes to. Sent as `Authorization: Bearer <token>` and `Harvest-Account-ID`. |
|
|
82
|
+
| `slack` | `SLACK_BOT_TOKEN` | Bot token with the `chat:write` scope. Sent as `Authorization: Bearer <token>`. |
|
|
83
|
+
| `gmail` | `GMAIL_SMTP_USERNAME`, `GMAIL_SMTP_APP_PASSWORD` | The sending Gmail address, and a Google Account *app password* (Google Account → Security → 2-Step Verification → App passwords) — not the account's normal login password. |
|
|
84
|
+
| `google_sheets` | `GOOGLE_CREDENTIALS_JSON` (inline service-account JSON) **or** `GOOGLE_APPLICATION_CREDENTIALS` (path to the service-account JSON file) | Structurally different from the others — a service-account credential is a multi-field JSON document, not a single flat secret. Only one of the two is needed; both env var names are also accepted as step `spec` fields (`credentials_json` / `credentials_path`), which take precedence over the environment — see `resolve_google_sheets_credentials`. If neither is set, `gspread`'s own default local discovery (`gspread.service_account()`) is used as a last resort. |
|
|
85
|
+
| `spreadsheet` | *(none)* | Purely generic — writes whatever rows it's given to an `.xlsx` file. No external system, no credentials. |
|
|
86
|
+
|
|
87
|
+
## Extension point: a cloud secret manager
|
|
88
|
+
|
|
89
|
+
A `KeyVaultCredentialsProvider` (Azure) or equivalent for another cloud,
|
|
90
|
+
implementing `CredentialsProvider.get()`, is the natural next source for
|
|
91
|
+
a client deploying there — added to the list `default_credentials_provider()`
|
|
92
|
+
builds, typically after the credentials file, so an explicit local
|
|
93
|
+
override still wins over the vault. Not implemented here; this paragraph
|
|
94
|
+
is the documented placeholder. When a client needs it, no executor
|
|
95
|
+
changes — only `default_credentials_provider()` and a new provider
|
|
96
|
+
class, in this package or in the client's own code (a client can always
|
|
97
|
+
pass its own `CredentialsProvider` into any executor's constructor
|
|
98
|
+
instead of relying on this package's default chain at all).
|
|
99
|
+
|
|
100
|
+
## Keeping secrets out of git
|
|
101
|
+
|
|
102
|
+
`.gitignore` already excludes `.env`, `.env.*` (except `.env.example`),
|
|
103
|
+
and common credentials-file names (`credentials.json`,
|
|
104
|
+
`*.credentials.json`, `service-account*.json`) — see the repo root
|
|
105
|
+
`.gitignore`. If you name your `FLOWLIT_INTEGRATION_CREDENTIALS_FILE`
|
|
106
|
+
something outside those patterns, add it to `.gitignore` yourself before
|
|
107
|
+
creating it.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "flowlit-integration"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Reusable flowlit executors (Harvest, Google Sheets, Gmail, Slack, spreadsheets) and their shared credentials abstraction, for any flowlit-based client."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "shadiwazir" }]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Topic :: Software Development :: Libraries",
|
|
21
|
+
"Typing :: Typed",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
dependencies = [
|
|
25
|
+
# The step-execution contract every executor here implements
|
|
26
|
+
# (Executor/BaseExecutor/ExecutorResult) and the spec-validation
|
|
27
|
+
# model every executor's Spec is built on. Bumped to 0.4.0 to track
|
|
28
|
+
# the current release; none of this package's own code depends on
|
|
29
|
+
# anything new in it (0.4.0 only moved workflow-status JSON
|
|
30
|
+
# formatting into flowlit itself, a concern this package never had).
|
|
31
|
+
"flowlit>=0.4.0",
|
|
32
|
+
"pydantic>=2.0",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://github.com/shadiwazir/flowlit-integration"
|
|
37
|
+
Repository = "https://github.com/shadiwazir/flowlit-integration"
|
|
38
|
+
Issues = "https://github.com/shadiwazir/flowlit-integration/issues"
|
|
39
|
+
|
|
40
|
+
[project.optional-dependencies]
|
|
41
|
+
# One extra per executor, so a client only pulls the third-party SDK(s)
|
|
42
|
+
# that specific integration needs -- see src/flowlit_integration/executors/
|
|
43
|
+
# for which executor uses which.
|
|
44
|
+
harvest = [
|
|
45
|
+
"aiohttp>=3.9",
|
|
46
|
+
]
|
|
47
|
+
google-sheets = [
|
|
48
|
+
"gspread>=6.0",
|
|
49
|
+
"google-auth>=2.0",
|
|
50
|
+
]
|
|
51
|
+
gmail = [
|
|
52
|
+
# Gmail dispatch uses stdlib smtplib (via asyncio.to_thread) --
|
|
53
|
+
# no extra SDK dependency needed. Kept as a named extra anyway so
|
|
54
|
+
# `flowlit-integration[gmail]` reads consistently with its siblings
|
|
55
|
+
# even though today it resolves to nothing extra.
|
|
56
|
+
]
|
|
57
|
+
slack = [
|
|
58
|
+
"aiohttp>=3.9",
|
|
59
|
+
]
|
|
60
|
+
spreadsheet = [
|
|
61
|
+
"openpyxl>=3.1",
|
|
62
|
+
]
|
|
63
|
+
all = [
|
|
64
|
+
"flowlit-integration[harvest,google-sheets,gmail,slack,spreadsheet]",
|
|
65
|
+
]
|
|
66
|
+
dev = [
|
|
67
|
+
"pytest>=8.0",
|
|
68
|
+
"pytest-asyncio>=0.24",
|
|
69
|
+
"pytest-mock>=3.14",
|
|
70
|
+
"pytest-cov>=5.0",
|
|
71
|
+
"ruff>=0.6",
|
|
72
|
+
"mypy>=1.11",
|
|
73
|
+
"types-openpyxl", # openpyxl ships no py.typed marker of its own
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
[tool.hatch.build.targets.wheel]
|
|
77
|
+
packages = ["src/flowlit_integration"]
|
|
78
|
+
|
|
79
|
+
[tool.pytest.ini_options]
|
|
80
|
+
asyncio_mode = "auto"
|
|
81
|
+
testpaths = ["tests"]
|
|
82
|
+
|
|
83
|
+
[tool.ruff]
|
|
84
|
+
target-version = "py311"
|
|
85
|
+
line-length = 100
|
|
86
|
+
|
|
87
|
+
[tool.ruff.lint]
|
|
88
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
89
|
+
|
|
90
|
+
[tool.mypy]
|
|
91
|
+
python_version = "3.11"
|
|
92
|
+
disallow_untyped_defs = true
|
|
93
|
+
warn_unused_ignores = true
|
|
94
|
+
no_implicit_optional = true
|
|
95
|
+
|
|
96
|
+
[[tool.mypy.overrides]]
|
|
97
|
+
module = "flowlit_integration.credentials.*"
|
|
98
|
+
strict = true
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""flowlit-integration: reusable flowlit executors for common external
|
|
2
|
+
systems (Harvest, Google Sheets, Gmail, Slack) and generic data output
|
|
3
|
+
(spreadsheets), plus the shared `CredentialsProvider` abstraction they
|
|
4
|
+
resolve their secrets through.
|
|
5
|
+
|
|
6
|
+
This package sits in the middle of a three-tier hierarchy:
|
|
7
|
+
|
|
8
|
+
flowlit -> flowlit-integration -> <client application>
|
|
9
|
+
|
|
10
|
+
`flowlit` is the execution engine (DAG plans, the `Executor` port, the
|
|
11
|
+
`EventBus`) and has no knowledge of any specific external system.
|
|
12
|
+
`flowlit-integration` knows how to talk to a handful of common ones, but
|
|
13
|
+
nothing about any particular business's use of them. A client
|
|
14
|
+
application (e.g. `ccq-automation`) imports the executors it needs from
|
|
15
|
+
here, registers them with `flowlit` itself
|
|
16
|
+
(`flowlit.register_executor(step_type, executor_instance)`), and wires
|
|
17
|
+
them into its own business-specific DAGs.
|
|
18
|
+
|
|
19
|
+
Registration is deliberately *not* automatic: importing this package, or
|
|
20
|
+
any module in it, performs no registration and no I/O. A client calls
|
|
21
|
+
`flowlit.register_executor(...)` explicitly for each executor it wants,
|
|
22
|
+
the same way it would for an executor it wrote itself -- see each
|
|
23
|
+
executor's own module docstring for its step-type name and spec shape.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|