backlot 0.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- backlot-0.0.0/LICENSE +21 -0
- backlot-0.0.0/PKG-INFO +410 -0
- backlot-0.0.0/README.md +340 -0
- backlot-0.0.0/backlot/__init__.py +5 -0
- backlot-0.0.0/backlot/__main__.py +10 -0
- backlot-0.0.0/backlot/acl.py +89 -0
- backlot-0.0.0/backlot/auth.py +209 -0
- backlot-0.0.0/backlot/cli.py +164 -0
- backlot-0.0.0/backlot/config.py +130 -0
- backlot-0.0.0/backlot/data/hello.jsonl +136 -0
- backlot-0.0.0/backlot/errors/__init__.py +44 -0
- backlot-0.0.0/backlot/errors/atlassian.py +46 -0
- backlot-0.0.0/backlot/errors/google.py +230 -0
- backlot-0.0.0/backlot/graphql/__init__.py +8 -0
- backlot-0.0.0/backlot/graphql/engine.py +147 -0
- backlot-0.0.0/backlot/graphql/fireflies.graphql +240 -0
- backlot-0.0.0/backlot/graphql/fireflies_resolvers.py +357 -0
- backlot-0.0.0/backlot/graphql/linear.graphql +836 -0
- backlot-0.0.0/backlot/graphql/linear_filters.py +458 -0
- backlot-0.0.0/backlot/graphql/linear_resolvers.py +1215 -0
- backlot-0.0.0/backlot/importer/__init__.py +5 -0
- backlot-0.0.0/backlot/importer/byo.py +1440 -0
- backlot-0.0.0/backlot/importer/erb.py +2458 -0
- backlot-0.0.0/backlot/integrations/__init__.py +11 -0
- backlot-0.0.0/backlot/integrations/llamaindex.py +312 -0
- backlot-0.0.0/backlot/integrations/mirage.py +100 -0
- backlot-0.0.0/backlot/main.py +403 -0
- backlot-0.0.0/backlot/oauth.py +160 -0
- backlot-0.0.0/backlot/openapi.py +137 -0
- backlot-0.0.0/backlot/pagination.py +116 -0
- backlot-0.0.0/backlot/routers/__init__.py +25 -0
- backlot-0.0.0/backlot/routers/atlassian.py +1154 -0
- backlot-0.0.0/backlot/routers/fireflies.py +69 -0
- backlot-0.0.0/backlot/routers/github.py +901 -0
- backlot-0.0.0/backlot/routers/google.py +2144 -0
- backlot-0.0.0/backlot/routers/hubspot.py +622 -0
- backlot-0.0.0/backlot/routers/linear.py +79 -0
- backlot-0.0.0/backlot/routers/notion.py +541 -0
- backlot-0.0.0/backlot/routers/oauth.py +65 -0
- backlot-0.0.0/backlot/routers/s3.py +376 -0
- backlot-0.0.0/backlot/routers/slack.py +862 -0
- backlot-0.0.0/backlot/schemas/README.md +217 -0
- backlot-0.0.0/backlot/schemas/confluence.schema.json +188 -0
- backlot-0.0.0/backlot/schemas/fireflies.schema.json +304 -0
- backlot-0.0.0/backlot/schemas/github.schema.json +255 -0
- backlot-0.0.0/backlot/schemas/gmail.schema.json +222 -0
- backlot-0.0.0/backlot/schemas/google_drive.schema.json +138 -0
- backlot-0.0.0/backlot/schemas/hubspot.schema.json +140 -0
- backlot-0.0.0/backlot/schemas/jira.schema.json +231 -0
- backlot-0.0.0/backlot/schemas/linear.schema.json +298 -0
- backlot-0.0.0/backlot/schemas/notion.schema.json +173 -0
- backlot-0.0.0/backlot/schemas/s3.schema.json +118 -0
- backlot-0.0.0/backlot/schemas/slack.schema.json +161 -0
- backlot-0.0.0/backlot/sigv4.py +121 -0
- backlot-0.0.0/backlot/store.py +1871 -0
- backlot-0.0.0/backlot/synth.py +823 -0
- backlot-0.0.0/backlot/testing.py +276 -0
- backlot-0.0.0/backlot/validation.py +91 -0
- backlot-0.0.0/backlot.egg-info/PKG-INFO +410 -0
- backlot-0.0.0/backlot.egg-info/SOURCES.txt +93 -0
- backlot-0.0.0/backlot.egg-info/dependency_links.txt +1 -0
- backlot-0.0.0/backlot.egg-info/entry_points.txt +2 -0
- backlot-0.0.0/backlot.egg-info/requires.txt +54 -0
- backlot-0.0.0/backlot.egg-info/top_level.txt +1 -0
- backlot-0.0.0/pyproject.toml +134 -0
- backlot-0.0.0/setup.cfg +4 -0
- backlot-0.0.0/tests/test_acl.py +393 -0
- backlot-0.0.0/tests/test_atlassian.py +364 -0
- backlot-0.0.0/tests/test_auth.py +131 -0
- backlot-0.0.0/tests/test_cli.py +229 -0
- backlot-0.0.0/tests/test_config.py +23 -0
- backlot-0.0.0/tests/test_cross_vendor.py +278 -0
- backlot-0.0.0/tests/test_fireflies.py +506 -0
- backlot-0.0.0/tests/test_github.py +542 -0
- backlot-0.0.0/tests/test_google.py +2140 -0
- backlot-0.0.0/tests/test_graphql.py +395 -0
- backlot-0.0.0/tests/test_hubspot.py +547 -0
- backlot-0.0.0/tests/test_importer_byo.py +1966 -0
- backlot-0.0.0/tests/test_importer_erb.py +2866 -0
- backlot-0.0.0/tests/test_integrations.py +270 -0
- backlot-0.0.0/tests/test_linear.py +891 -0
- backlot-0.0.0/tests/test_llamaindex.py +385 -0
- backlot-0.0.0/tests/test_mcp.py +468 -0
- backlot-0.0.0/tests/test_notion.py +237 -0
- backlot-0.0.0/tests/test_openapi.py +197 -0
- backlot-0.0.0/tests/test_packaging.py +56 -0
- backlot-0.0.0/tests/test_pagination.py +239 -0
- backlot-0.0.0/tests/test_s3.py +661 -0
- backlot-0.0.0/tests/test_schema.py +640 -0
- backlot-0.0.0/tests/test_sdk.py +606 -0
- backlot-0.0.0/tests/test_search.py +326 -0
- backlot-0.0.0/tests/test_slack.py +348 -0
- backlot-0.0.0/tests/test_store.py +857 -0
- backlot-0.0.0/tests/test_synth.py +222 -0
- backlot-0.0.0/tests/test_testing.py +136 -0
backlot-0.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 BrekkyLab
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
backlot-0.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: backlot
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: A read-only mock of enterprise SaaS knowledge APIs, served over your own corpus with per-document ACLs
|
|
5
|
+
Author-email: khj809 <onsealeatang@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/brekkylab/backlot
|
|
8
|
+
Project-URL: Repository, https://github.com/brekkylab/backlot
|
|
9
|
+
Project-URL: Issues, https://github.com/brekkylab/backlot/issues
|
|
10
|
+
Keywords: mock,rag,slack,gmail,google-drive,jira,confluence,github,notion,s3,amazon-s3,hubspot,linear,graphql,benchmark
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Framework :: FastAPI
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing :: Mocking
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: fastapi>=0.110
|
|
21
|
+
Requires-Dist: uvicorn[standard]>=0.29
|
|
22
|
+
Requires-Dist: pydantic>=2.6
|
|
23
|
+
Requires-Dist: pydantic-settings>=2.2
|
|
24
|
+
Requires-Dist: pyyaml>=6.0
|
|
25
|
+
Requires-Dist: python-multipart>=0.0.9
|
|
26
|
+
Requires-Dist: jsonschema>=4.20
|
|
27
|
+
Requires-Dist: pyjwt[crypto]>=2.8
|
|
28
|
+
Requires-Dist: httpx>=0.27
|
|
29
|
+
Requires-Dist: graphql-core>=3.2
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
32
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
33
|
+
Requires-Dist: ruff~=0.15.22; extra == "dev"
|
|
34
|
+
Provides-Extra: examples
|
|
35
|
+
Requires-Dist: httpx>=0.27; extra == "examples"
|
|
36
|
+
Requires-Dist: slack_sdk>=3.27; extra == "examples"
|
|
37
|
+
Requires-Dist: PyGithub>=2.3; extra == "examples"
|
|
38
|
+
Requires-Dist: atlassian-python-api>=3.41; extra == "examples"
|
|
39
|
+
Requires-Dist: google-api-python-client>=2.120; extra == "examples"
|
|
40
|
+
Requires-Dist: google-auth>=2.29; extra == "examples"
|
|
41
|
+
Requires-Dist: notion-client>=3.1; extra == "examples"
|
|
42
|
+
Requires-Dist: boto3>=1.34; extra == "examples"
|
|
43
|
+
Requires-Dist: hubspot-api-client>=12; extra == "examples"
|
|
44
|
+
Provides-Extra: mcp
|
|
45
|
+
Requires-Dist: mcp>=1.2; extra == "mcp"
|
|
46
|
+
Requires-Dist: anthropic[mcp]>=0.40; extra == "mcp"
|
|
47
|
+
Requires-Dist: openai>=1.40; extra == "mcp"
|
|
48
|
+
Requires-Dist: openai-agents>=0.1; extra == "mcp"
|
|
49
|
+
Requires-Dist: pyyaml>=6.0; extra == "mcp"
|
|
50
|
+
Requires-Dist: fastmcp>=2.0; extra == "mcp"
|
|
51
|
+
Provides-Extra: llamaindex
|
|
52
|
+
Requires-Dist: llama-index-readers-slack>=0.5; extra == "llamaindex"
|
|
53
|
+
Requires-Dist: llama-index-readers-google>=0.7; extra == "llamaindex"
|
|
54
|
+
Requires-Dist: llama-index-readers-github>=0.11; extra == "llamaindex"
|
|
55
|
+
Requires-Dist: llama-index-readers-jira>=0.6; extra == "llamaindex"
|
|
56
|
+
Requires-Dist: llama-index-readers-confluence>=0.7; extra == "llamaindex"
|
|
57
|
+
Requires-Dist: llama-index-readers-notion>=0.5; extra == "llamaindex"
|
|
58
|
+
Requires-Dist: llama-index-readers-s3>=0.6; extra == "llamaindex"
|
|
59
|
+
Requires-Dist: llama-index-readers-linear>=0.5; extra == "llamaindex"
|
|
60
|
+
Requires-Dist: jira>=3.10; extra == "llamaindex"
|
|
61
|
+
Requires-Dist: atlassian-python-api>=4.0; extra == "llamaindex"
|
|
62
|
+
Requires-Dist: slack_sdk>=3.43; extra == "llamaindex"
|
|
63
|
+
Requires-Dist: google-api-python-client>=2.198; extra == "llamaindex"
|
|
64
|
+
Requires-Dist: google-auth>=2.55; extra == "llamaindex"
|
|
65
|
+
Requires-Dist: boto3>=1.40; extra == "llamaindex"
|
|
66
|
+
Requires-Dist: s3fs>=2026.6; extra == "llamaindex"
|
|
67
|
+
Provides-Extra: mirage
|
|
68
|
+
Requires-Dist: mirage-ai[fuse]~=0.0.4; extra == "mirage"
|
|
69
|
+
Dynamic: license-file
|
|
70
|
+
|
|
71
|
+
# Backlot
|
|
72
|
+
|
|
73
|
+
[](https://github.com/brekkylab/backlot/actions/workflows/ci.yml)
|
|
74
|
+

|
|
75
|
+
[](LICENSE)
|
|
76
|
+
|
|
77
|
+
A **read-only** mock server that stands in for a whole stack of enterprise SaaS knowledge sources
|
|
78
|
+
at once.
|
|
79
|
+
It speaks each service's real read API — the exact response shapes, pagination schemes, auth,
|
|
80
|
+
and native permission endpoints their official SDKs expect — over a corpus **you** supply, so a
|
|
81
|
+
RAG/search connector built on those SDKs can be exercised **end-to-end** without the live
|
|
82
|
+
services.
|
|
83
|
+
|
|
84
|
+
## Quickstart
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install backlot
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
That puts the `backlot` command on PATH. A server needs a corpus, and one is bundled with the
|
|
91
|
+
package — 136 documents covering every source it serves — so there is nothing to fetch or write:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
backlot import --bundled # the bundled corpus -> data/mock.sqlite + data/tokens.yaml
|
|
95
|
+
backlot serve # http://127.0.0.1:8000
|
|
96
|
+
curl -s localhost:8000/health
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Or skip the CLI entirely and let a test spin one up on a free port, serving that same corpus
|
|
100
|
+
(`pip install "backlot[examples]"` for the vendor SDKs the first snippet uses):
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
import backlot
|
|
104
|
+
from slack_sdk import WebClient
|
|
105
|
+
|
|
106
|
+
with backlot.mock_server() as m: # no arguments: the bundled corpus
|
|
107
|
+
slack = WebClient(token=m.token, base_url=f"{m.base_url}/slack/api/")
|
|
108
|
+
print(slack.conversations_list()["channels"])
|
|
109
|
+
|
|
110
|
+
with backlot.mock_server(records=[ # or your own records, inline
|
|
111
|
+
{"source_type": "confluence", "space": "handbook", "title": "On-call",
|
|
112
|
+
"content": "Page for Sev1 and Sev2 only.", "author_email": "ava@acme.com"},
|
|
113
|
+
]) as m:
|
|
114
|
+
...
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
`python -m backlot` is the same CLI as the `backlot` script, for when the venv is not activated.
|
|
118
|
+
Working on Backlot itself instead of with it? [CONTRIBUTING.md](CONTRIBUTING.md) covers the
|
|
119
|
+
from-source install.
|
|
120
|
+
|
|
121
|
+
### Docker
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
docker build -t backlot . # server + the bundled corpus baked in; no download
|
|
125
|
+
docker run -p 8000:8000 backlot
|
|
126
|
+
curl -s localhost:8000/health
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
That image answers every endpoint of every source out of the box. Use
|
|
130
|
+
`--target serve` for a server with **no** corpus, for a deployment that mounts its own
|
|
131
|
+
`/app/data`.
|
|
132
|
+
|
|
133
|
+
## Why you need this
|
|
134
|
+
|
|
135
|
+
Connectors are easy to write and hard to trust. Proving one works means an account with every
|
|
136
|
+
vendor, an OAuth app per vendor, seeded data in each, and rate limits between you and every retry —
|
|
137
|
+
so most of it
|
|
138
|
+
gets tested against fixtures that agree with your assumptions instead of with the API. Backlot is
|
|
139
|
+
the API: same shapes, same pagination, same permission endpoints, over documents you control.
|
|
140
|
+
|
|
141
|
+
It is the right tool when you are:
|
|
142
|
+
|
|
143
|
+
- **building or upgrading a connector** — crawl a source to exhaustion, then diff what you got
|
|
144
|
+
against what you loaded, on a corpus small enough to reason about
|
|
145
|
+
- **testing ACL-scoped retrieval** — every document carries its own readers, and each user's token
|
|
146
|
+
sees only theirs, so "does this leak" is a test rather than an audit
|
|
147
|
+
- **running that suite in CI** — no accounts, no secrets, no network, no flakes from someone else's
|
|
148
|
+
outage; a server starts in a second and dies with the test
|
|
149
|
+
- **evaluating RAG or agents** — point an SDK, an MCP server, or a LlamaIndex reader at it and get
|
|
150
|
+
the same answers on every run, because ids and timestamps are derived, not random
|
|
151
|
+
- **reproducing a bug you cannot reach** — a paginated edge case, an odd MIME type, an empty thread:
|
|
152
|
+
write the document that causes it and serve it in seconds
|
|
153
|
+
|
|
154
|
+
It is **not** a sandbox for writes, a rate-limit or latency simulator, or a source of realistic
|
|
155
|
+
*content* — the documents are yours.
|
|
156
|
+
|
|
157
|
+
## Preparing a corpus
|
|
158
|
+
|
|
159
|
+
The server reads a corpus from `data/` (`mock.sqlite` + `tokens.yaml`). Build it from your own
|
|
160
|
+
documents, or load a public dataset.
|
|
161
|
+
|
|
162
|
+
You describe each document the way its own service would, and a per-source JSON Schema says what
|
|
163
|
+
that record may carry. `title` and `content` are served verbatim; so is every other field you set —
|
|
164
|
+
authors, timestamps, threads, comments, labels, states, ACLs — so no part of a response *has* to be
|
|
165
|
+
synthesized. What you leave out is filled in **deterministically**, each value hashed from the
|
|
166
|
+
stable key it belongs to (a document's `doc_id`, a container's name, an author's address), so ids
|
|
167
|
+
never move between calls or pages.
|
|
168
|
+
|
|
169
|
+
### Bring your own corpus
|
|
170
|
+
|
|
171
|
+
One JSONL document per line, validated against a per-service JSON Schema
|
|
172
|
+
([`backlot/schemas/`](backlot/schemas/)), then loaded:
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
backlot import mycorpus.jsonl # validate + load -> data/
|
|
176
|
+
backlot import mycorpus.jsonl --dry-run # validate only, no DB writes
|
|
177
|
+
backlot import mycorpus.jsonl --roster roster.yaml # state the principals, don't derive them
|
|
178
|
+
backlot import corpus.jsonl.gz # gzipped, read as a stream
|
|
179
|
+
backlot import artifact-dir/ # a sharded corpus + its manifest, digests verified
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
```json
|
|
183
|
+
{"source_type": "slack", "channel": "incidents", "author_email": "bob@acme.com", "content": "Anyone seeing 502s from the gateway?", "replies": [{"content": "Looking now.", "author_email": "ava@acme.com"}]}
|
|
184
|
+
{"source_type": "gmail", "mailbox": "ceo", "title": "Q1 board deck draft", "content": "Draft narrative for the Q1 board meeting.", "author_email": "ceo@acme.com", "to": "ava@acme.com", "readers": ["ceo@acme.com", "ava@acme.com"]}
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Only `source_type` and `content` are required (`title` too, for every source except Slack). Where
|
|
188
|
+
to look next:
|
|
189
|
+
|
|
190
|
+
| To learn | Read |
|
|
191
|
+
|---|---|
|
|
192
|
+
| every field a record may carry | [`examples/bring-your-own-corpus/sample_corpus.jsonl`](examples/bring-your-own-corpus/sample_corpus.jsonl) — the field reference, and a test keeps it exhaustive |
|
|
193
|
+
| the rules each source imposes | [`backlot/schemas/README.md`](backlot/schemas/README.md) |
|
|
194
|
+
| import → serve → query, runnable | [`examples/bring-your-own-corpus/run.py`](examples/bring-your-own-corpus/run.py) |
|
|
195
|
+
|
|
196
|
+
The schemas double as the contract for **LLM dataset generation**: hand one to a model as a
|
|
197
|
+
structured-output schema, generate records, then `--dry-run` before loading. See
|
|
198
|
+
[`backlot/schemas/README.md`](backlot/schemas/README.md).
|
|
199
|
+
|
|
200
|
+
### Load a public dataset
|
|
201
|
+
|
|
202
|
+
[EnterpriseRAG-Bench](https://github.com/onyx-dot-app/EnterpriseRAG-Bench) is ~500k synthetic
|
|
203
|
+
enterprise documents across nine of the supported sources. One command downloads, loads and
|
|
204
|
+
ACL-derives it:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
backlot import --type enterpriserag-bench # -t erb for short
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
What that dataset does and does not carry, and how to redistribute it as BYO-JSONL, is in
|
|
211
|
+
[`examples/import-enterpriserag-bench/`](examples/import-enterpriserag-bench/) — it is one corpus
|
|
212
|
+
you can load, not part of this server's contract.
|
|
213
|
+
|
|
214
|
+
## Auth & tokens
|
|
215
|
+
|
|
216
|
+
Each service authenticates its own way, and the mock expects what the real one does: a bearer token
|
|
217
|
+
for most, HTTP Basic for Jira/Confluence, a **bare** `Authorization` value for Linear's personal API
|
|
218
|
+
keys, SigV4 for S3, and Google's OAuth token exchange for a connector carrying a client config.
|
|
219
|
+
|
|
220
|
+
You construct none of it. Two mock-only endpoints hand out every credential the corpus generated —
|
|
221
|
+
an admin/service token that bypasses the ACL (use it to crawl), plus one identity per person, each
|
|
222
|
+
seeing only what their own ACL permits:
|
|
223
|
+
|
|
224
|
+
```bash
|
|
225
|
+
curl -s localhost:8000/_mock/users
|
|
226
|
+
```
|
|
227
|
+
```json
|
|
228
|
+
{ "org": "acme", "admin_token": "admin-service-token", "count": 10,
|
|
229
|
+
"admin_s3_access_key_id": "AKIA732S…", "admin_s3_secret_access_key": "l4sz5sXT…",
|
|
230
|
+
"users": [{ "email": "ava.chen@acme.com", "name": "Ava Chen", "token": "usr-29b84da570…",
|
|
231
|
+
"s3_access_key_id": "AKIADNLO…", "s3_secret_access_key": "0FdhlUUQ…",
|
|
232
|
+
"groups": ["engineering", "handbook", "product", "design"] }] }
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
curl -s localhost:8000/_mock/credentials # for a Google client that wants a config, not a token
|
|
237
|
+
```
|
|
238
|
+
```json
|
|
239
|
+
{ "org": "acme", "token_uri": "http://localhost:8000/oauth2/token",
|
|
240
|
+
"oauth_client": { "client_id": "e8ae7a….apps.googleusercontent.com", "client_secret": "GOCSPX-…" },
|
|
241
|
+
"service_account": { "type": "service_account", "client_email": "…", "private_key": "…" } }
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
Use a user's `token` and every API filters to that user, which is what makes per-user access a test
|
|
245
|
+
rather than an audit. `token_uri` points back at the mock, so a client library's own refresh lands
|
|
246
|
+
here and resolves to the same ACL — a user's refresh_token is just their bearer token. Both
|
|
247
|
+
endpoints serve credentials in the clear, so `BACKLOT_EXPOSE_TOKENS=false` closes them; the same
|
|
248
|
+
values are in `data/tokens.yaml`. Per-service detail:
|
|
249
|
+
[`examples/using-official-sdk/`](examples/using-official-sdk/).
|
|
250
|
+
|
|
251
|
+
## Example Usages
|
|
252
|
+
|
|
253
|
+
Every one of these points a real client at the mock's base URL — that is the only change from
|
|
254
|
+
talking to the live service.
|
|
255
|
+
|
|
256
|
+
### Official SDKs
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
from slack_sdk import WebClient
|
|
260
|
+
WebClient(token=TOKEN, base_url="http://localhost:8000/slack/api/")
|
|
261
|
+
|
|
262
|
+
from github import Github, Auth
|
|
263
|
+
Github(auth=Auth.Token(TOKEN), base_url="http://localhost:8000/github")
|
|
264
|
+
|
|
265
|
+
from atlassian import Jira, Confluence
|
|
266
|
+
Jira(url="http://localhost:8000/atlassian", username="svc@x", password=TOKEN)
|
|
267
|
+
Confluence(url="http://localhost:8000/atlassian/wiki", username="svc@x", password=TOKEN)
|
|
268
|
+
|
|
269
|
+
from googleapiclient.discovery import build
|
|
270
|
+
from google.api_core.client_options import ClientOptions
|
|
271
|
+
from google.oauth2.credentials import Credentials
|
|
272
|
+
creds = Credentials(token=TOKEN)
|
|
273
|
+
build("gmail", "v1", credentials=creds, client_options=ClientOptions(api_endpoint="http://localhost:8000"))
|
|
274
|
+
build("drive", "v3", credentials=creds, client_options=ClientOptions(api_endpoint="http://localhost:8000/drive/v3"))
|
|
275
|
+
|
|
276
|
+
from notion_client import Client
|
|
277
|
+
Client(auth=TOKEN, base_url="http://localhost:8000/notion") # SDK appends /v1/ itself
|
|
278
|
+
|
|
279
|
+
import boto3
|
|
280
|
+
from botocore.config import Config
|
|
281
|
+
boto3.client("s3", endpoint_url="http://localhost:8000/s3", aws_access_key_id=AK, aws_secret_access_key=SK,
|
|
282
|
+
region_name="us-east-1", config=Config(s3={"addressing_style": "path"}))
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
A runnable, self-contained script per service is in [`examples/using-official-sdk/`](examples/using-official-sdk/).
|
|
286
|
+
|
|
287
|
+
### MCP
|
|
288
|
+
|
|
289
|
+
An MCP server pointed at the mock retrieves through it, ACL-scoped to whatever token it
|
|
290
|
+
authenticates with. Some vendors publish a server that takes a base URL — use it directly:
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
# examples/using-mcp-with-agents/atlassian.py — the community-official mcp-atlassian, over Docker.
|
|
294
|
+
# The host must end in .atlassian.net for the server's Cloud detection, so alias it at the mock.
|
|
295
|
+
params = StdioServerParameters(command="docker", args=[
|
|
296
|
+
"run", "-i", "--rm", "--add-host=mock.atlassian.net:host-gateway",
|
|
297
|
+
"-e", "JIRA_URL=http://mock.atlassian.net:8000/atlassian",
|
|
298
|
+
"-e", "CONFLUENCE_URL=http://mock.atlassian.net:8000/atlassian/wiki",
|
|
299
|
+
"-e", "JIRA_USERNAME=svc@example.com", "-e", "CONFLUENCE_USERNAME=svc@example.com",
|
|
300
|
+
"-e", f"JIRA_API_TOKEN={token}", "-e", f"CONFLUENCE_API_TOKEN={token}", # a user token -> its ACL
|
|
301
|
+
"-e", "MCP_ALLOWED_URL_DOMAINS=atlassian.net", "-e", "READ_ONLY_MODE=true",
|
|
302
|
+
"ghcr.io/sooperset/mcp-atlassian:latest", "--transport", "stdio",
|
|
303
|
+
])
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
For the ones that don't — GitHub, Slack, Gmail, Drive, HubSpot — a generic **OpenAPI→MCP bridge**
|
|
307
|
+
turns the mock's own typed `/openapi.json` into tools instead (`GET /_mock/openapi/<source>` serves
|
|
308
|
+
the per-source slice):
|
|
309
|
+
|
|
310
|
+
```python
|
|
311
|
+
# examples/using-mcp-with-agents/github.py — no vendor SDK, no vendor MCP server
|
|
312
|
+
params = StdioServerParameters(command=sys.executable, args=[
|
|
313
|
+
"examples/using-mcp-with-agents/_openapi_bridge.py",
|
|
314
|
+
"--source", "github", "--base-url", mock.base_url, "--token", mock.token,
|
|
315
|
+
])
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
Either way an agent then calls `session.list_tools()` and retrieves. Runnable agents for both LLM backends (Anthropic + OpenAI), one file per service, are in [`examples/using-mcp-with-agents/`](examples/using-mcp-with-agents/).
|
|
319
|
+
|
|
320
|
+
### LlamaIndex readers
|
|
321
|
+
|
|
322
|
+
Point official [LlamaIndex readers](https://docs.llamaindex.ai/en/stable/module_guides/loading/connector/)
|
|
323
|
+
(`llama-index-readers-*`) at the mock and load an enterprise corpus as `Document` objects — the
|
|
324
|
+
first step of a LlamaIndex ingestion/RAG pipeline.
|
|
325
|
+
|
|
326
|
+
```python
|
|
327
|
+
from llama_index.readers.github import GitHubIssuesClient
|
|
328
|
+
GitHubIssuesClient(github_token=TOKEN, base_url="http://localhost:8000/github")
|
|
329
|
+
|
|
330
|
+
from llama_index.readers.confluence import ConfluenceReader
|
|
331
|
+
ConfluenceReader(base_url="http://localhost:8000/atlassian/wiki", cloud=False, api_token=TOKEN)
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
One runnable script per source is in [`examples/using-llamaindex-readers/`](examples/using-llamaindex-readers/).
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
### Mirage
|
|
338
|
+
|
|
339
|
+
[mirage](https://github.com/strukto-ai/mirage) mounts a SaaS backend as a **virtual
|
|
340
|
+
filesystem** an agent reads with shell commands (`ls`, `cat`, `grep`, `find`). Point its resources at the mock and you can drive a mirage agent over your corpus offline.
|
|
341
|
+
|
|
342
|
+
```python
|
|
343
|
+
from mirage import MountMode, Workspace
|
|
344
|
+
from mirage.resource.slack import SlackConfig, SlackResource
|
|
345
|
+
|
|
346
|
+
resource = SlackResource(SlackConfig(token=TOKEN, base_url="http://localhost:8000/slack/api"))
|
|
347
|
+
ws = Workspace({"/slack": resource}, mode=MountMode.READ)
|
|
348
|
+
await ws.execute("ls /slack/channels/") # then cat a channel's dated chat.jsonl
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
One runnable script per source plus a `unified.py` that greps
|
|
352
|
+
across Slack/Gmail/Google Drive at once are in [`examples/using-mirage/`](examples/using-mirage/).
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
## Endpoints (read-only)
|
|
356
|
+
|
|
357
|
+
| Prefix | Service | Endpoints |
|
|
358
|
+
|---|---|---|
|
|
359
|
+
| `/slack/api` | Slack | `conversations.list` (+`types`; this corpus has no DMs, so `im`/`mpim` select nothing, and an unknown value is `invalid_types`), `conversations.history` (+`oldest`/`latest`/`inclusive`), `conversations.replies`, `conversations.members` (per-channel, paginated), `users.list`, `users.info`, `auth.test`, `api.test` (auth-free connectivity check), `search.messages` |
|
|
360
|
+
| `/gmail/v1` | Gmail | `users/{u}/messages` (+`q`: free text / `from:` `to:` `subject:` `after:` `before:` `newer_than:` `older_than:` `label:` `has:attachment`), `messages/{id}` (`format=full\|metadata\|minimal`), `messages/{id}/attachments/{id}`, `threads` (+`q`), `threads/{id}`, `labels`, `profile`. Message and thread ids are Gmail-shaped — 16 lowercase hex under 2^63, sharing one id space as the real API does — and map back to the corpus document; an id the real API could not parse is refused the same way |
|
|
361
|
+
| `/drive/v3` | Drive | `files` (`q`: `fullText contains`, `name contains`, `mimeType`, `… in parents` incl. `'root'`, `trashed`, `modifiedTime`, `sharedWithMe`, `… in owners`; `orderBy`: `name`/`name_natural`/`createdTime`/`modifiedTime`/`recency`/`folder`/`starred`/`quotaBytesUsed`/`sharedWithMeTime` (+` desc`); `fields` projection, validated), `files/{id}` (+`fields`), `files/{id}/export`, `files/{id}/permissions`, `drives`, `about` (`fields` **required**, as in real Drive; `storageQuota` is measured from the caller's visible corpus). Folders are files here: they match `mimeType='…folder'`, project, sort and resolve permissions like stored rows. Trashed files are excluded unless `trashed = true` asks for them |
|
|
362
|
+
| `/docs/v1`, `/sheets/v4`, `/slides/v1` | Docs/Sheets/Slides | `documents/{id}`, `spreadsheets/{id}`, `presentations/{id}` — native-doc content for editor-aware clients (read structurally instead of via Drive export). `spreadsheets/{id}` returns structure only — cells need `includeGridData=true` (+ optional `ranges`), as in real Sheets. Sheets also serves `spreadsheets/{id}/values/{range}` and `spreadsheets/{id}/values:batchGet` (A1 ranges incl. `Sheet1!A1:B2`, `A:A`, `1:3`, `A2:B`, a bare sheet name quoted or not; `majorDimension`, `valueRenderOption`). A spreadsheet row is one stored **line**, held in a single cell verbatim — the mock picks no column delimiter, so splitting (CSV, pipes, …) stays the corpus owner's decision. Reading a file of the wrong type through any of the three APIs is refused, as real Google does, not reinterpreted |
|
|
363
|
+
| `/github` | GitHub | `search/issues` (`q`: free text + `repo:` `is:` `state:` `type:` `label:` `author:`), `orgs/{org}`, `orgs/{org}/repos`, `repos/{o}/{r}`, `.../issues[/{n}]`, `.../issues/{n}/comments`, `.../pulls[/{n}]`, `.../pulls/{n}/reviews`, `.../readme`, `.../contents[/{path}]`, `.../git/trees/{ref}`, `.../git/blobs/{sha}`, `.../branches/{branch}`, `.../commits/{sha}`, `.../collaborators`, `.../teams`, `orgs/{org}/teams` |
|
|
364
|
+
| `/atlassian/rest/api/3` | Jira | `search/jql` (JQL `project =`, `text\|summary\|description ~`), `issue/{key}`, `issue/{key}/comment`, `field`, `issueLinkType`, `project/search`, `project/{key}/role[/{id}]`, `serverInfo` (also under `rest/api/2`) |
|
|
365
|
+
| `/atlassian/wiki/rest/api` | Confluence | `content`, `content/{id}`, `content/{id}/child/comment`, `content/{id}/restriction/byOperation`, `search` (CQL), `space`, `space/{key}`, `space/{key}/permission` |
|
|
366
|
+
| `/notion/v1` | Notion | `search`, `pages/{id}`, `blocks/{id}`, `blocks/{id}/children`, `databases/{id}` (version-aware), `data_sources/{id}`, `data_sources/{id}/query`, `databases/{id}/query` (legacy), `users[/{id}]`, `users/me`, `comments` |
|
|
367
|
+
| `/hubspot/crm/v3`, `/hubspot/crm/v4` | HubSpot | `objects/{objectType}` (+`limit` max 100, `after`, `properties`, `archived`), `objects/{objectType}/{id}`, `objects/{objectType}/search` (`filterGroups` OR-ed, `filters` AND-ed, 13 operators over any property), `objects/{objectType}/batch/read`, `v4/objects/{type}/{id}/associations/{toType}` |
|
|
368
|
+
| `/s3` | Amazon S3 | `ListBuckets`, `HeadBucket`, `GetBucketLocation`, `ListObjectsV2` (`prefix`/`delimiter`/`continuation-token`), `GetObject` (+`Range`), `HeadObject` |
|
|
369
|
+
| `/linear/graphql` | Linear | **GraphQL only** (one `POST`): `issues`, `issue(id:)` (UUID *or* `ENG-123`), `team(id:)` (UUID, key, or name), `teams`, `comments`, `users`, `viewer`, plus the `Team.issues` / `Issue.{comments,labels,children,relations,inverseRelations,attachments,releases}` connections and the by-id roots (`user`, `workflowState`, `project`, `issueLabel`, `cycle`, `release`, `attachment`, `issueRelation`) the official SDK's lazy relation accessors call. Relay pagination (`first`/`after`, `last`/`before` → `{nodes, pageInfo}`), server-side `filter` compiled into SQL, and full introspection |
|
|
370
|
+
| `/fireflies/graphql` | Fireflies | **GraphQL only** (one `POST`): `transcripts`, `transcript(id:)`, `user[(id:)]`, `users`. Offset pagination — `limit` (**max 50**, clamped) / `skip`, returning a **bare list**, not a Relay connection — plus the documented filters: `keyword` × `scope` (`title`\|`sentences`\|`all`), `fromDate`/`toDate`, `host_email`, `organizers`, `participants`, `user_id`, `mine`, `channel_id`. Field names are snake_case, as Fireflies' own schema has them. Full introspection |
|
|
371
|
+
|
|
372
|
+
Mock-only endpoints: `/health`, `/_mock/users`, `/_mock/credentials`, `/_mock/openapi/<source>`,
|
|
373
|
+
`/openapi.json`.
|
|
374
|
+
|
|
375
|
+
## Tests
|
|
376
|
+
|
|
377
|
+
```bash
|
|
378
|
+
pytest # unit (synth/pagination/acl/schema/importer parsers) + HTTP endpoint tests
|
|
379
|
+
# (full-crawl completeness, content round-trip, ACL enforcement)
|
|
380
|
+
ruff check . && ruff format --check .
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
## Configuration
|
|
384
|
+
|
|
385
|
+
Every setting is an env var with a `BACKLOT_` prefix, and a `.env` file in the working directory is
|
|
386
|
+
read too. Defaults are what the server uses when the var is unset.
|
|
387
|
+
|
|
388
|
+
| Env var | Default | What it does |
|
|
389
|
+
|---|---|---|
|
|
390
|
+
| `BACKLOT_DATA_DIR` | `./data` (resolved against the cwd, **not** the install location) | Where the corpus lives: `mock.sqlite`, `tokens.yaml`, `credentials.yaml`. Both `backlot import` and `backlot serve` read it, which is how you keep several corpora side by side — `BACKLOT_DATA_DIR=/tmp/demo backlot import c.jsonl` |
|
|
391
|
+
| `BACKLOT_ADMIN_TOKEN` | `admin-service-token` | The token that bypasses ACL filtering — a full-crawl / service identity. Set it to anything for a shared deployment |
|
|
392
|
+
| `BACKLOT_ENFORCE_ACL` | `true` | When `false`, any well-formed token is treated as admin. The ACL is still *exposed* through each vendor's permission endpoints, just not *enforced* — useful for isolating whether a connector's gaps are permissions or parsing |
|
|
393
|
+
| `BACKLOT_EXPOSE_TOKENS` | `true` | Serves `GET /_mock/users` and `GET /_mock/credentials`, which hand out every user's token in the clear. Fine for a local mock; set `false` to close both |
|
|
394
|
+
| `BACKLOT_ORG_NAME` | inferred from the corpus (fallback `example`) | The org slug that shows up in `auth.test`, synthesized emails and self-URLs. Inferred from the dominant author email domain — `@acme.com` documents serve as org `acme` — so set it only to override that |
|
|
395
|
+
| `BACKLOT_ORG_DOMAIN` | inferred from the corpus (fallback `example.com`) | The domain half of the same inference, e.g. `acme.com`. Used for addresses the corpus does not state |
|
|
396
|
+
| `BACKLOT_DEFAULT_PAGE_SIZE` | `100` | Page size when a request names none |
|
|
397
|
+
| `BACKLOT_MAX_PAGE_SIZE` | `1000` | Ceiling a request may ask for. Per-vendor caps still win where the real API has one (Fireflies clamps to 50, HubSpot to 100) |
|
|
398
|
+
| `BACKLOT_SQLITE_MMAP_MB` | `256` | Memory-maps the DB so reads come from the OS page cache instead of a syscall each — the main lever against a slow first request after idle. SQLite maps `min(this, db size)`; raise it to at or above your DB size to map a big corpus fully |
|
|
399
|
+
| `BACKLOT_SQLITE_CACHE_MB` | `64` | SQLite's own page cache, per serving connection |
|
|
400
|
+
| `BACKLOT_SQLITE_BUSY_MS` | `5000` | How long a read waits for a lock instead of erroring, so reads ride through an out-of-band write (e.g. an in-place FTS rebuild) rather than 500ing |
|
|
401
|
+
|
|
402
|
+
## Contributing
|
|
403
|
+
|
|
404
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). The short version: fidelity to the real APIs is the point,
|
|
405
|
+
so a divergence is a bug — measure against the real service, and bring a test that fails without
|
|
406
|
+
your fix.
|
|
407
|
+
|
|
408
|
+
## License
|
|
409
|
+
|
|
410
|
+
[MIT](LICENSE)
|