backlot 0.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. backlot-0.0.0/LICENSE +21 -0
  2. backlot-0.0.0/PKG-INFO +410 -0
  3. backlot-0.0.0/README.md +340 -0
  4. backlot-0.0.0/backlot/__init__.py +5 -0
  5. backlot-0.0.0/backlot/__main__.py +10 -0
  6. backlot-0.0.0/backlot/acl.py +89 -0
  7. backlot-0.0.0/backlot/auth.py +209 -0
  8. backlot-0.0.0/backlot/cli.py +164 -0
  9. backlot-0.0.0/backlot/config.py +130 -0
  10. backlot-0.0.0/backlot/data/hello.jsonl +136 -0
  11. backlot-0.0.0/backlot/errors/__init__.py +44 -0
  12. backlot-0.0.0/backlot/errors/atlassian.py +46 -0
  13. backlot-0.0.0/backlot/errors/google.py +230 -0
  14. backlot-0.0.0/backlot/graphql/__init__.py +8 -0
  15. backlot-0.0.0/backlot/graphql/engine.py +147 -0
  16. backlot-0.0.0/backlot/graphql/fireflies.graphql +240 -0
  17. backlot-0.0.0/backlot/graphql/fireflies_resolvers.py +357 -0
  18. backlot-0.0.0/backlot/graphql/linear.graphql +836 -0
  19. backlot-0.0.0/backlot/graphql/linear_filters.py +458 -0
  20. backlot-0.0.0/backlot/graphql/linear_resolvers.py +1215 -0
  21. backlot-0.0.0/backlot/importer/__init__.py +5 -0
  22. backlot-0.0.0/backlot/importer/byo.py +1440 -0
  23. backlot-0.0.0/backlot/importer/erb.py +2458 -0
  24. backlot-0.0.0/backlot/integrations/__init__.py +11 -0
  25. backlot-0.0.0/backlot/integrations/llamaindex.py +312 -0
  26. backlot-0.0.0/backlot/integrations/mirage.py +100 -0
  27. backlot-0.0.0/backlot/main.py +403 -0
  28. backlot-0.0.0/backlot/oauth.py +160 -0
  29. backlot-0.0.0/backlot/openapi.py +137 -0
  30. backlot-0.0.0/backlot/pagination.py +116 -0
  31. backlot-0.0.0/backlot/routers/__init__.py +25 -0
  32. backlot-0.0.0/backlot/routers/atlassian.py +1154 -0
  33. backlot-0.0.0/backlot/routers/fireflies.py +69 -0
  34. backlot-0.0.0/backlot/routers/github.py +901 -0
  35. backlot-0.0.0/backlot/routers/google.py +2144 -0
  36. backlot-0.0.0/backlot/routers/hubspot.py +622 -0
  37. backlot-0.0.0/backlot/routers/linear.py +79 -0
  38. backlot-0.0.0/backlot/routers/notion.py +541 -0
  39. backlot-0.0.0/backlot/routers/oauth.py +65 -0
  40. backlot-0.0.0/backlot/routers/s3.py +376 -0
  41. backlot-0.0.0/backlot/routers/slack.py +862 -0
  42. backlot-0.0.0/backlot/schemas/README.md +217 -0
  43. backlot-0.0.0/backlot/schemas/confluence.schema.json +188 -0
  44. backlot-0.0.0/backlot/schemas/fireflies.schema.json +304 -0
  45. backlot-0.0.0/backlot/schemas/github.schema.json +255 -0
  46. backlot-0.0.0/backlot/schemas/gmail.schema.json +222 -0
  47. backlot-0.0.0/backlot/schemas/google_drive.schema.json +138 -0
  48. backlot-0.0.0/backlot/schemas/hubspot.schema.json +140 -0
  49. backlot-0.0.0/backlot/schemas/jira.schema.json +231 -0
  50. backlot-0.0.0/backlot/schemas/linear.schema.json +298 -0
  51. backlot-0.0.0/backlot/schemas/notion.schema.json +173 -0
  52. backlot-0.0.0/backlot/schemas/s3.schema.json +118 -0
  53. backlot-0.0.0/backlot/schemas/slack.schema.json +161 -0
  54. backlot-0.0.0/backlot/sigv4.py +121 -0
  55. backlot-0.0.0/backlot/store.py +1871 -0
  56. backlot-0.0.0/backlot/synth.py +823 -0
  57. backlot-0.0.0/backlot/testing.py +276 -0
  58. backlot-0.0.0/backlot/validation.py +91 -0
  59. backlot-0.0.0/backlot.egg-info/PKG-INFO +410 -0
  60. backlot-0.0.0/backlot.egg-info/SOURCES.txt +93 -0
  61. backlot-0.0.0/backlot.egg-info/dependency_links.txt +1 -0
  62. backlot-0.0.0/backlot.egg-info/entry_points.txt +2 -0
  63. backlot-0.0.0/backlot.egg-info/requires.txt +54 -0
  64. backlot-0.0.0/backlot.egg-info/top_level.txt +1 -0
  65. backlot-0.0.0/pyproject.toml +134 -0
  66. backlot-0.0.0/setup.cfg +4 -0
  67. backlot-0.0.0/tests/test_acl.py +393 -0
  68. backlot-0.0.0/tests/test_atlassian.py +364 -0
  69. backlot-0.0.0/tests/test_auth.py +131 -0
  70. backlot-0.0.0/tests/test_cli.py +229 -0
  71. backlot-0.0.0/tests/test_config.py +23 -0
  72. backlot-0.0.0/tests/test_cross_vendor.py +278 -0
  73. backlot-0.0.0/tests/test_fireflies.py +506 -0
  74. backlot-0.0.0/tests/test_github.py +542 -0
  75. backlot-0.0.0/tests/test_google.py +2140 -0
  76. backlot-0.0.0/tests/test_graphql.py +395 -0
  77. backlot-0.0.0/tests/test_hubspot.py +547 -0
  78. backlot-0.0.0/tests/test_importer_byo.py +1966 -0
  79. backlot-0.0.0/tests/test_importer_erb.py +2866 -0
  80. backlot-0.0.0/tests/test_integrations.py +270 -0
  81. backlot-0.0.0/tests/test_linear.py +891 -0
  82. backlot-0.0.0/tests/test_llamaindex.py +385 -0
  83. backlot-0.0.0/tests/test_mcp.py +468 -0
  84. backlot-0.0.0/tests/test_notion.py +237 -0
  85. backlot-0.0.0/tests/test_openapi.py +197 -0
  86. backlot-0.0.0/tests/test_packaging.py +56 -0
  87. backlot-0.0.0/tests/test_pagination.py +239 -0
  88. backlot-0.0.0/tests/test_s3.py +661 -0
  89. backlot-0.0.0/tests/test_schema.py +640 -0
  90. backlot-0.0.0/tests/test_sdk.py +606 -0
  91. backlot-0.0.0/tests/test_search.py +326 -0
  92. backlot-0.0.0/tests/test_slack.py +348 -0
  93. backlot-0.0.0/tests/test_store.py +857 -0
  94. backlot-0.0.0/tests/test_synth.py +222 -0
  95. backlot-0.0.0/tests/test_testing.py +136 -0
backlot-0.0.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 BrekkyLab
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
backlot-0.0.0/PKG-INFO ADDED
@@ -0,0 +1,410 @@
1
+ Metadata-Version: 2.4
2
+ Name: backlot
3
+ Version: 0.0.0
4
+ Summary: A read-only mock of enterprise SaaS knowledge APIs, served over your own corpus with per-document ACLs
5
+ Author-email: khj809 <onsealeatang@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/brekkylab/backlot
8
+ Project-URL: Repository, https://github.com/brekkylab/backlot
9
+ Project-URL: Issues, https://github.com/brekkylab/backlot/issues
10
+ Keywords: mock,rag,slack,gmail,google-drive,jira,confluence,github,notion,s3,amazon-s3,hubspot,linear,graphql,benchmark
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Framework :: FastAPI
16
+ Classifier: Topic :: Software Development :: Testing :: Mocking
17
+ Requires-Python: >=3.11
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Requires-Dist: fastapi>=0.110
21
+ Requires-Dist: uvicorn[standard]>=0.29
22
+ Requires-Dist: pydantic>=2.6
23
+ Requires-Dist: pydantic-settings>=2.2
24
+ Requires-Dist: pyyaml>=6.0
25
+ Requires-Dist: python-multipart>=0.0.9
26
+ Requires-Dist: jsonschema>=4.20
27
+ Requires-Dist: pyjwt[crypto]>=2.8
28
+ Requires-Dist: httpx>=0.27
29
+ Requires-Dist: graphql-core>=3.2
30
+ Provides-Extra: dev
31
+ Requires-Dist: pytest>=8.0; extra == "dev"
32
+ Requires-Dist: httpx>=0.27; extra == "dev"
33
+ Requires-Dist: ruff~=0.15.22; extra == "dev"
34
+ Provides-Extra: examples
35
+ Requires-Dist: httpx>=0.27; extra == "examples"
36
+ Requires-Dist: slack_sdk>=3.27; extra == "examples"
37
+ Requires-Dist: PyGithub>=2.3; extra == "examples"
38
+ Requires-Dist: atlassian-python-api>=3.41; extra == "examples"
39
+ Requires-Dist: google-api-python-client>=2.120; extra == "examples"
40
+ Requires-Dist: google-auth>=2.29; extra == "examples"
41
+ Requires-Dist: notion-client>=3.1; extra == "examples"
42
+ Requires-Dist: boto3>=1.34; extra == "examples"
43
+ Requires-Dist: hubspot-api-client>=12; extra == "examples"
44
+ Provides-Extra: mcp
45
+ Requires-Dist: mcp>=1.2; extra == "mcp"
46
+ Requires-Dist: anthropic[mcp]>=0.40; extra == "mcp"
47
+ Requires-Dist: openai>=1.40; extra == "mcp"
48
+ Requires-Dist: openai-agents>=0.1; extra == "mcp"
49
+ Requires-Dist: pyyaml>=6.0; extra == "mcp"
50
+ Requires-Dist: fastmcp>=2.0; extra == "mcp"
51
+ Provides-Extra: llamaindex
52
+ Requires-Dist: llama-index-readers-slack>=0.5; extra == "llamaindex"
53
+ Requires-Dist: llama-index-readers-google>=0.7; extra == "llamaindex"
54
+ Requires-Dist: llama-index-readers-github>=0.11; extra == "llamaindex"
55
+ Requires-Dist: llama-index-readers-jira>=0.6; extra == "llamaindex"
56
+ Requires-Dist: llama-index-readers-confluence>=0.7; extra == "llamaindex"
57
+ Requires-Dist: llama-index-readers-notion>=0.5; extra == "llamaindex"
58
+ Requires-Dist: llama-index-readers-s3>=0.6; extra == "llamaindex"
59
+ Requires-Dist: llama-index-readers-linear>=0.5; extra == "llamaindex"
60
+ Requires-Dist: jira>=3.10; extra == "llamaindex"
61
+ Requires-Dist: atlassian-python-api>=4.0; extra == "llamaindex"
62
+ Requires-Dist: slack_sdk>=3.43; extra == "llamaindex"
63
+ Requires-Dist: google-api-python-client>=2.198; extra == "llamaindex"
64
+ Requires-Dist: google-auth>=2.55; extra == "llamaindex"
65
+ Requires-Dist: boto3>=1.40; extra == "llamaindex"
66
+ Requires-Dist: s3fs>=2026.6; extra == "llamaindex"
67
+ Provides-Extra: mirage
68
+ Requires-Dist: mirage-ai[fuse]~=0.0.4; extra == "mirage"
69
+ Dynamic: license-file
70
+
71
+ # Backlot
72
+
73
+ [![tests](https://github.com/brekkylab/backlot/actions/workflows/ci.yml/badge.svg)](https://github.com/brekkylab/backlot/actions/workflows/ci.yml)
74
+ ![python](https://img.shields.io/badge/python-3.11%2B-blue)
75
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
76
+
77
+ A **read-only** mock server that stands in for a whole stack of enterprise SaaS knowledge sources
78
+ at once.
79
+ It speaks each service's real read API — the exact response shapes, pagination schemes, auth,
80
+ and native permission endpoints their official SDKs expect — over a corpus **you** supply, so a
81
+ RAG/search connector built on those SDKs can be exercised **end-to-end** without the live
82
+ services.
83
+
84
+ ## Quickstart
85
+
86
+ ```bash
87
+ pip install backlot
88
+ ```
89
+
90
+ That puts the `backlot` command on PATH. A server needs a corpus, and one is bundled with the
91
+ package — 136 documents covering every source it serves — so there is nothing to fetch or write:
92
+
93
+ ```bash
94
+ backlot import --bundled # the bundled corpus -> data/mock.sqlite + data/tokens.yaml
95
+ backlot serve # http://127.0.0.1:8000
96
+ curl -s localhost:8000/health
97
+ ```
98
+
99
+ Or skip the CLI entirely and let a test spin one up on a free port, serving that same corpus
100
+ (`pip install "backlot[examples]"` for the vendor SDKs the first snippet uses):
101
+
102
+ ```python
103
+ import backlot
104
+ from slack_sdk import WebClient
105
+
106
+ with backlot.mock_server() as m: # no arguments: the bundled corpus
107
+ slack = WebClient(token=m.token, base_url=f"{m.base_url}/slack/api/")
108
+ print(slack.conversations_list()["channels"])
109
+
110
+ with backlot.mock_server(records=[ # or your own records, inline
111
+ {"source_type": "confluence", "space": "handbook", "title": "On-call",
112
+ "content": "Page for Sev1 and Sev2 only.", "author_email": "ava@acme.com"},
113
+ ]) as m:
114
+ ...
115
+ ```
116
+
117
+ `python -m backlot` is the same CLI as the `backlot` script, for when the venv is not activated.
118
+ Working on Backlot itself instead of with it? [CONTRIBUTING.md](CONTRIBUTING.md) covers the
119
+ from-source install.
120
+
121
+ ### Docker
122
+
123
+ ```bash
124
+ docker build -t backlot . # server + the bundled corpus baked in; no download
125
+ docker run -p 8000:8000 backlot
126
+ curl -s localhost:8000/health
127
+ ```
128
+
129
+ That image answers every endpoint of every source out of the box. Use
130
+ `--target serve` for a server with **no** corpus, for a deployment that mounts its own
131
+ `/app/data`.
132
+
133
+ ## Why you need this
134
+
135
+ Connectors are easy to write and hard to trust. Proving one works means an account with every
136
+ vendor, an OAuth app per vendor, seeded data in each, and rate limits between you and every retry —
137
+ so most of it
138
+ gets tested against fixtures that agree with your assumptions instead of with the API. Backlot is
139
+ the API: same shapes, same pagination, same permission endpoints, over documents you control.
140
+
141
+ It is the right tool when you are:
142
+
143
+ - **building or upgrading a connector** — crawl a source to exhaustion, then diff what you got
144
+ against what you loaded, on a corpus small enough to reason about
145
+ - **testing ACL-scoped retrieval** — every document carries its own readers, and each user's token
146
+ sees only theirs, so "does this leak" is a test rather than an audit
147
+ - **running that suite in CI** — no accounts, no secrets, no network, no flakes from someone else's
148
+ outage; a server starts in a second and dies with the test
149
+ - **evaluating RAG or agents** — point an SDK, an MCP server, or a LlamaIndex reader at it and get
150
+ the same answers on every run, because ids and timestamps are derived, not random
151
+ - **reproducing a bug you cannot reach** — a paginated edge case, an odd MIME type, an empty thread:
152
+ write the document that causes it and serve it in seconds
153
+
154
+ It is **not** a sandbox for writes, a rate-limit or latency simulator, or a source of realistic
155
+ *content* — the documents are yours.
156
+
157
+ ## Preparing a corpus
158
+
159
+ The server reads a corpus from `data/` (`mock.sqlite` + `tokens.yaml`). Build it from your own
160
+ documents, or load a public dataset.
161
+
162
+ You describe each document the way its own service would, and a per-source JSON Schema says what
163
+ that record may carry. `title` and `content` are served verbatim; so is every other field you set —
164
+ authors, timestamps, threads, comments, labels, states, ACLs — so no part of a response *has* to be
165
+ synthesized. What you leave out is filled in **deterministically**, each value hashed from the
166
+ stable key it belongs to (a document's `doc_id`, a container's name, an author's address), so ids
167
+ never move between calls or pages.
168
+
169
+ ### Bring your own corpus
170
+
171
+ One JSONL document per line, validated against a per-service JSON Schema
172
+ ([`backlot/schemas/`](backlot/schemas/)), then loaded:
173
+
174
+ ```bash
175
+ backlot import mycorpus.jsonl # validate + load -> data/
176
+ backlot import mycorpus.jsonl --dry-run # validate only, no DB writes
177
+ backlot import mycorpus.jsonl --roster roster.yaml # state the principals, don't derive them
178
+ backlot import corpus.jsonl.gz # gzipped, read as a stream
179
+ backlot import artifact-dir/ # a sharded corpus + its manifest, digests verified
180
+ ```
181
+
182
+ ```json
183
+ {"source_type": "slack", "channel": "incidents", "author_email": "bob@acme.com", "content": "Anyone seeing 502s from the gateway?", "replies": [{"content": "Looking now.", "author_email": "ava@acme.com"}]}
184
+ {"source_type": "gmail", "mailbox": "ceo", "title": "Q1 board deck draft", "content": "Draft narrative for the Q1 board meeting.", "author_email": "ceo@acme.com", "to": "ava@acme.com", "readers": ["ceo@acme.com", "ava@acme.com"]}
185
+ ```
186
+
187
+ Only `source_type` and `content` are required (`title` too, for every source except Slack). Where
188
+ to look next:
189
+
190
+ | To learn | Read |
191
+ |---|---|
192
+ | every field a record may carry | [`examples/bring-your-own-corpus/sample_corpus.jsonl`](examples/bring-your-own-corpus/sample_corpus.jsonl) — the field reference, and a test keeps it exhaustive |
193
+ | the rules each source imposes | [`backlot/schemas/README.md`](backlot/schemas/README.md) |
194
+ | import → serve → query, runnable | [`examples/bring-your-own-corpus/run.py`](examples/bring-your-own-corpus/run.py) |
195
+
196
+ The schemas double as the contract for **LLM dataset generation**: hand one to a model as a
197
+ structured-output schema, generate records, then `--dry-run` before loading. See
198
+ [`backlot/schemas/README.md`](backlot/schemas/README.md).
199
+
200
+ ### Load a public dataset
201
+
202
+ [EnterpriseRAG-Bench](https://github.com/onyx-dot-app/EnterpriseRAG-Bench) is ~500k synthetic
203
+ enterprise documents across nine of the supported sources. One command downloads, loads and
204
+ ACL-derives it:
205
+
206
+ ```bash
207
+ backlot import --type enterpriserag-bench # -t erb for short
208
+ ```
209
+
210
+ What that dataset does and does not carry, and how to redistribute it as BYO-JSONL, is in
211
+ [`examples/import-enterpriserag-bench/`](examples/import-enterpriserag-bench/) — it is one corpus
212
+ you can load, not part of this server's contract.
213
+
214
+ ## Auth & tokens
215
+
216
+ Each service authenticates its own way, and the mock expects what the real one does: a bearer token
217
+ for most, HTTP Basic for Jira/Confluence, a **bare** `Authorization` value for Linear's personal API
218
+ keys, SigV4 for S3, and Google's OAuth token exchange for a connector carrying a client config.
219
+
220
+ You construct none of it. Two mock-only endpoints hand out every credential the corpus generated —
221
+ an admin/service token that bypasses the ACL (use it to crawl), plus one identity per person, each
222
+ seeing only what their own ACL permits:
223
+
224
+ ```bash
225
+ curl -s localhost:8000/_mock/users
226
+ ```
227
+ ```json
228
+ { "org": "acme", "admin_token": "admin-service-token", "count": 10,
229
+ "admin_s3_access_key_id": "AKIA732S…", "admin_s3_secret_access_key": "l4sz5sXT…",
230
+ "users": [{ "email": "ava.chen@acme.com", "name": "Ava Chen", "token": "usr-29b84da570…",
231
+ "s3_access_key_id": "AKIADNLO…", "s3_secret_access_key": "0FdhlUUQ…",
232
+ "groups": ["engineering", "handbook", "product", "design"] }] }
233
+ ```
234
+
235
+ ```bash
236
+ curl -s localhost:8000/_mock/credentials # for a Google client that wants a config, not a token
237
+ ```
238
+ ```json
239
+ { "org": "acme", "token_uri": "http://localhost:8000/oauth2/token",
240
+ "oauth_client": { "client_id": "e8ae7a….apps.googleusercontent.com", "client_secret": "GOCSPX-…" },
241
+ "service_account": { "type": "service_account", "client_email": "…", "private_key": "…" } }
242
+ ```
243
+
244
+ Use a user's `token` and every API filters to that user, which is what makes per-user access a test
245
+ rather than an audit. `token_uri` points back at the mock, so a client library's own refresh lands
246
+ here and resolves to the same ACL — a user's refresh_token is just their bearer token. Both
247
+ endpoints serve credentials in the clear, so `BACKLOT_EXPOSE_TOKENS=false` closes them; the same
248
+ values are in `data/tokens.yaml`. Per-service detail:
249
+ [`examples/using-official-sdk/`](examples/using-official-sdk/).
250
+
251
+ ## Example Usages
252
+
253
+ Every one of these points a real client at the mock's base URL — that is the only change from
254
+ talking to the live service.
255
+
256
+ ### Official SDKs
257
+
258
+ ```python
259
+ from slack_sdk import WebClient
260
+ WebClient(token=TOKEN, base_url="http://localhost:8000/slack/api/")
261
+
262
+ from github import Github, Auth
263
+ Github(auth=Auth.Token(TOKEN), base_url="http://localhost:8000/github")
264
+
265
+ from atlassian import Jira, Confluence
266
+ Jira(url="http://localhost:8000/atlassian", username="svc@x", password=TOKEN)
267
+ Confluence(url="http://localhost:8000/atlassian/wiki", username="svc@x", password=TOKEN)
268
+
269
+ from googleapiclient.discovery import build
270
+ from google.api_core.client_options import ClientOptions
271
+ from google.oauth2.credentials import Credentials
272
+ creds = Credentials(token=TOKEN)
273
+ build("gmail", "v1", credentials=creds, client_options=ClientOptions(api_endpoint="http://localhost:8000"))
274
+ build("drive", "v3", credentials=creds, client_options=ClientOptions(api_endpoint="http://localhost:8000/drive/v3"))
275
+
276
+ from notion_client import Client
277
+ Client(auth=TOKEN, base_url="http://localhost:8000/notion") # SDK appends /v1/ itself
278
+
279
+ import boto3
280
+ from botocore.config import Config
281
+ boto3.client("s3", endpoint_url="http://localhost:8000/s3", aws_access_key_id=AK, aws_secret_access_key=SK,
282
+ region_name="us-east-1", config=Config(s3={"addressing_style": "path"}))
283
+ ```
284
+
285
+ A runnable, self-contained script per service is in [`examples/using-official-sdk/`](examples/using-official-sdk/).
286
+
287
+ ### MCP
288
+
289
+ An MCP server pointed at the mock retrieves through it, ACL-scoped to whatever token it
290
+ authenticates with. Some vendors publish a server that takes a base URL — use it directly:
291
+
292
+ ```python
293
+ # examples/using-mcp-with-agents/atlassian.py — the community-official mcp-atlassian, over Docker.
294
+ # The host must end in .atlassian.net for the server's Cloud detection, so alias it at the mock.
295
+ params = StdioServerParameters(command="docker", args=[
296
+ "run", "-i", "--rm", "--add-host=mock.atlassian.net:host-gateway",
297
+ "-e", "JIRA_URL=http://mock.atlassian.net:8000/atlassian",
298
+ "-e", "CONFLUENCE_URL=http://mock.atlassian.net:8000/atlassian/wiki",
299
+ "-e", "JIRA_USERNAME=svc@example.com", "-e", "CONFLUENCE_USERNAME=svc@example.com",
300
+ "-e", f"JIRA_API_TOKEN={token}", "-e", f"CONFLUENCE_API_TOKEN={token}", # a user token -> its ACL
301
+ "-e", "MCP_ALLOWED_URL_DOMAINS=atlassian.net", "-e", "READ_ONLY_MODE=true",
302
+ "ghcr.io/sooperset/mcp-atlassian:latest", "--transport", "stdio",
303
+ ])
304
+ ```
305
+
306
+ For the ones that don't — GitHub, Slack, Gmail, Drive, HubSpot — a generic **OpenAPI→MCP bridge**
307
+ turns the mock's own typed `/openapi.json` into tools instead (`GET /_mock/openapi/<source>` serves
308
+ the per-source slice):
309
+
310
+ ```python
311
+ # examples/using-mcp-with-agents/github.py — no vendor SDK, no vendor MCP server
312
+ params = StdioServerParameters(command=sys.executable, args=[
313
+ "examples/using-mcp-with-agents/_openapi_bridge.py",
314
+ "--source", "github", "--base-url", mock.base_url, "--token", mock.token,
315
+ ])
316
+ ```
317
+
318
+ Either way an agent then calls `session.list_tools()` and retrieves. Runnable agents for both LLM backends (Anthropic + OpenAI), one file per service, are in [`examples/using-mcp-with-agents/`](examples/using-mcp-with-agents/).
319
+
320
+ ### LlamaIndex readers
321
+
322
+ Point official [LlamaIndex readers](https://docs.llamaindex.ai/en/stable/module_guides/loading/connector/)
323
+ (`llama-index-readers-*`) at the mock and load an enterprise corpus as `Document` objects — the
324
+ first step of a LlamaIndex ingestion/RAG pipeline.
325
+
326
+ ```python
327
+ from llama_index.readers.github import GitHubIssuesClient
328
+ GitHubIssuesClient(github_token=TOKEN, base_url="http://localhost:8000/github")
329
+
330
+ from llama_index.readers.confluence import ConfluenceReader
331
+ ConfluenceReader(base_url="http://localhost:8000/atlassian/wiki", cloud=False, api_token=TOKEN)
332
+ ```
333
+
334
+ One runnable script per source is in [`examples/using-llamaindex-readers/`](examples/using-llamaindex-readers/).
335
+
336
+
337
+ ### Mirage
338
+
339
+ [mirage](https://github.com/strukto-ai/mirage) mounts a SaaS backend as a **virtual
340
+ filesystem** an agent reads with shell commands (`ls`, `cat`, `grep`, `find`). Point its resources at the mock and you can drive a mirage agent over your corpus offline.
341
+
342
+ ```python
343
+ from mirage import MountMode, Workspace
344
+ from mirage.resource.slack import SlackConfig, SlackResource
345
+
346
+ resource = SlackResource(SlackConfig(token=TOKEN, base_url="http://localhost:8000/slack/api"))
347
+ ws = Workspace({"/slack": resource}, mode=MountMode.READ)
348
+ await ws.execute("ls /slack/channels/") # then cat a channel's dated chat.jsonl
349
+ ```
350
+
351
+ One runnable script per source plus a `unified.py` that greps
352
+ across Slack/Gmail/Google Drive at once are in [`examples/using-mirage/`](examples/using-mirage/).
353
+
354
+
355
+ ## Endpoints (read-only)
356
+
357
+ | Prefix | Service | Endpoints |
358
+ |---|---|---|
359
+ | `/slack/api` | Slack | `conversations.list` (+`types`; this corpus has no DMs, so `im`/`mpim` select nothing, and an unknown value is `invalid_types`), `conversations.history` (+`oldest`/`latest`/`inclusive`), `conversations.replies`, `conversations.members` (per-channel, paginated), `users.list`, `users.info`, `auth.test`, `api.test` (auth-free connectivity check), `search.messages` |
360
+ | `/gmail/v1` | Gmail | `users/{u}/messages` (+`q`: free text / `from:` `to:` `subject:` `after:` `before:` `newer_than:` `older_than:` `label:` `has:attachment`), `messages/{id}` (`format=full\|metadata\|minimal`), `messages/{id}/attachments/{id}`, `threads` (+`q`), `threads/{id}`, `labels`, `profile`. Message and thread ids are Gmail-shaped — 16 lowercase hex under 2^63, sharing one id space as the real API does — and map back to the corpus document; an id the real API could not parse is refused the same way |
361
+ | `/drive/v3` | Drive | `files` (`q`: `fullText contains`, `name contains`, `mimeType`, `… in parents` incl. `'root'`, `trashed`, `modifiedTime`, `sharedWithMe`, `… in owners`; `orderBy`: `name`/`name_natural`/`createdTime`/`modifiedTime`/`recency`/`folder`/`starred`/`quotaBytesUsed`/`sharedWithMeTime` (+` desc`); `fields` projection, validated), `files/{id}` (+`fields`), `files/{id}/export`, `files/{id}/permissions`, `drives`, `about` (`fields` **required**, as in real Drive; `storageQuota` is measured from the caller's visible corpus). Folders are files here: they match `mimeType='…folder'`, project, sort and resolve permissions like stored rows. Trashed files are excluded unless `trashed = true` asks for them |
362
+ | `/docs/v1`, `/sheets/v4`, `/slides/v1` | Docs/Sheets/Slides | `documents/{id}`, `spreadsheets/{id}`, `presentations/{id}` — native-doc content for editor-aware clients (read structurally instead of via Drive export). `spreadsheets/{id}` returns structure only — cells need `includeGridData=true` (+ optional `ranges`), as in real Sheets. Sheets also serves `spreadsheets/{id}/values/{range}` and `spreadsheets/{id}/values:batchGet` (A1 ranges incl. `Sheet1!A1:B2`, `A:A`, `1:3`, `A2:B`, a bare sheet name quoted or not; `majorDimension`, `valueRenderOption`). A spreadsheet row is one stored **line**, held in a single cell verbatim — the mock picks no column delimiter, so splitting (CSV, pipes, …) stays the corpus owner's decision. Reading a file of the wrong type through any of the three APIs is refused, as real Google does, not reinterpreted |
363
+ | `/github` | GitHub | `search/issues` (`q`: free text + `repo:` `is:` `state:` `type:` `label:` `author:`), `orgs/{org}`, `orgs/{org}/repos`, `repos/{o}/{r}`, `.../issues[/{n}]`, `.../issues/{n}/comments`, `.../pulls[/{n}]`, `.../pulls/{n}/reviews`, `.../readme`, `.../contents[/{path}]`, `.../git/trees/{ref}`, `.../git/blobs/{sha}`, `.../branches/{branch}`, `.../commits/{sha}`, `.../collaborators`, `.../teams`, `orgs/{org}/teams` |
364
+ | `/atlassian/rest/api/3` | Jira | `search/jql` (JQL `project =`, `text\|summary\|description ~`), `issue/{key}`, `issue/{key}/comment`, `field`, `issueLinkType`, `project/search`, `project/{key}/role[/{id}]`, `serverInfo` (also under `rest/api/2`) |
365
+ | `/atlassian/wiki/rest/api` | Confluence | `content`, `content/{id}`, `content/{id}/child/comment`, `content/{id}/restriction/byOperation`, `search` (CQL), `space`, `space/{key}`, `space/{key}/permission` |
366
+ | `/notion/v1` | Notion | `search`, `pages/{id}`, `blocks/{id}`, `blocks/{id}/children`, `databases/{id}` (version-aware), `data_sources/{id}`, `data_sources/{id}/query`, `databases/{id}/query` (legacy), `users[/{id}]`, `users/me`, `comments` |
367
+ | `/hubspot/crm/v3`, `/hubspot/crm/v4` | HubSpot | `objects/{objectType}` (+`limit` max 100, `after`, `properties`, `archived`), `objects/{objectType}/{id}`, `objects/{objectType}/search` (`filterGroups` OR-ed, `filters` AND-ed, 13 operators over any property), `objects/{objectType}/batch/read`, `v4/objects/{type}/{id}/associations/{toType}` |
368
+ | `/s3` | Amazon S3 | `ListBuckets`, `HeadBucket`, `GetBucketLocation`, `ListObjectsV2` (`prefix`/`delimiter`/`continuation-token`), `GetObject` (+`Range`), `HeadObject` |
369
+ | `/linear/graphql` | Linear | **GraphQL only** (one `POST`): `issues`, `issue(id:)` (UUID *or* `ENG-123`), `team(id:)` (UUID, key, or name), `teams`, `comments`, `users`, `viewer`, plus the `Team.issues` / `Issue.{comments,labels,children,relations,inverseRelations,attachments,releases}` connections and the by-id roots (`user`, `workflowState`, `project`, `issueLabel`, `cycle`, `release`, `attachment`, `issueRelation`) the official SDK's lazy relation accessors call. Relay pagination (`first`/`after`, `last`/`before` → `{nodes, pageInfo}`), server-side `filter` compiled into SQL, and full introspection |
370
+ | `/fireflies/graphql` | Fireflies | **GraphQL only** (one `POST`): `transcripts`, `transcript(id:)`, `user[(id:)]`, `users`. Offset pagination — `limit` (**max 50**, clamped) / `skip`, returning a **bare list**, not a Relay connection — plus the documented filters: `keyword` × `scope` (`title`\|`sentences`\|`all`), `fromDate`/`toDate`, `host_email`, `organizers`, `participants`, `user_id`, `mine`, `channel_id`. Field names are snake_case, as Fireflies' own schema has them. Full introspection |
371
+
372
+ Mock-only endpoints: `/health`, `/_mock/users`, `/_mock/credentials`, `/_mock/openapi/<source>`,
373
+ `/openapi.json`.
374
+
375
+ ## Tests
376
+
377
+ ```bash
378
+ pytest # unit (synth/pagination/acl/schema/importer parsers) + HTTP endpoint tests
379
+ # (full-crawl completeness, content round-trip, ACL enforcement)
380
+ ruff check . && ruff format --check .
381
+ ```
382
+
383
+ ## Configuration
384
+
385
+ Every setting is an env var with a `BACKLOT_` prefix, and a `.env` file in the working directory is
386
+ read too. Defaults are what the server uses when the var is unset.
387
+
388
+ | Env var | Default | What it does |
389
+ |---|---|---|
390
+ | `BACKLOT_DATA_DIR` | `./data` (resolved against the cwd, **not** the install location) | Where the corpus lives: `mock.sqlite`, `tokens.yaml`, `credentials.yaml`. Both `backlot import` and `backlot serve` read it, which is how you keep several corpora side by side — `BACKLOT_DATA_DIR=/tmp/demo backlot import c.jsonl` |
391
+ | `BACKLOT_ADMIN_TOKEN` | `admin-service-token` | The token that bypasses ACL filtering — a full-crawl / service identity. Set it to anything for a shared deployment |
392
+ | `BACKLOT_ENFORCE_ACL` | `true` | When `false`, any well-formed token is treated as admin. The ACL is still *exposed* through each vendor's permission endpoints, just not *enforced* — useful for isolating whether a connector's gaps are permissions or parsing |
393
+ | `BACKLOT_EXPOSE_TOKENS` | `true` | Serves `GET /_mock/users` and `GET /_mock/credentials`, which hand out every user's token in the clear. Fine for a local mock; set `false` to close both |
394
+ | `BACKLOT_ORG_NAME` | inferred from the corpus (fallback `example`) | The org slug that shows up in `auth.test`, synthesized emails and self-URLs. Inferred from the dominant author email domain — `@acme.com` documents serve as org `acme` — so set it only to override that |
395
+ | `BACKLOT_ORG_DOMAIN` | inferred from the corpus (fallback `example.com`) | The domain half of the same inference, e.g. `acme.com`. Used for addresses the corpus does not state |
396
+ | `BACKLOT_DEFAULT_PAGE_SIZE` | `100` | Page size when a request names none |
397
+ | `BACKLOT_MAX_PAGE_SIZE` | `1000` | Ceiling a request may ask for. Per-vendor caps still win where the real API has one (Fireflies clamps to 50, HubSpot to 100) |
398
+ | `BACKLOT_SQLITE_MMAP_MB` | `256` | Memory-maps the DB so reads come from the OS page cache instead of a syscall each — the main lever against a slow first request after idle. SQLite maps `min(this, db size)`; raise it to at or above your DB size to map a big corpus fully |
399
+ | `BACKLOT_SQLITE_CACHE_MB` | `64` | SQLite's own page cache, per serving connection |
400
+ | `BACKLOT_SQLITE_BUSY_MS` | `5000` | How long a read waits for a lock instead of erroring, so reads ride through an out-of-band write (e.g. an in-place FTS rebuild) rather than 500ing |
401
+
402
+ ## Contributing
403
+
404
+ See [CONTRIBUTING.md](CONTRIBUTING.md). The short version: fidelity to the real APIs is the point,
405
+ so a divergence is a bug — measure against the real service, and bring a test that fails without
406
+ your fix.
407
+
408
+ ## License
409
+
410
+ [MIT](LICENSE)