surveyshield-py 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. surveyshield_py-0.1.0/LICENSE +21 -0
  2. surveyshield_py-0.1.0/PKG-INFO +320 -0
  3. surveyshield_py-0.1.0/README.md +252 -0
  4. surveyshield_py-0.1.0/pyproject.toml +85 -0
  5. surveyshield_py-0.1.0/setup.cfg +4 -0
  6. surveyshield_py-0.1.0/surveyshield/__init__.py +87 -0
  7. surveyshield_py-0.1.0/surveyshield/_providers.py +15 -0
  8. surveyshield_py-0.1.0/surveyshield/_version.py +6 -0
  9. surveyshield_py-0.1.0/surveyshield/cli.py +203 -0
  10. surveyshield_py-0.1.0/surveyshield/live/__init__.py +48 -0
  11. surveyshield_py-0.1.0/surveyshield/live/analyzer.py +804 -0
  12. surveyshield_py-0.1.0/surveyshield/live/patches.py +157 -0
  13. surveyshield_py-0.1.0/surveyshield/live/prompts.py +256 -0
  14. surveyshield_py-0.1.0/surveyshield/models/__init__.py +0 -0
  15. surveyshield_py-0.1.0/surveyshield/models/instrument_review.py +151 -0
  16. surveyshield_py-0.1.0/surveyshield/models/survey_result.py +77 -0
  17. surveyshield_py-0.1.0/surveyshield/review/__init__.py +0 -0
  18. surveyshield_py-0.1.0/surveyshield/review/dimensions.py +191 -0
  19. surveyshield_py-0.1.0/surveyshield/review/mechanism_context.py +76 -0
  20. surveyshield_py-0.1.0/surveyshield/review/parser.py +209 -0
  21. surveyshield_py-0.1.0/surveyshield/review/reviewer.py +770 -0
  22. surveyshield_py-0.1.0/surveyshield/review/templates/instrument_report.html +501 -0
  23. surveyshield_py-0.1.0/surveyshield/serve/__init__.py +0 -0
  24. surveyshield_py-0.1.0/surveyshield/serve/api/__init__.py +0 -0
  25. surveyshield_py-0.1.0/surveyshield/serve/api/instrument.py +231 -0
  26. surveyshield_py-0.1.0/surveyshield/serve/api/survey.py +224 -0
  27. surveyshield_py-0.1.0/surveyshield/serve/app.py +113 -0
  28. surveyshield_py-0.1.0/surveyshield/serve/config.py +17 -0
  29. surveyshield_py-0.1.0/surveyshield/serve/static/asset-manifest.json +13 -0
  30. surveyshield_py-0.1.0/surveyshield/serve/static/favicon.ico +0 -0
  31. surveyshield_py-0.1.0/surveyshield/serve/static/index.html +1 -0
  32. surveyshield_py-0.1.0/surveyshield/serve/static/logo192.png +0 -0
  33. surveyshield_py-0.1.0/surveyshield/serve/static/logo512.png +0 -0
  34. surveyshield_py-0.1.0/surveyshield/serve/static/manifest.json +29 -0
  35. surveyshield_py-0.1.0/surveyshield/serve/static/static/css/main.6419f939.css +4 -0
  36. surveyshield_py-0.1.0/surveyshield/serve/static/static/css/main.6419f939.css.map +1 -0
  37. surveyshield_py-0.1.0/surveyshield/serve/static/static/js/main.71e274b1.js +3 -0
  38. surveyshield_py-0.1.0/surveyshield/serve/static/static/js/main.71e274b1.js.LICENSE.txt +58 -0
  39. surveyshield_py-0.1.0/surveyshield/serve/static/static/js/main.71e274b1.js.map +1 -0
  40. surveyshield_py-0.1.0/surveyshield_py.egg-info/PKG-INFO +320 -0
  41. surveyshield_py-0.1.0/surveyshield_py.egg-info/SOURCES.txt +49 -0
  42. surveyshield_py-0.1.0/surveyshield_py.egg-info/dependency_links.txt +1 -0
  43. surveyshield_py-0.1.0/surveyshield_py.egg-info/entry_points.txt +2 -0
  44. surveyshield_py-0.1.0/surveyshield_py.egg-info/requires.txt +28 -0
  45. surveyshield_py-0.1.0/surveyshield_py.egg-info/top_level.txt +1 -0
  46. surveyshield_py-0.1.0/tests/test_aggregate.py +161 -0
  47. surveyshield_py-0.1.0/tests/test_cli.py +97 -0
  48. surveyshield_py-0.1.0/tests/test_qsf_parser.py +77 -0
  49. surveyshield_py-0.1.0/tests/test_review_dimension.py +181 -0
  50. surveyshield_py-0.1.0/tests/test_serve.py +53 -0
  51. surveyshield_py-0.1.0/tests/test_verify_quote.py +105 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Kianté Fernandez
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,320 @@
1
+ Metadata-Version: 2.4
2
+ Name: surveyshield-py
3
+ Version: 0.1.0
4
+ Summary: Static review of online survey instruments for resistance to AI/bot respondents
5
+ Author: Kianté Fernandez, Andrew Low, Jonathan Bogard, Craig R. Fox
6
+ License: MIT License
7
+
8
+ Copyright (c) 2025 Kianté Fernandez
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
27
+
28
+ Project-URL: Homepage, https://github.com/kiante-fernandez/survey-shield
29
+ Project-URL: Repository, https://github.com/kiante-fernandez/survey-shield
30
+ Project-URL: Issues, https://github.com/kiante-fernandez/survey-shield/issues
31
+ Classifier: Development Status :: 4 - Beta
32
+ Classifier: Intended Audience :: Science/Research
33
+ Classifier: License :: OSI Approved :: MIT License
34
+ Classifier: Programming Language :: Python :: 3.10
35
+ Classifier: Programming Language :: Python :: 3.11
36
+ Classifier: Programming Language :: Python :: 3.12
37
+ Classifier: Topic :: Scientific/Engineering
38
+ Requires-Python: >=3.10
39
+ Description-Content-Type: text/markdown
40
+ License-File: LICENSE
41
+ Requires-Dist: fastapi>=0.110
42
+ Requires-Dist: uvicorn[standard]>=0.27
43
+ Requires-Dist: gunicorn>=21.0
44
+ Requires-Dist: python-multipart>=0.0.7
45
+ Requires-Dist: jinja2>=3.1
46
+ Requires-Dist: pydantic>=2.5
47
+ Requires-Dist: python-dotenv>=1.0
48
+ Requires-Dist: openai>=1.0
49
+ Requires-Dist: langchain-core>=0.3
50
+ Requires-Dist: langchain-openai>=0.2
51
+ Requires-Dist: langchain-google-genai>=2.0
52
+ Requires-Dist: typer>=0.12
53
+ Requires-Dist: httpx>=0.24
54
+ Requires-Dist: requests>=2.28
55
+ Requires-Dist: aiofiles>=23.0
56
+ Provides-Extra: live
57
+ Requires-Dist: browser-use>=0.9.5; extra == "live"
58
+ Requires-Dist: playwright>=1.40; extra == "live"
59
+ Provides-Extra: dev
60
+ Requires-Dist: pytest>=7; extra == "dev"
61
+ Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
62
+ Requires-Dist: pytest-cov>=4; extra == "dev"
63
+ Requires-Dist: black>=24; extra == "dev"
64
+ Requires-Dist: flake8>=6; extra == "dev"
65
+ Requires-Dist: build>=1.0; extra == "dev"
66
+ Requires-Dist: twine>=4; extra == "dev"
67
+ Dynamic: license-file
68
+
69
+ # 🛡️ Survey Shield
70
+
71
+ Static review of online survey instruments for resistance to AI/bot respondents — plus an optional live runtime that drives a real browser through your survey.
72
+
73
+ ## What is Survey Shield?
74
+
75
+ Survey Shield gives researchers feedback on whether their survey instrument is hardened against AI respondents. Two paths:
76
+
77
+ 1. **Instrument Review** *(primary, static, no browser)* — point it at a Qualtrics `.qsf` export. Multi-agent LLM reviewers fan out across bot-resistance dimensions (attention checks, identity questions, visual-perceptual traps), produce a peer-review-style verdict with verbatim-grounded findings, and render a self-contained HTML report with a copy-paste Methods statement and APA/BibTeX citation.
78
+ 2. **Take Survey (live runtime)** *(optional `[live]` extra)* — drives a real browser ([browser-use](https://docs.browser-use.com)) through a live Qualtrics URL, reports detected mechanisms after the fact. Costs ~5–10 minutes and real LLM credit per run, so the hosted demo doesn't expose it; install the `[live]` extra to run it locally.
79
+
80
+ Researchers using Survey Shield mostly want Instrument Review. Reach for the live runtime when you need to exercise the survey end-to-end.
81
+
82
+ ## Install
83
+
84
+ ```bash
85
+ pip install surveyshield-py # static review only — small, no browser
86
+ pip install "surveyshield-py[live]" # adds browser-use + Playwright
87
+ playwright install chromium # only if you installed [live]
88
+ ```
89
+
90
+ The package name on PyPI is `surveyshield-py` (after [`openreview-py`](https://pypi.org/project/openreview-py/)); the import name is `surveyshield`.
91
+
92
+ Set an LLM provider key in your environment (or in a `.env` file in the working directory — Survey Shield loads it via `python-dotenv`):
93
+
94
+ ```env
95
+ OPENAI_API_KEY=sk-... # used unless the model name starts with "gemini"
96
+ GOOGLE_API_KEY=... # used for Gemini models
97
+ ```
98
+
99
+ ## CLI
100
+
101
+ ```bash
102
+ surveyshield review your_survey.qsf
103
+ # → writes your_survey.report.html next to the input
104
+
105
+ surveyshield review your_survey.qsf --output report.html --json review.json \
106
+ --model gpt-4o-mini
107
+
108
+ surveyshield take https://qualtrics.com/jfe/form/SV_xxx # requires [live]
109
+ --model gemini-3-flash-preview --max-steps 150
110
+
111
+ surveyshield serve --host 127.0.0.1 --port 8000
112
+ # → boots the FastAPI app + bundled React SPA
113
+ ```
114
+
115
+ `surveyshield --help` lists every command and flag.
116
+
117
+ ## Python API
118
+
119
+ ```python
120
+ import asyncio
121
+ import surveyshield
122
+
123
+ review, parsed = asyncio.run(
124
+ surveyshield.review_qsf(
125
+ "your_survey.qsf",
126
+ model="gpt-4o-mini",
127
+ # api_key="sk-...", # or rely on env vars
128
+ # dimensions=["attention_checks"], # default = all
129
+ )
130
+ )
131
+
132
+ print(review.overall_score, review.overall_feedback.headline)
133
+
134
+ with open("report.html", "w") as f:
135
+ f.write(surveyshield.render_html(review, parsed))
136
+ ```
137
+
138
+ The `review` object is a `surveyshield.InstrumentReview` Pydantic model. Power users can compose the lower-level seams directly: `parse_qsf`, `run_review`, `drop_unverified_quotes`, `consolidate_and_summarize`, `aggregate`. See `surveyshield/__init__.py` for the public surface.
139
+
140
+ For live runtime:
141
+
142
+ ```python
143
+ import asyncio, surveyshield # surveyshield-py[live] installed
144
+
145
+ result = asyncio.run(surveyshield.take_survey(
146
+ "https://qualtrics.com/jfe/form/SV_xxx",
147
+ model="gemini-3-flash-preview",
148
+ max_steps=150,
149
+ ))
150
+ print(result.success_probability, [m.name for m in result.detected_mechanisms])
151
+ ```
152
+
153
+ If the `[live]` extra isn't installed, `surveyshield.take_survey` resolves to `None` and the CLI's `take` command exits with a clear install hint.
154
+
155
+ ## Self-host the hosted UI
156
+
157
+ ```bash
158
+ git clone https://github.com/kiante-fernandez/survey-shield
159
+ cd survey-shield
160
+ ./setup.sh # creates ../.conda env
161
+ echo "OPENAI_API_KEY=sk-..." > backend/.env # or GOOGLE_API_KEY
162
+ cd backend && ./start.sh # → http://localhost:8000
163
+ ```
164
+
165
+ The `Take Survey` tab is gated on `live_take_enabled` — `GET /api/v1/survey/config` flips it to `true` once a key is detected in the env.
166
+
167
+ ### Endpoints
168
+
169
+ - **Web UI**: http://localhost:8000
170
+ - **Interactive API docs**: http://localhost:8000/docs
171
+ - **Health check**: http://localhost:8000/health
172
+ - **Live-runtime config**: http://localhost:8000/api/v1/survey/config
173
+
174
+ ### Models
175
+
176
+ The hosted UI does not expose a model picker — Instrument Review reviewers run on a sensible default (`gpt-4o-mini`). Self-hosters who want a different model can pass `model_name` directly to the API or CLI. The backend has no allowlist; any model name `langchain-openai`'s `ChatOpenAI` or `langchain-google-genai`'s `ChatGoogleGenerativeAI` accept will be routed by prefix:
177
+
178
+ - Names starting with `gemini` → Google (requires `GOOGLE_API_KEY`)
179
+ - Everything else → OpenAI (requires `OPENAI_API_KEY`)
180
+
181
+ ## API usage (self-host)
182
+
183
+ ### Instrument Review (primary)
184
+
185
+ ```bash
186
+ # Submit a QSF for review
187
+ curl -F "file=@your_survey.qsf" \
188
+ http://localhost:8000/api/v1/instrument/review
189
+ # → {"review_id": "<uuid>", "status": "queued", ...}
190
+
191
+ # Poll
192
+ curl http://localhost:8000/api/v1/instrument/status/<uuid>
193
+ # queued → running → completed (~30–90 s)
194
+
195
+ # Structured JSON
196
+ curl http://localhost:8000/api/v1/instrument/results/<uuid>
197
+
198
+ # Human-readable HTML report
199
+ curl "http://localhost:8000/api/v1/instrument/report/<uuid>"
200
+
201
+ # Download as a file
202
+ curl -OJ "http://localhost:8000/api/v1/instrument/report/<uuid>?download=1"
203
+ ```
204
+
205
+ ### Live runtime (self-host only)
206
+
207
+ ```bash
208
+ curl -X POST "http://localhost:8000/api/v1/survey/analyze" \
209
+ -H "Content-Type: application/json" \
210
+ -d '{
211
+ "survey_url": "https://example.com/survey",
212
+ "model_name": "gpt-4o-mini",
213
+ "max_steps": 150,
214
+ "use_vision": true
215
+ }'
216
+ # Then poll /api/v1/survey/status/<id> and fetch /api/v1/survey/results/<id>.
217
+ ```
218
+
219
+ ## What Survey Shield evaluates
220
+
221
+ ### Instrument Review dimensions (primary)
222
+
223
+ The reviewer fans out across plug-in dimensions defined in [`surveyshield/review/dimensions.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/review/dimensions.py). v1 ships three:
224
+
225
+ - **`attention_checks`** — explicit IMCs and instructional manipulation checks (Westwood, 2025; PNAS).
226
+ - **`identity_questions`** — direct LLM-resistance items (identity probes, reverse-shibboleth questions, impossible-event questions).
227
+ - **`visual_perceptual_traps`** — image- and layout-based cognitive traps that exploit vision-language model architectural constraints (Affonso, 2026; JCR).
228
+
229
+ Adding a new dimension is one entry in `dimensions.py` plus a Pydantic output schema — the reviewer fan-out, aggregator, and HTML report are all plug-in-driven.
230
+
231
+ Findings are **grounded**: every reviewer finding must cite a verbatim excerpt from the source survey. Survey Shield substring-checks each excerpt against the parsed survey and drops anything that can't be located. What reaches the report is grounded in real survey content.
232
+
233
+ The product is scoped strictly to **bot resistance**. We do not critique a survey's substantive research design, theoretical framing, or question wording — those remain the researcher's domain.
234
+
235
+ ### Live-runtime mechanisms
236
+
237
+ When you run a live analysis against a real Qualtrics URL, Survey Shield's browser-use Agent navigates the survey end-to-end and inventories detected mechanisms by category — hover traps, invisible text, timing requirements, CAPTCHAs, honeypot fields, attention checks, and behavioural / mouse-tracking signals. Per-mechanism severity weights (1–10) feed into success-probability and difficulty scores. See [`surveyshield/live/analyzer.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/analyzer.py) for the taxonomy and [`surveyshield/live/prompts.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/prompts.py) for the agent task.
238
+
239
+ ## Development
240
+
241
+ ### Project structure
242
+
243
+ ```
244
+ survey-shield/
245
+ ├── surveyshield/ # the importable package
246
+ │ ├── __init__.py # public API (review_qsf, render_html, take_survey, …)
247
+ │ ├── cli.py # Typer CLI (review / take / serve)
248
+ │ ├── models/ # Pydantic schemas
249
+ │ ├── review/ # static review pipeline
250
+ │ │ ├── parser.py # QSF → ParsedSurvey
251
+ │ │ ├── dimensions.py # plug-in dimension registry
252
+ │ │ ├── reviewer.py # fan-out, verify, consolidate, aggregate
253
+ │ │ ├── mechanism_context.py
254
+ │ │ └── templates/ # Jinja2 self-contained HTML report
255
+ │ ├── live/ # browser-use runtime ([live] extra)
256
+ │ │ ├── analyzer.py # SurveyAnalyzer / take_survey
257
+ │ │ ├── prompts.py
258
+ │ │ └── patches.py
259
+ │ └── serve/ # FastAPI app + bundled React SPA
260
+ │ ├── app.py
261
+ │ ├── config.py
262
+ │ ├── api/{survey,instrument}.py
263
+ │ └── static/ # built React (populated by bin/build.sh)
264
+ ├── frontend/ # React/TypeScript source (CRA)
265
+ ├── tests/ # pytest suite + tiny QSF fixture
266
+ ├── backend/start.sh # convenience wrapper around `surveyshield serve`
267
+ ├── pyproject.toml # canonical package metadata
268
+ ├── Procfile # web: gunicorn surveyshield.serve.app:app
269
+ ├── bin/build.sh # React build → surveyshield/serve/static/
270
+ └── .github/workflows/ # test.yml + release.yml (PyPI trusted publishing)
271
+ ```
272
+
273
+ ### Local dev
274
+
275
+ ```bash
276
+ ./setup.sh # one-time: conda env at ../.conda
277
+ pip install -e ".[dev,live]" # editable install + tests + browser-use
278
+ pytest -q # ~30 tests, no LLM calls
279
+ cd frontend && npx tsc --noEmit && npm run build
280
+ ```
281
+
282
+ ### Releasing
283
+
284
+ ```bash
285
+ git tag v0.1.0 && git push --tags
286
+ # .github/workflows/release.yml builds the wheel + sdist (with the React SPA
287
+ # bundled into surveyshield/serve/static/) and publishes to PyPI via OIDC.
288
+ ```
289
+
290
+ The PyPI project must be configured with this repo + `release.yml` as a Trusted Publisher before the first push.
291
+
292
+ ## Contributing
293
+
294
+ 1. Fork the repository
295
+ 2. Create a feature branch
296
+ 3. Make changes with tests
297
+ 4. Submit a pull request
298
+
299
+ ## License
300
+
301
+ MIT License — see [LICENSE](https://github.com/kiante-fernandez/survey-shield/blob/main/LICENSE).
302
+
303
+ ## Citation
304
+
305
+ If you use Survey Shield in published work:
306
+
307
+ ```bibtex
308
+ @misc{fernandez2026surveyshield,
309
+ author = {Fernandez, K. and Low, A. and Bogard, J. and Fox, C. R.},
310
+ title = {Survey Shield: Static review of online survey instruments for resistance to non-human responses},
311
+ year = {2026},
312
+ note = {Manuscript in preparation},
313
+ }
314
+ ```
315
+
316
+ Every report includes the same citation pre-formatted (APA + BibTeX).
317
+
318
+ ## Disclaimer
319
+
320
+ Survey Shield is intended for research and testing purposes.
@@ -0,0 +1,252 @@
1
+ # 🛡️ Survey Shield
2
+
3
+ Static review of online survey instruments for resistance to AI/bot respondents — plus an optional live runtime that drives a real browser through your survey.
4
+
5
+ ## What is Survey Shield?
6
+
7
+ Survey Shield gives researchers feedback on whether their survey instrument is hardened against AI respondents. Two paths:
8
+
9
+ 1. **Instrument Review** *(primary, static, no browser)* — point it at a Qualtrics `.qsf` export. Multi-agent LLM reviewers fan out across bot-resistance dimensions (attention checks, identity questions, visual-perceptual traps), produce a peer-review-style verdict with verbatim-grounded findings, and render a self-contained HTML report with a copy-paste Methods statement and APA/BibTeX citation.
10
+ 2. **Take Survey (live runtime)** *(optional `[live]` extra)* — drives a real browser ([browser-use](https://docs.browser-use.com)) through a live Qualtrics URL, reports detected mechanisms after the fact. Costs ~5–10 minutes and real LLM credit per run, so the hosted demo doesn't expose it; install the `[live]` extra to run it locally.
11
+
12
+ Researchers using Survey Shield mostly want Instrument Review. Reach for the live runtime when you need to exercise the survey end-to-end.
13
+
14
+ ## Install
15
+
16
+ ```bash
17
+ pip install surveyshield-py # static review only — small, no browser
18
+ pip install "surveyshield-py[live]" # adds browser-use + Playwright
19
+ playwright install chromium # only if you installed [live]
20
+ ```
21
+
22
+ The package name on PyPI is `surveyshield-py` (after [`openreview-py`](https://pypi.org/project/openreview-py/)); the import name is `surveyshield`.
23
+
24
+ Set an LLM provider key in your environment (or in a `.env` file in the working directory — Survey Shield loads it via `python-dotenv`):
25
+
26
+ ```env
27
+ OPENAI_API_KEY=sk-... # used unless the model name starts with "gemini"
28
+ GOOGLE_API_KEY=... # used for Gemini models
29
+ ```
30
+
31
+ ## CLI
32
+
33
+ ```bash
34
+ surveyshield review your_survey.qsf
35
+ # → writes your_survey.report.html next to the input
36
+
37
+ surveyshield review your_survey.qsf --output report.html --json review.json \
38
+ --model gpt-4o-mini
39
+
40
+ surveyshield take https://qualtrics.com/jfe/form/SV_xxx # requires [live]
41
+ --model gemini-3-flash-preview --max-steps 150
42
+
43
+ surveyshield serve --host 127.0.0.1 --port 8000
44
+ # → boots the FastAPI app + bundled React SPA
45
+ ```
46
+
47
+ `surveyshield --help` lists every command and flag.
48
+
49
+ ## Python API
50
+
51
+ ```python
52
+ import asyncio
53
+ import surveyshield
54
+
55
+ review, parsed = asyncio.run(
56
+ surveyshield.review_qsf(
57
+ "your_survey.qsf",
58
+ model="gpt-4o-mini",
59
+ # api_key="sk-...", # or rely on env vars
60
+ # dimensions=["attention_checks"], # default = all
61
+ )
62
+ )
63
+
64
+ print(review.overall_score, review.overall_feedback.headline)
65
+
66
+ with open("report.html", "w") as f:
67
+ f.write(surveyshield.render_html(review, parsed))
68
+ ```
69
+
70
+ The `review` object is a `surveyshield.InstrumentReview` Pydantic model. Power users can compose the lower-level seams directly: `parse_qsf`, `run_review`, `drop_unverified_quotes`, `consolidate_and_summarize`, `aggregate`. See `surveyshield/__init__.py` for the public surface.
71
+
72
+ For live runtime:
73
+
74
+ ```python
75
+ import asyncio, surveyshield # surveyshield-py[live] installed
76
+
77
+ result = asyncio.run(surveyshield.take_survey(
78
+ "https://qualtrics.com/jfe/form/SV_xxx",
79
+ model="gemini-3-flash-preview",
80
+ max_steps=150,
81
+ ))
82
+ print(result.success_probability, [m.name for m in result.detected_mechanisms])
83
+ ```
84
+
85
+ If the `[live]` extra isn't installed, `surveyshield.take_survey` resolves to `None` and the CLI's `take` command exits with a clear install hint.
86
+
87
+ ## Self-host the hosted UI
88
+
89
+ ```bash
90
+ git clone https://github.com/kiante-fernandez/survey-shield
91
+ cd survey-shield
92
+ ./setup.sh # creates ../.conda env
93
+ echo "OPENAI_API_KEY=sk-..." > backend/.env # or GOOGLE_API_KEY
94
+ cd backend && ./start.sh # → http://localhost:8000
95
+ ```
96
+
97
+ The `Take Survey` tab is gated on `live_take_enabled` — `GET /api/v1/survey/config` flips it to `true` once a key is detected in the env.
98
+
99
+ ### Endpoints
100
+
101
+ - **Web UI**: http://localhost:8000
102
+ - **Interactive API docs**: http://localhost:8000/docs
103
+ - **Health check**: http://localhost:8000/health
104
+ - **Live-runtime config**: http://localhost:8000/api/v1/survey/config
105
+
106
+ ### Models
107
+
108
+ The hosted UI does not expose a model picker — Instrument Review reviewers run on a sensible default (`gpt-4o-mini`). Self-hosters who want a different model can pass `model_name` directly to the API or CLI. The backend has no allowlist; any model name `langchain-openai`'s `ChatOpenAI` or `langchain-google-genai`'s `ChatGoogleGenerativeAI` accept will be routed by prefix:
109
+
110
+ - Names starting with `gemini` → Google (requires `GOOGLE_API_KEY`)
111
+ - Everything else → OpenAI (requires `OPENAI_API_KEY`)
112
+
113
+ ## API usage (self-host)
114
+
115
+ ### Instrument Review (primary)
116
+
117
+ ```bash
118
+ # Submit a QSF for review
119
+ curl -F "file=@your_survey.qsf" \
120
+ http://localhost:8000/api/v1/instrument/review
121
+ # → {"review_id": "<uuid>", "status": "queued", ...}
122
+
123
+ # Poll
124
+ curl http://localhost:8000/api/v1/instrument/status/<uuid>
125
+ # queued → running → completed (~30–90 s)
126
+
127
+ # Structured JSON
128
+ curl http://localhost:8000/api/v1/instrument/results/<uuid>
129
+
130
+ # Human-readable HTML report
131
+ curl "http://localhost:8000/api/v1/instrument/report/<uuid>"
132
+
133
+ # Download as a file
134
+ curl -OJ "http://localhost:8000/api/v1/instrument/report/<uuid>?download=1"
135
+ ```
136
+
137
+ ### Live runtime (self-host only)
138
+
139
+ ```bash
140
+ curl -X POST "http://localhost:8000/api/v1/survey/analyze" \
141
+ -H "Content-Type: application/json" \
142
+ -d '{
143
+ "survey_url": "https://example.com/survey",
144
+ "model_name": "gpt-4o-mini",
145
+ "max_steps": 150,
146
+ "use_vision": true
147
+ }'
148
+ # Then poll /api/v1/survey/status/<id> and fetch /api/v1/survey/results/<id>.
149
+ ```
150
+
151
+ ## What Survey Shield evaluates
152
+
153
+ ### Instrument Review dimensions (primary)
154
+
155
+ The reviewer fans out across plug-in dimensions defined in [`surveyshield/review/dimensions.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/review/dimensions.py). v1 ships three:
156
+
157
+ - **`attention_checks`** — explicit IMCs and instructional manipulation checks (Westwood, 2025; PNAS).
158
+ - **`identity_questions`** — direct LLM-resistance items (identity probes, reverse-shibboleth questions, impossible-event questions).
159
+ - **`visual_perceptual_traps`** — image- and layout-based cognitive traps that exploit vision-language model architectural constraints (Affonso, 2026; JCR).
160
+
161
+ Adding a new dimension is one entry in `dimensions.py` plus a Pydantic output schema — the reviewer fan-out, aggregator, and HTML report are all plug-in-driven.
162
+
163
+ Findings are **grounded**: every reviewer finding must cite a verbatim excerpt from the source survey. Survey Shield substring-checks each excerpt against the parsed survey and drops anything that can't be located. What reaches the report is grounded in real survey content.
164
+
165
+ The product is scoped strictly to **bot resistance**. We do not critique a survey's substantive research design, theoretical framing, or question wording — those remain the researcher's domain.
166
+
167
+ ### Live-runtime mechanisms
168
+
169
+ When you run a live analysis against a real Qualtrics URL, Survey Shield's browser-use Agent navigates the survey end-to-end and inventories detected mechanisms by category — hover traps, invisible text, timing requirements, CAPTCHAs, honeypot fields, attention checks, and behavioural / mouse-tracking signals. Per-mechanism severity weights (1–10) feed into success-probability and difficulty scores. See [`surveyshield/live/analyzer.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/analyzer.py) for the taxonomy and [`surveyshield/live/prompts.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/prompts.py) for the agent task.
170
+
171
+ ## Development
172
+
173
+ ### Project structure
174
+
175
+ ```
176
+ survey-shield/
177
+ ├── surveyshield/ # the importable package
178
+ │ ├── __init__.py # public API (review_qsf, render_html, take_survey, …)
179
+ │ ├── cli.py # Typer CLI (review / take / serve)
180
+ │ ├── models/ # Pydantic schemas
181
+ │ ├── review/ # static review pipeline
182
+ │ │ ├── parser.py # QSF → ParsedSurvey
183
+ │ │ ├── dimensions.py # plug-in dimension registry
184
+ │ │ ├── reviewer.py # fan-out, verify, consolidate, aggregate
185
+ │ │ ├── mechanism_context.py
186
+ │ │ └── templates/ # Jinja2 self-contained HTML report
187
+ │ ├── live/ # browser-use runtime ([live] extra)
188
+ │ │ ├── analyzer.py # SurveyAnalyzer / take_survey
189
+ │ │ ├── prompts.py
190
+ │ │ └── patches.py
191
+ │ └── serve/ # FastAPI app + bundled React SPA
192
+ │ ├── app.py
193
+ │ ├── config.py
194
+ │ ├── api/{survey,instrument}.py
195
+ │ └── static/ # built React (populated by bin/build.sh)
196
+ ├── frontend/ # React/TypeScript source (CRA)
197
+ ├── tests/ # pytest suite + tiny QSF fixture
198
+ ├── backend/start.sh # convenience wrapper around `surveyshield serve`
199
+ ├── pyproject.toml # canonical package metadata
200
+ ├── Procfile # web: gunicorn surveyshield.serve.app:app
201
+ ├── bin/build.sh # React build → surveyshield/serve/static/
202
+ └── .github/workflows/ # test.yml + release.yml (PyPI trusted publishing)
203
+ ```
204
+
205
+ ### Local dev
206
+
207
+ ```bash
208
+ ./setup.sh # one-time: conda env at ../.conda
209
+ pip install -e ".[dev,live]" # editable install + tests + browser-use
210
+ pytest -q # ~30 tests, no LLM calls
211
+ cd frontend && npx tsc --noEmit && npm run build
212
+ ```
213
+
214
+ ### Releasing
215
+
216
+ ```bash
217
+ git tag v0.1.0 && git push --tags
218
+ # .github/workflows/release.yml builds the wheel + sdist (with the React SPA
219
+ # bundled into surveyshield/serve/static/) and publishes to PyPI via OIDC.
220
+ ```
221
+
222
+ The PyPI project must be configured with this repo + `release.yml` as a Trusted Publisher before the first push.
223
+
224
+ ## Contributing
225
+
226
+ 1. Fork the repository
227
+ 2. Create a feature branch
228
+ 3. Make changes with tests
229
+ 4. Submit a pull request
230
+
231
+ ## License
232
+
233
+ MIT License — see [LICENSE](https://github.com/kiante-fernandez/survey-shield/blob/main/LICENSE).
234
+
235
+ ## Citation
236
+
237
+ If you use Survey Shield in published work:
238
+
239
+ ```bibtex
240
+ @misc{fernandez2026surveyshield,
241
+ author = {Fernandez, K. and Low, A. and Bogard, J. and Fox, C. R.},
242
+ title = {Survey Shield: Static review of online survey instruments for resistance to non-human responses},
243
+ year = {2026},
244
+ note = {Manuscript in preparation},
245
+ }
246
+ ```
247
+
248
+ Every report includes the same citation pre-formatted (APA + BibTeX).
249
+
250
+ ## Disclaimer
251
+
252
+ Survey Shield is intended for research and testing purposes.
@@ -0,0 +1,85 @@
1
+ [project]
2
+ name = "surveyshield-py"
3
+ description = "Static review of online survey instruments for resistance to AI/bot respondents"
4
+ readme = "README.md"
5
+ license = { file = "LICENSE" }
6
+ authors = [
7
+ { name = "Kianté Fernandez" },
8
+ { name = "Andrew Low" },
9
+ { name = "Jonathan Bogard" },
10
+ { name = "Craig R. Fox" },
11
+ ]
12
+ requires-python = ">=3.10"
13
+ dynamic = ["version"]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Intended Audience :: Science/Research",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3.10",
19
+ "Programming Language :: Python :: 3.11",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Topic :: Scientific/Engineering",
22
+ ]
23
+
24
+ dependencies = [
25
+ "fastapi>=0.110",
26
+ "uvicorn[standard]>=0.27",
27
+ "gunicorn>=21.0",
28
+ "python-multipart>=0.0.7",
29
+ "jinja2>=3.1",
30
+ "pydantic>=2.5",
31
+ "python-dotenv>=1.0",
32
+ "openai>=1.0",
33
+ "langchain-core>=0.3",
34
+ "langchain-openai>=0.2",
35
+ "langchain-google-genai>=2.0",
36
+ "typer>=0.12",
37
+ "httpx>=0.24",
38
+ "requests>=2.28",
39
+ "aiofiles>=23.0",
40
+ ]
41
+
42
+ [project.optional-dependencies]
43
+ live = [
44
+ "browser-use>=0.9.5",
45
+ "playwright>=1.40",
46
+ ]
47
+ dev = [
48
+ "pytest>=7",
49
+ "pytest-asyncio>=0.21",
50
+ "pytest-cov>=4",
51
+ "black>=24",
52
+ "flake8>=6",
53
+ "build>=1.0",
54
+ "twine>=4",
55
+ ]
56
+
57
+ [project.scripts]
58
+ surveyshield = "surveyshield.cli:app"
59
+
60
+ [project.urls]
61
+ Homepage = "https://github.com/kiante-fernandez/survey-shield"
62
+ Repository = "https://github.com/kiante-fernandez/survey-shield"
63
+ Issues = "https://github.com/kiante-fernandez/survey-shield/issues"
64
+
65
+ [build-system]
66
+ requires = ["setuptools>=68", "wheel"]
67
+ build-backend = "setuptools.build_meta"
68
+
69
+ [tool.setuptools.dynamic]
70
+ version = { attr = "surveyshield._version.__version__" }
71
+
72
+ [tool.setuptools.packages.find]
73
+ where = ["."]
74
+ include = ["surveyshield*"]
75
+ exclude = ["backend*", "frontend*", "tests*", "example_qsf*", "bin*"]
76
+
77
+ [tool.setuptools.package-data]
78
+ surveyshield = [
79
+ "review/templates/*.html",
80
+ "serve/static/**/*",
81
+ ]
82
+
83
+ [tool.pytest.ini_options]
84
+ testpaths = ["tests"]
85
+ asyncio_mode = "auto"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+