surveyshield-py 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- surveyshield_py-0.1.0/LICENSE +21 -0
- surveyshield_py-0.1.0/PKG-INFO +320 -0
- surveyshield_py-0.1.0/README.md +252 -0
- surveyshield_py-0.1.0/pyproject.toml +85 -0
- surveyshield_py-0.1.0/setup.cfg +4 -0
- surveyshield_py-0.1.0/surveyshield/__init__.py +87 -0
- surveyshield_py-0.1.0/surveyshield/_providers.py +15 -0
- surveyshield_py-0.1.0/surveyshield/_version.py +6 -0
- surveyshield_py-0.1.0/surveyshield/cli.py +203 -0
- surveyshield_py-0.1.0/surveyshield/live/__init__.py +48 -0
- surveyshield_py-0.1.0/surveyshield/live/analyzer.py +804 -0
- surveyshield_py-0.1.0/surveyshield/live/patches.py +157 -0
- surveyshield_py-0.1.0/surveyshield/live/prompts.py +256 -0
- surveyshield_py-0.1.0/surveyshield/models/__init__.py +0 -0
- surveyshield_py-0.1.0/surveyshield/models/instrument_review.py +151 -0
- surveyshield_py-0.1.0/surveyshield/models/survey_result.py +77 -0
- surveyshield_py-0.1.0/surveyshield/review/__init__.py +0 -0
- surveyshield_py-0.1.0/surveyshield/review/dimensions.py +191 -0
- surveyshield_py-0.1.0/surveyshield/review/mechanism_context.py +76 -0
- surveyshield_py-0.1.0/surveyshield/review/parser.py +209 -0
- surveyshield_py-0.1.0/surveyshield/review/reviewer.py +770 -0
- surveyshield_py-0.1.0/surveyshield/review/templates/instrument_report.html +501 -0
- surveyshield_py-0.1.0/surveyshield/serve/__init__.py +0 -0
- surveyshield_py-0.1.0/surveyshield/serve/api/__init__.py +0 -0
- surveyshield_py-0.1.0/surveyshield/serve/api/instrument.py +231 -0
- surveyshield_py-0.1.0/surveyshield/serve/api/survey.py +224 -0
- surveyshield_py-0.1.0/surveyshield/serve/app.py +113 -0
- surveyshield_py-0.1.0/surveyshield/serve/config.py +17 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/asset-manifest.json +13 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/favicon.ico +0 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/index.html +1 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/logo192.png +0 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/logo512.png +0 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/manifest.json +29 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/static/css/main.6419f939.css +4 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/static/css/main.6419f939.css.map +1 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/static/js/main.71e274b1.js +3 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/static/js/main.71e274b1.js.LICENSE.txt +58 -0
- surveyshield_py-0.1.0/surveyshield/serve/static/static/js/main.71e274b1.js.map +1 -0
- surveyshield_py-0.1.0/surveyshield_py.egg-info/PKG-INFO +320 -0
- surveyshield_py-0.1.0/surveyshield_py.egg-info/SOURCES.txt +49 -0
- surveyshield_py-0.1.0/surveyshield_py.egg-info/dependency_links.txt +1 -0
- surveyshield_py-0.1.0/surveyshield_py.egg-info/entry_points.txt +2 -0
- surveyshield_py-0.1.0/surveyshield_py.egg-info/requires.txt +28 -0
- surveyshield_py-0.1.0/surveyshield_py.egg-info/top_level.txt +1 -0
- surveyshield_py-0.1.0/tests/test_aggregate.py +161 -0
- surveyshield_py-0.1.0/tests/test_cli.py +97 -0
- surveyshield_py-0.1.0/tests/test_qsf_parser.py +77 -0
- surveyshield_py-0.1.0/tests/test_review_dimension.py +181 -0
- surveyshield_py-0.1.0/tests/test_serve.py +53 -0
- surveyshield_py-0.1.0/tests/test_verify_quote.py +105 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Kianté Fernandez
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: surveyshield-py
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Static review of online survey instruments for resistance to AI/bot respondents
|
|
5
|
+
Author: Kianté Fernandez, Andrew Low, Jonathan Bogard, Craig R. Fox
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 Kianté Fernandez
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/kiante-fernandez/survey-shield
|
|
29
|
+
Project-URL: Repository, https://github.com/kiante-fernandez/survey-shield
|
|
30
|
+
Project-URL: Issues, https://github.com/kiante-fernandez/survey-shield/issues
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Intended Audience :: Science/Research
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
37
|
+
Classifier: Topic :: Scientific/Engineering
|
|
38
|
+
Requires-Python: >=3.10
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
License-File: LICENSE
|
|
41
|
+
Requires-Dist: fastapi>=0.110
|
|
42
|
+
Requires-Dist: uvicorn[standard]>=0.27
|
|
43
|
+
Requires-Dist: gunicorn>=21.0
|
|
44
|
+
Requires-Dist: python-multipart>=0.0.7
|
|
45
|
+
Requires-Dist: jinja2>=3.1
|
|
46
|
+
Requires-Dist: pydantic>=2.5
|
|
47
|
+
Requires-Dist: python-dotenv>=1.0
|
|
48
|
+
Requires-Dist: openai>=1.0
|
|
49
|
+
Requires-Dist: langchain-core>=0.3
|
|
50
|
+
Requires-Dist: langchain-openai>=0.2
|
|
51
|
+
Requires-Dist: langchain-google-genai>=2.0
|
|
52
|
+
Requires-Dist: typer>=0.12
|
|
53
|
+
Requires-Dist: httpx>=0.24
|
|
54
|
+
Requires-Dist: requests>=2.28
|
|
55
|
+
Requires-Dist: aiofiles>=23.0
|
|
56
|
+
Provides-Extra: live
|
|
57
|
+
Requires-Dist: browser-use>=0.9.5; extra == "live"
|
|
58
|
+
Requires-Dist: playwright>=1.40; extra == "live"
|
|
59
|
+
Provides-Extra: dev
|
|
60
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
61
|
+
Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
|
|
62
|
+
Requires-Dist: pytest-cov>=4; extra == "dev"
|
|
63
|
+
Requires-Dist: black>=24; extra == "dev"
|
|
64
|
+
Requires-Dist: flake8>=6; extra == "dev"
|
|
65
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
66
|
+
Requires-Dist: twine>=4; extra == "dev"
|
|
67
|
+
Dynamic: license-file
|
|
68
|
+
|
|
69
|
+
# 🛡️ Survey Shield
|
|
70
|
+
|
|
71
|
+
Static review of online survey instruments for resistance to AI/bot respondents — plus an optional live runtime that drives a real browser through your survey.
|
|
72
|
+
|
|
73
|
+
## What is Survey Shield?
|
|
74
|
+
|
|
75
|
+
Survey Shield gives researchers feedback on whether their survey instrument is hardened against AI respondents. Two paths:
|
|
76
|
+
|
|
77
|
+
1. **Instrument Review** *(primary, static, no browser)* — point it at a Qualtrics `.qsf` export. Multi-agent LLM reviewers fan out across bot-resistance dimensions (attention checks, identity questions, visual-perceptual traps), produce a peer-review-style verdict with verbatim-grounded findings, and render a self-contained HTML report with a copy-paste Methods statement and APA/BibTeX citation.
|
|
78
|
+
2. **Take Survey (live runtime)** *(optional `[live]` extra)* — drives a real browser ([browser-use](https://docs.browser-use.com)) through a live Qualtrics URL, reports detected mechanisms after the fact. Costs ~5–10 minutes and real LLM credit per run, so the hosted demo doesn't expose it; install the `[live]` extra to run it locally.
|
|
79
|
+
|
|
80
|
+
Researchers using Survey Shield mostly want Instrument Review. Reach for the live runtime when you need to exercise the survey end-to-end.
|
|
81
|
+
|
|
82
|
+
## Install
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install surveyshield-py # static review only — small, no browser
|
|
86
|
+
pip install "surveyshield-py[live]" # adds browser-use + Playwright
|
|
87
|
+
playwright install chromium # only if you installed [live]
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The package name on PyPI is `surveyshield-py` (after [`openreview-py`](https://pypi.org/project/openreview-py/)); the import name is `surveyshield`.
|
|
91
|
+
|
|
92
|
+
Set an LLM provider key in your environment (or in a `.env` file in the working directory — Survey Shield loads it via `python-dotenv`):
|
|
93
|
+
|
|
94
|
+
```env
|
|
95
|
+
OPENAI_API_KEY=sk-... # used unless the model name starts with "gemini"
|
|
96
|
+
GOOGLE_API_KEY=... # used for Gemini models
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## CLI
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
surveyshield review your_survey.qsf
|
|
103
|
+
# → writes your_survey.report.html next to the input
|
|
104
|
+
|
|
105
|
+
surveyshield review your_survey.qsf --output report.html --json review.json \
|
|
106
|
+
--model gpt-4o-mini
|
|
107
|
+
|
|
108
|
+
surveyshield take https://qualtrics.com/jfe/form/SV_xxx # requires [live]
|
|
109
|
+
--model gemini-3-flash-preview --max-steps 150
|
|
110
|
+
|
|
111
|
+
surveyshield serve --host 127.0.0.1 --port 8000
|
|
112
|
+
# → boots the FastAPI app + bundled React SPA
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
`surveyshield --help` lists every command and flag.
|
|
116
|
+
|
|
117
|
+
## Python API
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
import asyncio
|
|
121
|
+
import surveyshield
|
|
122
|
+
|
|
123
|
+
review, parsed = asyncio.run(
|
|
124
|
+
surveyshield.review_qsf(
|
|
125
|
+
"your_survey.qsf",
|
|
126
|
+
model="gpt-4o-mini",
|
|
127
|
+
# api_key="sk-...", # or rely on env vars
|
|
128
|
+
# dimensions=["attention_checks"], # default = all
|
|
129
|
+
)
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
print(review.overall_score, review.overall_feedback.headline)
|
|
133
|
+
|
|
134
|
+
with open("report.html", "w") as f:
|
|
135
|
+
f.write(surveyshield.render_html(review, parsed))
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The `review` object is a `surveyshield.InstrumentReview` Pydantic model. Power users can compose the lower-level seams directly: `parse_qsf`, `run_review`, `drop_unverified_quotes`, `consolidate_and_summarize`, `aggregate`. See `surveyshield/__init__.py` for the public surface.
|
|
139
|
+
|
|
140
|
+
For live runtime:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
import asyncio, surveyshield # surveyshield-py[live] installed
|
|
144
|
+
|
|
145
|
+
result = asyncio.run(surveyshield.take_survey(
|
|
146
|
+
"https://qualtrics.com/jfe/form/SV_xxx",
|
|
147
|
+
model="gemini-3-flash-preview",
|
|
148
|
+
max_steps=150,
|
|
149
|
+
))
|
|
150
|
+
print(result.success_probability, [m.name for m in result.detected_mechanisms])
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
If the `[live]` extra isn't installed, `surveyshield.take_survey` resolves to `None` and the CLI's `take` command exits with a clear install hint.
|
|
154
|
+
|
|
155
|
+
## Self-host the hosted UI
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
git clone https://github.com/kiante-fernandez/survey-shield
|
|
159
|
+
cd survey-shield
|
|
160
|
+
./setup.sh # creates ../.conda env
|
|
161
|
+
echo "OPENAI_API_KEY=sk-..." > backend/.env # or GOOGLE_API_KEY
|
|
162
|
+
cd backend && ./start.sh # → http://localhost:8000
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
The `Take Survey` tab is gated on `live_take_enabled` — `GET /api/v1/survey/config` flips it to `true` once a key is detected in the env.
|
|
166
|
+
|
|
167
|
+
### Endpoints
|
|
168
|
+
|
|
169
|
+
- **Web UI**: http://localhost:8000
|
|
170
|
+
- **Interactive API docs**: http://localhost:8000/docs
|
|
171
|
+
- **Health check**: http://localhost:8000/health
|
|
172
|
+
- **Live-runtime config**: http://localhost:8000/api/v1/survey/config
|
|
173
|
+
|
|
174
|
+
### Models
|
|
175
|
+
|
|
176
|
+
The hosted UI does not expose a model picker — Instrument Review reviewers run on a sensible default (`gpt-4o-mini`). Self-hosters who want a different model can pass `model_name` directly to the API or CLI. The backend has no allowlist; any model name `langchain-openai`'s `ChatOpenAI` or `langchain-google-genai`'s `ChatGoogleGenerativeAI` accept will be routed by prefix:
|
|
177
|
+
|
|
178
|
+
- Names starting with `gemini` → Google (requires `GOOGLE_API_KEY`)
|
|
179
|
+
- Everything else → OpenAI (requires `OPENAI_API_KEY`)
|
|
180
|
+
|
|
181
|
+
## API usage (self-host)
|
|
182
|
+
|
|
183
|
+
### Instrument Review (primary)
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
# Submit a QSF for review
|
|
187
|
+
curl -F "file=@your_survey.qsf" \
|
|
188
|
+
http://localhost:8000/api/v1/instrument/review
|
|
189
|
+
# → {"review_id": "<uuid>", "status": "queued", ...}
|
|
190
|
+
|
|
191
|
+
# Poll
|
|
192
|
+
curl http://localhost:8000/api/v1/instrument/status/<uuid>
|
|
193
|
+
# queued → running → completed (~30–90 s)
|
|
194
|
+
|
|
195
|
+
# Structured JSON
|
|
196
|
+
curl http://localhost:8000/api/v1/instrument/results/<uuid>
|
|
197
|
+
|
|
198
|
+
# Human-readable HTML report
|
|
199
|
+
curl "http://localhost:8000/api/v1/instrument/report/<uuid>"
|
|
200
|
+
|
|
201
|
+
# Download as a file
|
|
202
|
+
curl -OJ "http://localhost:8000/api/v1/instrument/report/<uuid>?download=1"
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Live runtime (self-host only)
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
curl -X POST "http://localhost:8000/api/v1/survey/analyze" \
|
|
209
|
+
-H "Content-Type: application/json" \
|
|
210
|
+
-d '{
|
|
211
|
+
"survey_url": "https://example.com/survey",
|
|
212
|
+
"model_name": "gpt-4o-mini",
|
|
213
|
+
"max_steps": 150,
|
|
214
|
+
"use_vision": true
|
|
215
|
+
}'
|
|
216
|
+
# Then poll /api/v1/survey/status/<id> and fetch /api/v1/survey/results/<id>.
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
## What Survey Shield evaluates
|
|
220
|
+
|
|
221
|
+
### Instrument Review dimensions (primary)
|
|
222
|
+
|
|
223
|
+
The reviewer fans out across plug-in dimensions defined in [`surveyshield/review/dimensions.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/review/dimensions.py). v1 ships three:
|
|
224
|
+
|
|
225
|
+
- **`attention_checks`** — explicit IMCs and instructional manipulation checks (Westwood, 2025; PNAS).
|
|
226
|
+
- **`identity_questions`** — direct LLM-resistance items (identity probes, reverse-shibboleth questions, impossible-event questions).
|
|
227
|
+
- **`visual_perceptual_traps`** — image- and layout-based cognitive traps that exploit vision-language model architectural constraints (Affonso, 2026; JCR).
|
|
228
|
+
|
|
229
|
+
Adding a new dimension is one entry in `dimensions.py` plus a Pydantic output schema — the reviewer fan-out, aggregator, and HTML report are all plug-in-driven.
|
|
230
|
+
|
|
231
|
+
Findings are **grounded**: every reviewer finding must cite a verbatim excerpt from the source survey. Survey Shield substring-checks each excerpt against the parsed survey and drops anything that can't be located. What reaches the report is grounded in real survey content.
|
|
232
|
+
|
|
233
|
+
The product is scoped strictly to **bot resistance**. We do not critique a survey's substantive research design, theoretical framing, or question wording — those remain the researcher's domain.
|
|
234
|
+
|
|
235
|
+
### Live-runtime mechanisms
|
|
236
|
+
|
|
237
|
+
When you run a live analysis against a real Qualtrics URL, Survey Shield's browser-use Agent navigates the survey end-to-end and inventories detected mechanisms by category — hover traps, invisible text, timing requirements, CAPTCHAs, honeypot fields, attention checks, and behavioural / mouse-tracking signals. Per-mechanism severity weights (1–10) feed into success-probability and difficulty scores. See [`surveyshield/live/analyzer.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/analyzer.py) for the taxonomy and [`surveyshield/live/prompts.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/prompts.py) for the agent task.
|
|
238
|
+
|
|
239
|
+
## Development
|
|
240
|
+
|
|
241
|
+
### Project structure
|
|
242
|
+
|
|
243
|
+
```
|
|
244
|
+
survey-shield/
|
|
245
|
+
├── surveyshield/ # the importable package
|
|
246
|
+
│ ├── __init__.py # public API (review_qsf, render_html, take_survey, …)
|
|
247
|
+
│ ├── cli.py # Typer CLI (review / take / serve)
|
|
248
|
+
│ ├── models/ # Pydantic schemas
|
|
249
|
+
│ ├── review/ # static review pipeline
|
|
250
|
+
│ │ ├── parser.py # QSF → ParsedSurvey
|
|
251
|
+
│ │ ├── dimensions.py # plug-in dimension registry
|
|
252
|
+
│ │ ├── reviewer.py # fan-out, verify, consolidate, aggregate
|
|
253
|
+
│ │ ├── mechanism_context.py
|
|
254
|
+
│ │ └── templates/ # Jinja2 self-contained HTML report
|
|
255
|
+
│ ├── live/ # browser-use runtime ([live] extra)
|
|
256
|
+
│ │ ├── analyzer.py # SurveyAnalyzer / take_survey
|
|
257
|
+
│ │ ├── prompts.py
|
|
258
|
+
│ │ └── patches.py
|
|
259
|
+
│ └── serve/ # FastAPI app + bundled React SPA
|
|
260
|
+
│ ├── app.py
|
|
261
|
+
│ ├── config.py
|
|
262
|
+
│ ├── api/{survey,instrument}.py
|
|
263
|
+
│ └── static/ # built React (populated by bin/build.sh)
|
|
264
|
+
├── frontend/ # React/TypeScript source (CRA)
|
|
265
|
+
├── tests/ # pytest suite + tiny QSF fixture
|
|
266
|
+
├── backend/start.sh # convenience wrapper around `surveyshield serve`
|
|
267
|
+
├── pyproject.toml # canonical package metadata
|
|
268
|
+
├── Procfile # web: gunicorn surveyshield.serve.app:app
|
|
269
|
+
├── bin/build.sh # React build → surveyshield/serve/static/
|
|
270
|
+
└── .github/workflows/ # test.yml + release.yml (PyPI trusted publishing)
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
### Local dev
|
|
274
|
+
|
|
275
|
+
```bash
|
|
276
|
+
./setup.sh # one-time: conda env at ../.conda
|
|
277
|
+
pip install -e ".[dev,live]" # editable install + tests + browser-use
|
|
278
|
+
pytest -q # ~30 tests, no LLM calls
|
|
279
|
+
cd frontend && npx tsc --noEmit && npm run build
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
### Releasing
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
git tag v0.1.0 && git push --tags
|
|
286
|
+
# .github/workflows/release.yml builds the wheel + sdist (with the React SPA
|
|
287
|
+
# bundled into surveyshield/serve/static/) and publishes to PyPI via OIDC.
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
The PyPI project must be configured with this repo + `release.yml` as a Trusted Publisher before the first push.
|
|
291
|
+
|
|
292
|
+
## Contributing
|
|
293
|
+
|
|
294
|
+
1. Fork the repository
|
|
295
|
+
2. Create a feature branch
|
|
296
|
+
3. Make changes with tests
|
|
297
|
+
4. Submit a pull request
|
|
298
|
+
|
|
299
|
+
## License
|
|
300
|
+
|
|
301
|
+
MIT License — see [LICENSE](https://github.com/kiante-fernandez/survey-shield/blob/main/LICENSE).
|
|
302
|
+
|
|
303
|
+
## Citation
|
|
304
|
+
|
|
305
|
+
If you use Survey Shield in published work:
|
|
306
|
+
|
|
307
|
+
```bibtex
|
|
308
|
+
@misc{fernandez2026surveyshield,
|
|
309
|
+
author = {Fernandez, K. and Low, A. and Bogard, J. and Fox, C. R.},
|
|
310
|
+
title = {Survey Shield: Static review of online survey instruments for resistance to non-human responses},
|
|
311
|
+
year = {2026},
|
|
312
|
+
note = {Manuscript in preparation},
|
|
313
|
+
}
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
Every report includes the same citation pre-formatted (APA + BibTeX).
|
|
317
|
+
|
|
318
|
+
## Disclaimer
|
|
319
|
+
|
|
320
|
+
Survey Shield is intended for research and testing purposes.
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
# 🛡️ Survey Shield
|
|
2
|
+
|
|
3
|
+
Static review of online survey instruments for resistance to AI/bot respondents — plus an optional live runtime that drives a real browser through your survey.
|
|
4
|
+
|
|
5
|
+
## What is Survey Shield?
|
|
6
|
+
|
|
7
|
+
Survey Shield gives researchers feedback on whether their survey instrument is hardened against AI respondents. Two paths:
|
|
8
|
+
|
|
9
|
+
1. **Instrument Review** *(primary, static, no browser)* — point it at a Qualtrics `.qsf` export. Multi-agent LLM reviewers fan out across bot-resistance dimensions (attention checks, identity questions, visual-perceptual traps), produce a peer-review-style verdict with verbatim-grounded findings, and render a self-contained HTML report with a copy-paste Methods statement and APA/BibTeX citation.
|
|
10
|
+
2. **Take Survey (live runtime)** *(optional `[live]` extra)* — drives a real browser ([browser-use](https://docs.browser-use.com)) through a live Qualtrics URL, reports detected mechanisms after the fact. Costs ~5–10 minutes and real LLM credit per run, so the hosted demo doesn't expose it; install the `[live]` extra to run it locally.
|
|
11
|
+
|
|
12
|
+
Researchers using Survey Shield mostly want Instrument Review. Reach for the live runtime when you need to exercise the survey end-to-end.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install surveyshield-py # static review only — small, no browser
|
|
18
|
+
pip install "surveyshield-py[live]" # adds browser-use + Playwright
|
|
19
|
+
playwright install chromium # only if you installed [live]
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The package name on PyPI is `surveyshield-py` (after [`openreview-py`](https://pypi.org/project/openreview-py/)); the import name is `surveyshield`.
|
|
23
|
+
|
|
24
|
+
Set an LLM provider key in your environment (or in a `.env` file in the working directory — Survey Shield loads it via `python-dotenv`):
|
|
25
|
+
|
|
26
|
+
```env
|
|
27
|
+
OPENAI_API_KEY=sk-... # used unless the model name starts with "gemini"
|
|
28
|
+
GOOGLE_API_KEY=... # used for Gemini models
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## CLI
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
surveyshield review your_survey.qsf
|
|
35
|
+
# → writes your_survey.report.html next to the input
|
|
36
|
+
|
|
37
|
+
surveyshield review your_survey.qsf --output report.html --json review.json \
|
|
38
|
+
--model gpt-4o-mini
|
|
39
|
+
|
|
40
|
+
surveyshield take https://qualtrics.com/jfe/form/SV_xxx # requires [live]
|
|
41
|
+
--model gemini-3-flash-preview --max-steps 150
|
|
42
|
+
|
|
43
|
+
surveyshield serve --host 127.0.0.1 --port 8000
|
|
44
|
+
# → boots the FastAPI app + bundled React SPA
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
`surveyshield --help` lists every command and flag.
|
|
48
|
+
|
|
49
|
+
## Python API
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import asyncio
|
|
53
|
+
import surveyshield
|
|
54
|
+
|
|
55
|
+
review, parsed = asyncio.run(
|
|
56
|
+
surveyshield.review_qsf(
|
|
57
|
+
"your_survey.qsf",
|
|
58
|
+
model="gpt-4o-mini",
|
|
59
|
+
# api_key="sk-...", # or rely on env vars
|
|
60
|
+
# dimensions=["attention_checks"], # default = all
|
|
61
|
+
)
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
print(review.overall_score, review.overall_feedback.headline)
|
|
65
|
+
|
|
66
|
+
with open("report.html", "w") as f:
|
|
67
|
+
f.write(surveyshield.render_html(review, parsed))
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The `review` object is a `surveyshield.InstrumentReview` Pydantic model. Power users can compose the lower-level seams directly: `parse_qsf`, `run_review`, `drop_unverified_quotes`, `consolidate_and_summarize`, `aggregate`. See `surveyshield/__init__.py` for the public surface.
|
|
71
|
+
|
|
72
|
+
For live runtime:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
import asyncio, surveyshield # surveyshield-py[live] installed
|
|
76
|
+
|
|
77
|
+
result = asyncio.run(surveyshield.take_survey(
|
|
78
|
+
"https://qualtrics.com/jfe/form/SV_xxx",
|
|
79
|
+
model="gemini-3-flash-preview",
|
|
80
|
+
max_steps=150,
|
|
81
|
+
))
|
|
82
|
+
print(result.success_probability, [m.name for m in result.detected_mechanisms])
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
If the `[live]` extra isn't installed, `surveyshield.take_survey` resolves to `None` and the CLI's `take` command exits with a clear install hint.
|
|
86
|
+
|
|
87
|
+
## Self-host the hosted UI
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
git clone https://github.com/kiante-fernandez/survey-shield
|
|
91
|
+
cd survey-shield
|
|
92
|
+
./setup.sh # creates ../.conda env
|
|
93
|
+
echo "OPENAI_API_KEY=sk-..." > backend/.env # or GOOGLE_API_KEY
|
|
94
|
+
cd backend && ./start.sh # → http://localhost:8000
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
The `Take Survey` tab is gated on `live_take_enabled` — `GET /api/v1/survey/config` flips it to `true` once a key is detected in the env.
|
|
98
|
+
|
|
99
|
+
### Endpoints
|
|
100
|
+
|
|
101
|
+
- **Web UI**: http://localhost:8000
|
|
102
|
+
- **Interactive API docs**: http://localhost:8000/docs
|
|
103
|
+
- **Health check**: http://localhost:8000/health
|
|
104
|
+
- **Live-runtime config**: http://localhost:8000/api/v1/survey/config
|
|
105
|
+
|
|
106
|
+
### Models
|
|
107
|
+
|
|
108
|
+
The hosted UI does not expose a model picker — Instrument Review reviewers run on a sensible default (`gpt-4o-mini`). Self-hosters who want a different model can pass `model_name` directly to the API or CLI. The backend has no allowlist; any model name `langchain-openai`'s `ChatOpenAI` or `langchain-google-genai`'s `ChatGoogleGenerativeAI` accept will be routed by prefix:
|
|
109
|
+
|
|
110
|
+
- Names starting with `gemini` → Google (requires `GOOGLE_API_KEY`)
|
|
111
|
+
- Everything else → OpenAI (requires `OPENAI_API_KEY`)
|
|
112
|
+
|
|
113
|
+
## API usage (self-host)
|
|
114
|
+
|
|
115
|
+
### Instrument Review (primary)
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
# Submit a QSF for review
|
|
119
|
+
curl -F "file=@your_survey.qsf" \
|
|
120
|
+
http://localhost:8000/api/v1/instrument/review
|
|
121
|
+
# → {"review_id": "<uuid>", "status": "queued", ...}
|
|
122
|
+
|
|
123
|
+
# Poll
|
|
124
|
+
curl http://localhost:8000/api/v1/instrument/status/<uuid>
|
|
125
|
+
# queued → running → completed (~30–90 s)
|
|
126
|
+
|
|
127
|
+
# Structured JSON
|
|
128
|
+
curl http://localhost:8000/api/v1/instrument/results/<uuid>
|
|
129
|
+
|
|
130
|
+
# Human-readable HTML report
|
|
131
|
+
curl "http://localhost:8000/api/v1/instrument/report/<uuid>"
|
|
132
|
+
|
|
133
|
+
# Download as a file
|
|
134
|
+
curl -OJ "http://localhost:8000/api/v1/instrument/report/<uuid>?download=1"
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Live runtime (self-host only)
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
curl -X POST "http://localhost:8000/api/v1/survey/analyze" \
|
|
141
|
+
-H "Content-Type: application/json" \
|
|
142
|
+
-d '{
|
|
143
|
+
"survey_url": "https://example.com/survey",
|
|
144
|
+
"model_name": "gpt-4o-mini",
|
|
145
|
+
"max_steps": 150,
|
|
146
|
+
"use_vision": true
|
|
147
|
+
}'
|
|
148
|
+
# Then poll /api/v1/survey/status/<id> and fetch /api/v1/survey/results/<id>.
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## What Survey Shield evaluates
|
|
152
|
+
|
|
153
|
+
### Instrument Review dimensions (primary)
|
|
154
|
+
|
|
155
|
+
The reviewer fans out across plug-in dimensions defined in [`surveyshield/review/dimensions.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/review/dimensions.py). v1 ships three:
|
|
156
|
+
|
|
157
|
+
- **`attention_checks`** — explicit IMCs and instructional manipulation checks (Westwood, 2025; PNAS).
|
|
158
|
+
- **`identity_questions`** — direct LLM-resistance items (identity probes, reverse-shibboleth questions, impossible-event questions).
|
|
159
|
+
- **`visual_perceptual_traps`** — image- and layout-based cognitive traps that exploit vision-language model architectural constraints (Affonso, 2026; JCR).
|
|
160
|
+
|
|
161
|
+
Adding a new dimension is one entry in `dimensions.py` plus a Pydantic output schema — the reviewer fan-out, aggregator, and HTML report are all plug-in-driven.
|
|
162
|
+
|
|
163
|
+
Findings are **grounded**: every reviewer finding must cite a verbatim excerpt from the source survey. Survey Shield substring-checks each excerpt against the parsed survey and drops anything that can't be located. What reaches the report is grounded in real survey content.
|
|
164
|
+
|
|
165
|
+
The product is scoped strictly to **bot resistance**. We do not critique a survey's substantive research design, theoretical framing, or question wording — those remain the researcher's domain.
|
|
166
|
+
|
|
167
|
+
### Live-runtime mechanisms
|
|
168
|
+
|
|
169
|
+
When you run a live analysis against a real Qualtrics URL, Survey Shield's browser-use Agent navigates the survey end-to-end and inventories detected mechanisms by category — hover traps, invisible text, timing requirements, CAPTCHAs, honeypot fields, attention checks, and behavioural / mouse-tracking signals. Per-mechanism severity weights (1–10) feed into success-probability and difficulty scores. See [`surveyshield/live/analyzer.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/analyzer.py) for the taxonomy and [`surveyshield/live/prompts.py`](https://github.com/kiante-fernandez/survey-shield/blob/main/surveyshield/live/prompts.py) for the agent task.
|
|
170
|
+
|
|
171
|
+
## Development
|
|
172
|
+
|
|
173
|
+
### Project structure
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
survey-shield/
|
|
177
|
+
├── surveyshield/ # the importable package
|
|
178
|
+
│ ├── __init__.py # public API (review_qsf, render_html, take_survey, …)
|
|
179
|
+
│ ├── cli.py # Typer CLI (review / take / serve)
|
|
180
|
+
│ ├── models/ # Pydantic schemas
|
|
181
|
+
│ ├── review/ # static review pipeline
|
|
182
|
+
│ │ ├── parser.py # QSF → ParsedSurvey
|
|
183
|
+
│ │ ├── dimensions.py # plug-in dimension registry
|
|
184
|
+
│ │ ├── reviewer.py # fan-out, verify, consolidate, aggregate
|
|
185
|
+
│ │ ├── mechanism_context.py
|
|
186
|
+
│ │ └── templates/ # Jinja2 self-contained HTML report
|
|
187
|
+
│ ├── live/ # browser-use runtime ([live] extra)
|
|
188
|
+
│ │ ├── analyzer.py # SurveyAnalyzer / take_survey
|
|
189
|
+
│ │ ├── prompts.py
|
|
190
|
+
│ │ └── patches.py
|
|
191
|
+
│ └── serve/ # FastAPI app + bundled React SPA
|
|
192
|
+
│ ├── app.py
|
|
193
|
+
│ ├── config.py
|
|
194
|
+
│ ├── api/{survey,instrument}.py
|
|
195
|
+
│ └── static/ # built React (populated by bin/build.sh)
|
|
196
|
+
├── frontend/ # React/TypeScript source (CRA)
|
|
197
|
+
├── tests/ # pytest suite + tiny QSF fixture
|
|
198
|
+
├── backend/start.sh # convenience wrapper around `surveyshield serve`
|
|
199
|
+
├── pyproject.toml # canonical package metadata
|
|
200
|
+
├── Procfile # web: gunicorn surveyshield.serve.app:app
|
|
201
|
+
├── bin/build.sh # React build → surveyshield/serve/static/
|
|
202
|
+
└── .github/workflows/ # test.yml + release.yml (PyPI trusted publishing)
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Local dev
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
./setup.sh # one-time: conda env at ../.conda
|
|
209
|
+
pip install -e ".[dev,live]" # editable install + tests + browser-use
|
|
210
|
+
pytest -q # ~30 tests, no LLM calls
|
|
211
|
+
cd frontend && npx tsc --noEmit && npm run build
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### Releasing
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
git tag v0.1.0 && git push --tags
|
|
218
|
+
# .github/workflows/release.yml builds the wheel + sdist (with the React SPA
|
|
219
|
+
# bundled into surveyshield/serve/static/) and publishes to PyPI via OIDC.
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
The PyPI project must be configured with this repo + `release.yml` as a Trusted Publisher before the first push.
|
|
223
|
+
|
|
224
|
+
## Contributing
|
|
225
|
+
|
|
226
|
+
1. Fork the repository
|
|
227
|
+
2. Create a feature branch
|
|
228
|
+
3. Make changes with tests
|
|
229
|
+
4. Submit a pull request
|
|
230
|
+
|
|
231
|
+
## License
|
|
232
|
+
|
|
233
|
+
MIT License — see [LICENSE](https://github.com/kiante-fernandez/survey-shield/blob/main/LICENSE).
|
|
234
|
+
|
|
235
|
+
## Citation
|
|
236
|
+
|
|
237
|
+
If you use Survey Shield in published work:
|
|
238
|
+
|
|
239
|
+
```bibtex
|
|
240
|
+
@misc{fernandez2026surveyshield,
|
|
241
|
+
author = {Fernandez, K. and Low, A. and Bogard, J. and Fox, C. R.},
|
|
242
|
+
title = {Survey Shield: Static review of online survey instruments for resistance to non-human responses},
|
|
243
|
+
year = {2026},
|
|
244
|
+
note = {Manuscript in preparation},
|
|
245
|
+
}
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Every report includes the same citation pre-formatted (APA + BibTeX).
|
|
249
|
+
|
|
250
|
+
## Disclaimer
|
|
251
|
+
|
|
252
|
+
Survey Shield is intended for research and testing purposes.
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "surveyshield-py"
|
|
3
|
+
description = "Static review of online survey instruments for resistance to AI/bot respondents"
|
|
4
|
+
readme = "README.md"
|
|
5
|
+
license = { file = "LICENSE" }
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "Kianté Fernandez" },
|
|
8
|
+
{ name = "Andrew Low" },
|
|
9
|
+
{ name = "Jonathan Bogard" },
|
|
10
|
+
{ name = "Craig R. Fox" },
|
|
11
|
+
]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
dynamic = ["version"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3.10",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Topic :: Scientific/Engineering",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
dependencies = [
|
|
25
|
+
"fastapi>=0.110",
|
|
26
|
+
"uvicorn[standard]>=0.27",
|
|
27
|
+
"gunicorn>=21.0",
|
|
28
|
+
"python-multipart>=0.0.7",
|
|
29
|
+
"jinja2>=3.1",
|
|
30
|
+
"pydantic>=2.5",
|
|
31
|
+
"python-dotenv>=1.0",
|
|
32
|
+
"openai>=1.0",
|
|
33
|
+
"langchain-core>=0.3",
|
|
34
|
+
"langchain-openai>=0.2",
|
|
35
|
+
"langchain-google-genai>=2.0",
|
|
36
|
+
"typer>=0.12",
|
|
37
|
+
"httpx>=0.24",
|
|
38
|
+
"requests>=2.28",
|
|
39
|
+
"aiofiles>=23.0",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[project.optional-dependencies]
|
|
43
|
+
live = [
|
|
44
|
+
"browser-use>=0.9.5",
|
|
45
|
+
"playwright>=1.40",
|
|
46
|
+
]
|
|
47
|
+
dev = [
|
|
48
|
+
"pytest>=7",
|
|
49
|
+
"pytest-asyncio>=0.21",
|
|
50
|
+
"pytest-cov>=4",
|
|
51
|
+
"black>=24",
|
|
52
|
+
"flake8>=6",
|
|
53
|
+
"build>=1.0",
|
|
54
|
+
"twine>=4",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[project.scripts]
|
|
58
|
+
surveyshield = "surveyshield.cli:app"
|
|
59
|
+
|
|
60
|
+
[project.urls]
|
|
61
|
+
Homepage = "https://github.com/kiante-fernandez/survey-shield"
|
|
62
|
+
Repository = "https://github.com/kiante-fernandez/survey-shield"
|
|
63
|
+
Issues = "https://github.com/kiante-fernandez/survey-shield/issues"
|
|
64
|
+
|
|
65
|
+
[build-system]
|
|
66
|
+
requires = ["setuptools>=68", "wheel"]
|
|
67
|
+
build-backend = "setuptools.build_meta"
|
|
68
|
+
|
|
69
|
+
[tool.setuptools.dynamic]
|
|
70
|
+
version = { attr = "surveyshield._version.__version__" }
|
|
71
|
+
|
|
72
|
+
[tool.setuptools.packages.find]
|
|
73
|
+
where = ["."]
|
|
74
|
+
include = ["surveyshield*"]
|
|
75
|
+
exclude = ["backend*", "frontend*", "tests*", "example_qsf*", "bin*"]
|
|
76
|
+
|
|
77
|
+
[tool.setuptools.package-data]
|
|
78
|
+
surveyshield = [
|
|
79
|
+
"review/templates/*.html",
|
|
80
|
+
"serve/static/**/*",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
[tool.pytest.ini_options]
|
|
84
|
+
testpaths = ["tests"]
|
|
85
|
+
asyncio_mode = "auto"
|