survey-scribe 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- survey_scribe-0.1.0/.gitignore +73 -0
- survey_scribe-0.1.0/CHANGELOG.md +47 -0
- survey_scribe-0.1.0/LICENSE +21 -0
- survey_scribe-0.1.0/PKG-INFO +200 -0
- survey_scribe-0.1.0/README.md +157 -0
- survey_scribe-0.1.0/pyproject.toml +184 -0
- survey_scribe-0.1.0/src/survey_scribe/__init__.py +146 -0
- survey_scribe-0.1.0/src/survey_scribe/cli.py +539 -0
- survey_scribe-0.1.0/src/survey_scribe/client.py +467 -0
- survey_scribe-0.1.0/src/survey_scribe/config.py +482 -0
- survey_scribe-0.1.0/src/survey_scribe/errors.py +196 -0
- survey_scribe-0.1.0/src/survey_scribe/models/__init__.py +79 -0
- survey_scribe-0.1.0/src/survey_scribe/models/routing.py +1252 -0
- survey_scribe-0.1.0/src/survey_scribe/models/svis.py +126 -0
- survey_scribe-0.1.0/src/survey_scribe/pipeline.py +1234 -0
- survey_scribe-0.1.0/src/survey_scribe/providers/__init__.py +26 -0
- survey_scribe-0.1.0/src/survey_scribe/providers/anthropic.py +98 -0
- survey_scribe-0.1.0/src/survey_scribe/providers/azure.py +278 -0
- survey_scribe-0.1.0/src/survey_scribe/providers/base.py +309 -0
- survey_scribe-0.1.0/src/survey_scribe/providers/capabilities.py +169 -0
- survey_scribe-0.1.0/src/survey_scribe/providers/openai_compatible.py +532 -0
- survey_scribe-0.1.0/src/survey_scribe/providers/testing.py +225 -0
- survey_scribe-0.1.0/src/survey_scribe/py.typed +0 -0
- survey_scribe-0.1.0/src/survey_scribe/results.py +311 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/__init__.py +183 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/algorithms.py +119 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/config.py +5 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/contracts.py +479 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/diagnostics.py +147 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/extraction.py +1241 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/identity.py +519 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/inventory.py +335 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/native.py +432 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/normalization.py +58 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/pipeline.py +589 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/prompts.py +544 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/reconcile.py +1211 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/review.py +309 -0
- survey_scribe-0.1.0/src/survey_scribe/routing/validate.py +1040 -0
- survey_scribe-0.1.0/src/survey_scribe/serialization/__init__.py +28 -0
- survey_scribe-0.1.0/src/survey_scribe/serialization/artifacts.py +1727 -0
- survey_scribe-0.1.0/src/survey_scribe/serialization/legacy.py +63 -0
- survey_scribe-0.1.0/src/survey_scribe/serialization/routing.py +158 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/__init__.py +55 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/base.py +649 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/chunking.py +372 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/docling.py +1384 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/ocr.py +268 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/registry.py +355 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/tabular.py +338 -0
- survey_scribe-0.1.0/src/survey_scribe/sources/xlsform.py +1160 -0
- survey_scribe-0.1.0/uv.lock +4694 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.pyo
|
|
5
|
+
.venv/
|
|
6
|
+
venv/
|
|
7
|
+
env/
|
|
8
|
+
*.egg-info/
|
|
9
|
+
dist/
|
|
10
|
+
build/
|
|
11
|
+
site/
|
|
12
|
+
|
|
13
|
+
# Environment variables - NEVER commit these files
|
|
14
|
+
.env
|
|
15
|
+
.env.*
|
|
16
|
+
!.env.example
|
|
17
|
+
|
|
18
|
+
# Generated outputs
|
|
19
|
+
output/*
|
|
20
|
+
!output/.gitkeep
|
|
21
|
+
survey-scribe.toml
|
|
22
|
+
**/.survey-scribe/
|
|
23
|
+
*_svis.json
|
|
24
|
+
!tests/fixtures/**/*_svis.json
|
|
25
|
+
craft_mlt_25k.zip
|
|
26
|
+
english_g2.zip
|
|
27
|
+
|
|
28
|
+
# Test samples- questionnaire PDFs may be restricted data
|
|
29
|
+
tests/samples/*
|
|
30
|
+
!tests/samples/.gitkeep
|
|
31
|
+
|
|
32
|
+
# IDE
|
|
33
|
+
.vscode/
|
|
34
|
+
.idea/
|
|
35
|
+
*.swp
|
|
36
|
+
|
|
37
|
+
# OS
|
|
38
|
+
.DS_Store
|
|
39
|
+
Thumbs.db
|
|
40
|
+
|
|
41
|
+
# Jupyter
|
|
42
|
+
.ipynb_checkpoints/
|
|
43
|
+
*.ipynb
|
|
44
|
+
|
|
45
|
+
# Local uv configuration
|
|
46
|
+
uv.toml
|
|
47
|
+
|
|
48
|
+
# Test and tooling caches
|
|
49
|
+
.pytest_cache/
|
|
50
|
+
.ruff_cache/
|
|
51
|
+
.coverage
|
|
52
|
+
coverage.xml
|
|
53
|
+
htmlcov/
|
|
54
|
+
.cache/
|
|
55
|
+
|
|
56
|
+
# Quarto book
|
|
57
|
+
book/_book/
|
|
58
|
+
book/.quarto/
|
|
59
|
+
|
|
60
|
+
/.quarto/
|
|
61
|
+
**/*.quarto_ipynb
|
|
62
|
+
|
|
63
|
+
tmp/
|
|
64
|
+
|
|
65
|
+
# Compound GPID managed items (junctions + copied file - do not commit)
|
|
66
|
+
.compound-gpid/managed-files.json
|
|
67
|
+
.kilo/AGENTS.md
|
|
68
|
+
.kilo/agents
|
|
69
|
+
.kilo/commands
|
|
70
|
+
.kilo/instructions
|
|
71
|
+
.kilo/kilo.json
|
|
72
|
+
.kilo/shared
|
|
73
|
+
.kilo/skills
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to Survey Scribe are documented in this file. The format is
|
|
4
|
+
based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the
|
|
5
|
+
project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- PyPI-ready project metadata and MIT license declaration.
|
|
12
|
+
- MkDocs documentation with installation, usage, examples, and API reference.
|
|
13
|
+
- Coverage enforcement and release-artifact validation in CI.
|
|
14
|
+
- Additive routed SVIS models, deterministic directed-multigraph validation,
|
|
15
|
+
source-grounded evidence, and append-only discrepancy review.
|
|
16
|
+
- `QuestionnaireRouter` with native XLSForm routing and structured-provider integration.
|
|
17
|
+
- Deterministic routing-quality evaluation, routing-schema export, and routing documentation.
|
|
18
|
+
- Validated static metadata headers and per-attempt auxiliary secret headers for
|
|
19
|
+
direct `AzureOpenAIProvider` injection through compatible gateways.
|
|
20
|
+
- Extraction-first public guides for completed questionnaires, skip patterns,
|
|
21
|
+
Palantir Foundry, Microsoft Foundry, mAI Factory, and AI providers.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- Runtime dependencies now use compatible ranges while the committed `uv.lock`
|
|
26
|
+
retains exact engineering versions.
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
|
|
30
|
+
- Native XLSForm `source_format="xlsform"` now binds to its validated XLSX
|
|
31
|
+
snapshot for provider-free questionnaire routing.
|
|
32
|
+
- Public installation guidance no longer presents an unavailable PyPI release or
|
|
33
|
+
a stale source revision as the primary installation path.
|
|
34
|
+
|
|
35
|
+
## 0.1.0 - Unpublished
|
|
36
|
+
|
|
37
|
+
This version remains a source-tree milestone. No approved package-index or GitHub
|
|
38
|
+
release is available.
|
|
39
|
+
|
|
40
|
+
### Added
|
|
41
|
+
|
|
42
|
+
- Installable `survey-scribe` package with typed SVIS Pydantic models.
|
|
43
|
+
- Bootstrap `survey-scribe` command.
|
|
44
|
+
- Legacy schema re-export and characterization suite.
|
|
45
|
+
- Cross-platform Python 3.11-3.13 CI and clean-wheel installation checks.
|
|
46
|
+
|
|
47
|
+
[Unreleased]: https://github.com/GMD-hub/survey-scribe/commits/main
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Survey Scribe contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: survey-scribe
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local-first questionnaire extraction to the Survey Variable Information Schema
|
|
5
|
+
Project-URL: Homepage, https://github.com/GMD-hub/survey-scribe
|
|
6
|
+
Project-URL: Documentation, https://gmd-hub.github.io/survey-scribe/
|
|
7
|
+
Project-URL: Repository, https://github.com/GMD-hub/survey-scribe.git
|
|
8
|
+
Project-URL: Issues, https://github.com/GMD-hub/survey-scribe/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/GMD-hub/survey-scribe/blob/main/CHANGELOG.md
|
|
10
|
+
Author: World Bank Global Monitoring Database team
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: SVIS,data extraction,household surveys,metadata,pydantic,questionnaires,schema,structured data,survey harmonization,survey metadata
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: <3.14,>=3.11
|
|
25
|
+
Requires-Dist: defusedxml<1,>=0.7.1
|
|
26
|
+
Requires-Dist: pydantic<3,>=2.11.7
|
|
27
|
+
Provides-Extra: anthropic
|
|
28
|
+
Requires-Dist: anthropic<1,>=0.64.0; extra == 'anthropic'
|
|
29
|
+
Requires-Dist: instructor<2,>=1.10.0; extra == 'anthropic'
|
|
30
|
+
Provides-Extra: openai
|
|
31
|
+
Requires-Dist: instructor<2,>=1.10.0; extra == 'openai'
|
|
32
|
+
Requires-Dist: openai<2,>=1.99.9; extra == 'openai'
|
|
33
|
+
Requires-Dist: tenacity<10,>=9.1.2; extra == 'openai'
|
|
34
|
+
Requires-Dist: tiktoken<1,>=0.11.0; extra == 'openai'
|
|
35
|
+
Provides-Extra: pdf
|
|
36
|
+
Requires-Dist: docling<3,>=2.125.0; extra == 'pdf'
|
|
37
|
+
Requires-Dist: easyocr<2,>=1.7.2; extra == 'pdf'
|
|
38
|
+
Requires-Dist: lingua-language-detector<3,>=2.1.1; extra == 'pdf'
|
|
39
|
+
Requires-Dist: pymupdf<2,>=1.26.4; extra == 'pdf'
|
|
40
|
+
Provides-Extra: tabular
|
|
41
|
+
Requires-Dist: openpyxl<4,>=3.1.5; extra == 'tabular'
|
|
42
|
+
Description-Content-Type: text/markdown
|
|
43
|
+
|
|
44
|
+
# Survey Scribe
|
|
45
|
+
|
|
46
|
+
[](https://github.com/GMD-hub/survey-scribe/actions/workflows/ci.yml)
|
|
47
|
+
[](https://www.python.org/)
|
|
48
|
+
[](LICENSE)
|
|
49
|
+
|
|
50
|
+
Survey Scribe converts local survey questionnaires to the typed Survey Variable
|
|
51
|
+
Information Schema (SVIS). It provides synchronous, asynchronous, and batch APIs,
|
|
52
|
+
provider adapters, safe local source normalization, deterministic chunking,
|
|
53
|
+
questionnaire routing graphs, secure configuration, and versioned artifacts.
|
|
54
|
+
|
|
55
|
+
> **Alpha status:** The source tree declares version `0.1.0` and contains the
|
|
56
|
+
> public `SurveyScribe` API, conversion CLI, typed models, source and provider
|
|
57
|
+
> adapters, transactional artifacts, and `QuestionnaireRouter`. No approved PyPI
|
|
58
|
+
> release is currently available.
|
|
59
|
+
|
|
60
|
+
## Features
|
|
61
|
+
|
|
62
|
+
- Typed SVIS models with Pydantic validation and JSON serialization.
|
|
63
|
+
- Stable top-level imports for schema consumers.
|
|
64
|
+
- Numeric, categorical, text, date, and other variable classifications.
|
|
65
|
+
- Missing-category, source-provenance, confidence, and review metadata.
|
|
66
|
+
- Local PDF, DOCX, XLSX, CSV, HTML, Markdown, and text normalization.
|
|
67
|
+
- Token-aware chunking with stable overlap and table provenance.
|
|
68
|
+
- Credential-safe configuration and versioned artifact manifests.
|
|
69
|
+
- Source-grounded directed routing multigraphs with separate evidence and audit history.
|
|
70
|
+
- Native XLSForm relevance and repeat routing without a provider call.
|
|
71
|
+
- Caller-defined structured pipelines for completed-form records.
|
|
72
|
+
- A PEP 561 `py.typed` marker for editor and type-checker support.
|
|
73
|
+
- An installed CLI for single and batch conversion, configuration checks,
|
|
74
|
+
provider listing, and deterministic routing-schema export.
|
|
75
|
+
|
|
76
|
+
The standard client extracts questionnaire instrument metadata. It does not
|
|
77
|
+
produce respondent microdata, classify skipped answers, execute routing, or
|
|
78
|
+
expand repeat instances.
|
|
79
|
+
|
|
80
|
+
## Installation
|
|
81
|
+
|
|
82
|
+
After publication is approved, install the base schema package with:
|
|
83
|
+
|
|
84
|
+
```console
|
|
85
|
+
pip install survey-scribe
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
For development from this repository, use the locked environment:
|
|
89
|
+
|
|
90
|
+
```console
|
|
91
|
+
uv sync --locked --python 3.11
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Optional dependency groups are available for provider and document adapters:
|
|
95
|
+
|
|
96
|
+
```console
|
|
97
|
+
pip install "survey-scribe[openai]"
|
|
98
|
+
pip install "survey-scribe[anthropic]"
|
|
99
|
+
pip install "survey-scribe[pdf]"
|
|
100
|
+
pip install "survey-scribe[tabular]"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The base package includes the CLI. Install provider and source extras required by
|
|
104
|
+
the selected conversion path.
|
|
105
|
+
|
|
106
|
+
## Quick Start
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from datetime import date
|
|
110
|
+
|
|
111
|
+
from survey_scribe.sources import SourceRegistry
|
|
112
|
+
|
|
113
|
+
conversion = SourceRegistry.default().convert_for_svis(
|
|
114
|
+
"questionnaire.xlsx",
|
|
115
|
+
extraction_date=date.today(),
|
|
116
|
+
)
|
|
117
|
+
if conversion.svis is None:
|
|
118
|
+
raise RuntimeError("The workbook is not a supported XLSForm")
|
|
119
|
+
|
|
120
|
+
review_codes = tuple(item.code for item in conversion.document.diagnostics)
|
|
121
|
+
if conversion.native is not None:
|
|
122
|
+
review_codes += tuple(item.code for item in conversion.native.diagnostics)
|
|
123
|
+
if review_codes:
|
|
124
|
+
raise RuntimeError(f"Review XLSForm diagnostics before use: {review_codes}")
|
|
125
|
+
|
|
126
|
+
survey = conversion.svis
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
This native XLSForm path makes no provider call. PDF, DOCX, and other
|
|
130
|
+
questionnaire instruments use `SurveyScribe` with a configured provider.
|
|
131
|
+
|
|
132
|
+
Inspect the installed command without loading optional providers:
|
|
133
|
+
|
|
134
|
+
```console
|
|
135
|
+
survey-scribe --help
|
|
136
|
+
survey-scribe --version
|
|
137
|
+
survey-scribe providers
|
|
138
|
+
survey-scribe config check
|
|
139
|
+
survey-scribe convert questionnaire.pdf --output-dir output
|
|
140
|
+
survey-scribe batch questionnaire-a.pdf questionnaire-b.xlsx --output-dir output
|
|
141
|
+
survey-scribe schema export routing > questionnaire-routing-graph-v1.0.json
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Set credentials with environment variables or use `--prompt-api-key` /
|
|
145
|
+
`--prompt-bearer-token`. The CLI writes sidecars and manifests by default,
|
|
146
|
+
refuses existing artifacts unless `--overwrite` is present, and supports
|
|
147
|
+
`--strict` when partial output must produce a nonzero exit. See the
|
|
148
|
+
[CLI guide](docs/cli.md) and [migration guide](docs/migration.md).
|
|
149
|
+
|
|
150
|
+
## Documentation
|
|
151
|
+
|
|
152
|
+
The public website includes end-to-end questionnaire extraction, completed-form
|
|
153
|
+
contracts, skip-pattern examples, Palantir Foundry deployment, Microsoft Foundry,
|
|
154
|
+
mAI Factory, AI provider setup, artifacts, security, privacy, and generated API
|
|
155
|
+
references.
|
|
156
|
+
|
|
157
|
+
- [Documentation website](https://gmd-hub.github.io/survey-scribe/)
|
|
158
|
+
- [DeepWiki project guide](https://deepwiki.com/GMD-hub/survey-scribe)
|
|
159
|
+
- [Extraction guide](docs/guides/extraction.md)
|
|
160
|
+
- [Completed questionnaires](docs/guides/completed-questionnaires.md)
|
|
161
|
+
- [Skip patterns](docs/guides/skip-patterns.md)
|
|
162
|
+
- [Palantir Foundry](docs/platforms/palantir-foundry.md)
|
|
163
|
+
- [mAI Factory](docs/integrations/mai-factory.md)
|
|
164
|
+
- [AI providers](docs/guides/ai-providers.md)
|
|
165
|
+
|
|
166
|
+
```console
|
|
167
|
+
uv run mkdocs serve
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The local site is available at `http://127.0.0.1:8000/` while the server runs.
|
|
171
|
+
Pushes to `main` build and deploy the strict static site to GitHub Pages through
|
|
172
|
+
`.github/workflows/deploy-docs.yml`.
|
|
173
|
+
|
|
174
|
+
## Development
|
|
175
|
+
|
|
176
|
+
```console
|
|
177
|
+
uv run ruff check .
|
|
178
|
+
uv run ruff format --check .
|
|
179
|
+
uv run pyright
|
|
180
|
+
uv run pytest tests/unit tests/characterization tests/test_schema.py \
|
|
181
|
+
--cov=survey_scribe --cov-branch --cov-report=term-missing
|
|
182
|
+
uv run mkdocs build --strict
|
|
183
|
+
uv build
|
|
184
|
+
uv run twine check --strict dist/*
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
See [`CONTRIBUTING.md`](CONTRIBUTING.md) for fixture controls and pull request
|
|
188
|
+
requirements. Security reports follow [`SECURITY.md`](SECURITY.md).
|
|
189
|
+
|
|
190
|
+
## Versioning
|
|
191
|
+
|
|
192
|
+
Survey Scribe uses PEP 440 and Semantic Versioning. The current static version is
|
|
193
|
+
declared once in `pyproject.toml`; the runtime `survey_scribe.__version__` value
|
|
194
|
+
is read from installed distribution metadata. Release changes are recorded in
|
|
195
|
+
[`CHANGELOG.md`](CHANGELOG.md).
|
|
196
|
+
|
|
197
|
+
## License
|
|
198
|
+
|
|
199
|
+
Survey Scribe is licensed under the [MIT License](LICENSE). Package publication
|
|
200
|
+
is a separate operational decision and remains gated.
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# Survey Scribe
|
|
2
|
+
|
|
3
|
+
[](https://github.com/GMD-hub/survey-scribe/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Survey Scribe converts local survey questionnaires to the typed Survey Variable
|
|
8
|
+
Information Schema (SVIS). It provides synchronous, asynchronous, and batch APIs,
|
|
9
|
+
provider adapters, safe local source normalization, deterministic chunking,
|
|
10
|
+
questionnaire routing graphs, secure configuration, and versioned artifacts.
|
|
11
|
+
|
|
12
|
+
> **Alpha status:** The source tree declares version `0.1.0` and contains the
|
|
13
|
+
> public `SurveyScribe` API, conversion CLI, typed models, source and provider
|
|
14
|
+
> adapters, transactional artifacts, and `QuestionnaireRouter`. No approved PyPI
|
|
15
|
+
> release is currently available.
|
|
16
|
+
|
|
17
|
+
## Features
|
|
18
|
+
|
|
19
|
+
- Typed SVIS models with Pydantic validation and JSON serialization.
|
|
20
|
+
- Stable top-level imports for schema consumers.
|
|
21
|
+
- Numeric, categorical, text, date, and other variable classifications.
|
|
22
|
+
- Missing-category, source-provenance, confidence, and review metadata.
|
|
23
|
+
- Local PDF, DOCX, XLSX, CSV, HTML, Markdown, and text normalization.
|
|
24
|
+
- Token-aware chunking with stable overlap and table provenance.
|
|
25
|
+
- Credential-safe configuration and versioned artifact manifests.
|
|
26
|
+
- Source-grounded directed routing multigraphs with separate evidence and audit history.
|
|
27
|
+
- Native XLSForm relevance and repeat routing without a provider call.
|
|
28
|
+
- Caller-defined structured pipelines for completed-form records.
|
|
29
|
+
- A PEP 561 `py.typed` marker for editor and type-checker support.
|
|
30
|
+
- An installed CLI for single and batch conversion, configuration checks,
|
|
31
|
+
provider listing, and deterministic routing-schema export.
|
|
32
|
+
|
|
33
|
+
The standard client extracts questionnaire instrument metadata. It does not
|
|
34
|
+
produce respondent microdata, classify skipped answers, execute routing, or
|
|
35
|
+
expand repeat instances.
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
After publication is approved, install the base schema package with:
|
|
40
|
+
|
|
41
|
+
```console
|
|
42
|
+
pip install survey-scribe
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
For development from this repository, use the locked environment:
|
|
46
|
+
|
|
47
|
+
```console
|
|
48
|
+
uv sync --locked --python 3.11
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Optional dependency groups are available for provider and document adapters:
|
|
52
|
+
|
|
53
|
+
```console
|
|
54
|
+
pip install "survey-scribe[openai]"
|
|
55
|
+
pip install "survey-scribe[anthropic]"
|
|
56
|
+
pip install "survey-scribe[pdf]"
|
|
57
|
+
pip install "survey-scribe[tabular]"
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The base package includes the CLI. Install provider and source extras required by
|
|
61
|
+
the selected conversion path.
|
|
62
|
+
|
|
63
|
+
## Quick Start
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from datetime import date
|
|
67
|
+
|
|
68
|
+
from survey_scribe.sources import SourceRegistry
|
|
69
|
+
|
|
70
|
+
conversion = SourceRegistry.default().convert_for_svis(
|
|
71
|
+
"questionnaire.xlsx",
|
|
72
|
+
extraction_date=date.today(),
|
|
73
|
+
)
|
|
74
|
+
if conversion.svis is None:
|
|
75
|
+
raise RuntimeError("The workbook is not a supported XLSForm")
|
|
76
|
+
|
|
77
|
+
review_codes = tuple(item.code for item in conversion.document.diagnostics)
|
|
78
|
+
if conversion.native is not None:
|
|
79
|
+
review_codes += tuple(item.code for item in conversion.native.diagnostics)
|
|
80
|
+
if review_codes:
|
|
81
|
+
raise RuntimeError(f"Review XLSForm diagnostics before use: {review_codes}")
|
|
82
|
+
|
|
83
|
+
survey = conversion.svis
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
This native XLSForm path makes no provider call. PDF, DOCX, and other
|
|
87
|
+
questionnaire instruments use `SurveyScribe` with a configured provider.
|
|
88
|
+
|
|
89
|
+
Inspect the installed command without loading optional providers:
|
|
90
|
+
|
|
91
|
+
```console
|
|
92
|
+
survey-scribe --help
|
|
93
|
+
survey-scribe --version
|
|
94
|
+
survey-scribe providers
|
|
95
|
+
survey-scribe config check
|
|
96
|
+
survey-scribe convert questionnaire.pdf --output-dir output
|
|
97
|
+
survey-scribe batch questionnaire-a.pdf questionnaire-b.xlsx --output-dir output
|
|
98
|
+
survey-scribe schema export routing > questionnaire-routing-graph-v1.0.json
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Set credentials with environment variables or use `--prompt-api-key` /
|
|
102
|
+
`--prompt-bearer-token`. The CLI writes sidecars and manifests by default,
|
|
103
|
+
refuses existing artifacts unless `--overwrite` is present, and supports
|
|
104
|
+
`--strict` when partial output must produce a nonzero exit. See the
|
|
105
|
+
[CLI guide](docs/cli.md) and [migration guide](docs/migration.md).
|
|
106
|
+
|
|
107
|
+
## Documentation
|
|
108
|
+
|
|
109
|
+
The public website includes end-to-end questionnaire extraction, completed-form
|
|
110
|
+
contracts, skip-pattern examples, Palantir Foundry deployment, Microsoft Foundry,
|
|
111
|
+
mAI Factory, AI provider setup, artifacts, security, privacy, and generated API
|
|
112
|
+
references.
|
|
113
|
+
|
|
114
|
+
- [Documentation website](https://gmd-hub.github.io/survey-scribe/)
|
|
115
|
+
- [DeepWiki project guide](https://deepwiki.com/GMD-hub/survey-scribe)
|
|
116
|
+
- [Extraction guide](docs/guides/extraction.md)
|
|
117
|
+
- [Completed questionnaires](docs/guides/completed-questionnaires.md)
|
|
118
|
+
- [Skip patterns](docs/guides/skip-patterns.md)
|
|
119
|
+
- [Palantir Foundry](docs/platforms/palantir-foundry.md)
|
|
120
|
+
- [mAI Factory](docs/integrations/mai-factory.md)
|
|
121
|
+
- [AI providers](docs/guides/ai-providers.md)
|
|
122
|
+
|
|
123
|
+
```console
|
|
124
|
+
uv run mkdocs serve
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The local site is available at `http://127.0.0.1:8000/` while the server runs.
|
|
128
|
+
Pushes to `main` build and deploy the strict static site to GitHub Pages through
|
|
129
|
+
`.github/workflows/deploy-docs.yml`.
|
|
130
|
+
|
|
131
|
+
## Development
|
|
132
|
+
|
|
133
|
+
```console
|
|
134
|
+
uv run ruff check .
|
|
135
|
+
uv run ruff format --check .
|
|
136
|
+
uv run pyright
|
|
137
|
+
uv run pytest tests/unit tests/characterization tests/test_schema.py \
|
|
138
|
+
--cov=survey_scribe --cov-branch --cov-report=term-missing
|
|
139
|
+
uv run mkdocs build --strict
|
|
140
|
+
uv build
|
|
141
|
+
uv run twine check --strict dist/*
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
See [`CONTRIBUTING.md`](CONTRIBUTING.md) for fixture controls and pull request
|
|
145
|
+
requirements. Security reports follow [`SECURITY.md`](SECURITY.md).
|
|
146
|
+
|
|
147
|
+
## Versioning
|
|
148
|
+
|
|
149
|
+
Survey Scribe uses PEP 440 and Semantic Versioning. The current static version is
|
|
150
|
+
declared once in `pyproject.toml`; the runtime `survey_scribe.__version__` value
|
|
151
|
+
is read from installed distribution metadata. Release changes are recorded in
|
|
152
|
+
[`CHANGELOG.md`](CHANGELOG.md).
|
|
153
|
+
|
|
154
|
+
## License
|
|
155
|
+
|
|
156
|
+
Survey Scribe is licensed under the [MIT License](LICENSE). Package publication
|
|
157
|
+
is a separate operational decision and remains gated.
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling==1.27.0"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "survey-scribe"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Local-first questionnaire extraction to the Survey Variable Information Schema"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11,<3.14"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{name = "World Bank Global Monitoring Database team"},
|
|
15
|
+
]
|
|
16
|
+
keywords = [
|
|
17
|
+
"data extraction",
|
|
18
|
+
"household surveys",
|
|
19
|
+
"metadata",
|
|
20
|
+
"pydantic",
|
|
21
|
+
"questionnaires",
|
|
22
|
+
"schema",
|
|
23
|
+
"structured data",
|
|
24
|
+
"survey harmonization",
|
|
25
|
+
"survey metadata",
|
|
26
|
+
"SVIS",
|
|
27
|
+
]
|
|
28
|
+
classifiers = [
|
|
29
|
+
"Development Status :: 3 - Alpha",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
"Intended Audience :: Science/Research",
|
|
32
|
+
"Operating System :: OS Independent",
|
|
33
|
+
"Programming Language :: Python :: 3",
|
|
34
|
+
"Programming Language :: Python :: 3.11",
|
|
35
|
+
"Programming Language :: Python :: 3.12",
|
|
36
|
+
"Programming Language :: Python :: 3.13",
|
|
37
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
38
|
+
"Typing :: Typed",
|
|
39
|
+
]
|
|
40
|
+
dependencies = [
|
|
41
|
+
"defusedxml>=0.7.1,<1",
|
|
42
|
+
"pydantic>=2.11.7,<3",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[project.optional-dependencies]
|
|
46
|
+
openai = [
|
|
47
|
+
"instructor>=1.10.0,<2",
|
|
48
|
+
"openai>=1.99.9,<2",
|
|
49
|
+
"tenacity>=9.1.2,<10",
|
|
50
|
+
"tiktoken>=0.11.0,<1",
|
|
51
|
+
]
|
|
52
|
+
anthropic = [
|
|
53
|
+
"anthropic>=0.64.0,<1",
|
|
54
|
+
"instructor>=1.10.0,<2",
|
|
55
|
+
]
|
|
56
|
+
pdf = [
|
|
57
|
+
"docling>=2.125.0,<3",
|
|
58
|
+
"easyocr>=1.7.2,<2",
|
|
59
|
+
"lingua-language-detector>=2.1.1,<3",
|
|
60
|
+
"pymupdf>=1.26.4,<2",
|
|
61
|
+
]
|
|
62
|
+
tabular = [
|
|
63
|
+
"openpyxl>=3.1.5,<4",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[project.urls]
|
|
67
|
+
Homepage = "https://github.com/GMD-hub/survey-scribe"
|
|
68
|
+
Documentation = "https://gmd-hub.github.io/survey-scribe/"
|
|
69
|
+
Repository = "https://github.com/GMD-hub/survey-scribe.git"
|
|
70
|
+
Issues = "https://github.com/GMD-hub/survey-scribe/issues"
|
|
71
|
+
Changelog = "https://github.com/GMD-hub/survey-scribe/blob/main/CHANGELOG.md"
|
|
72
|
+
|
|
73
|
+
[[tool.uv.index]]
|
|
74
|
+
url = "https://pypi.org/simple"
|
|
75
|
+
default = true
|
|
76
|
+
|
|
77
|
+
[project.scripts]
|
|
78
|
+
survey-scribe = "survey_scribe.cli:main"
|
|
79
|
+
|
|
80
|
+
[dependency-groups]
|
|
81
|
+
dev = [
|
|
82
|
+
"axe-playwright-python==0.1.8",
|
|
83
|
+
"bandit==1.9.4",
|
|
84
|
+
"build==1.3.0",
|
|
85
|
+
"check-wheel-contents==0.6.3",
|
|
86
|
+
"cyclonedx-bom==7.3.1",
|
|
87
|
+
"detect-secrets==1.5.0",
|
|
88
|
+
"hatchling==1.27.0",
|
|
89
|
+
"loguru==0.7.3",
|
|
90
|
+
"mkdocs==1.6.1",
|
|
91
|
+
"mkdocs-material==9.7.7",
|
|
92
|
+
"mkdocstrings[python]==0.30.0",
|
|
93
|
+
"pip-audit==2.10.1",
|
|
94
|
+
"playwright==1.62.0",
|
|
95
|
+
"pyright==1.1.404",
|
|
96
|
+
"pytest==9.1.1",
|
|
97
|
+
"pytest-asyncio==1.4.0",
|
|
98
|
+
"pytest-cov==6.2.1",
|
|
99
|
+
"pytest-socket==0.8.1",
|
|
100
|
+
"linkchecker==10.6.0",
|
|
101
|
+
"pyyaml==6.0.3",
|
|
102
|
+
"ruff==0.12.8",
|
|
103
|
+
"tomli==2.2.1",
|
|
104
|
+
"twine==6.1.0",
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
[tool.hatch.build.targets.wheel]
|
|
108
|
+
packages = ["src/survey_scribe"]
|
|
109
|
+
|
|
110
|
+
[tool.hatch.build.targets.sdist]
|
|
111
|
+
include = [
|
|
112
|
+
"/CHANGELOG.md",
|
|
113
|
+
"/LICENSE",
|
|
114
|
+
"/README.md",
|
|
115
|
+
"/pyproject.toml",
|
|
116
|
+
"/src/survey_scribe",
|
|
117
|
+
"/uv.lock",
|
|
118
|
+
]
|
|
119
|
+
exclude = ["/.gitignore"]
|
|
120
|
+
|
|
121
|
+
[tool.pytest.ini_options]
|
|
122
|
+
testpaths = ["tests"]
|
|
123
|
+
addopts = "-ra --strict-markers --strict-config"
|
|
124
|
+
asyncio_mode = "auto"
|
|
125
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
126
|
+
asyncio_default_test_loop_scope = "function"
|
|
127
|
+
|
|
128
|
+
[tool.ruff]
|
|
129
|
+
line-length = 100
|
|
130
|
+
target-version = "py311"
|
|
131
|
+
|
|
132
|
+
[tool.ruff.lint]
|
|
133
|
+
select = ["B", "E", "F", "I", "SIM", "UP", "W"]
|
|
134
|
+
ignore = ["E501"]
|
|
135
|
+
|
|
136
|
+
[tool.ruff.format]
|
|
137
|
+
quote-style = "double"
|
|
138
|
+
|
|
139
|
+
[tool.pyright]
|
|
140
|
+
pythonVersion = "3.11"
|
|
141
|
+
typeCheckingMode = "basic"
|
|
142
|
+
include = [
|
|
143
|
+
"src",
|
|
144
|
+
"tests/characterization",
|
|
145
|
+
"tests/cli",
|
|
146
|
+
"tests/compat",
|
|
147
|
+
"tests/contract",
|
|
148
|
+
"tests/architecture",
|
|
149
|
+
"tests/browser",
|
|
150
|
+
"tests/docs",
|
|
151
|
+
"tests/integration",
|
|
152
|
+
"tests/package",
|
|
153
|
+
"tests/quality",
|
|
154
|
+
"tests/security",
|
|
155
|
+
"tests/test_schema.py",
|
|
156
|
+
"tests/unit",
|
|
157
|
+
"scripts/check_workflow_policy.py",
|
|
158
|
+
"scripts/build_wheel_sbom.py",
|
|
159
|
+
"scripts/generate_docs_reference.py",
|
|
160
|
+
"scripts/evaluate_quality.py",
|
|
161
|
+
"scripts/evaluate_routing.py",
|
|
162
|
+
"scripts/run_security_gates.py",
|
|
163
|
+
"scripts/validate_ocr_artifacts.py",
|
|
164
|
+
"scripts/validate_routing_fixtures.py",
|
|
165
|
+
"scripts/cg_pr_preflight.py",
|
|
166
|
+
"docling_pipeline.py",
|
|
167
|
+
]
|
|
168
|
+
|
|
169
|
+
[tool.coverage.run]
|
|
170
|
+
source = ["survey_scribe"]
|
|
171
|
+
branch = true
|
|
172
|
+
|
|
173
|
+
[tool.coverage.report]
|
|
174
|
+
fail_under = 95
|
|
175
|
+
precision = 1
|
|
176
|
+
show_missing = true
|
|
177
|
+
skip_covered = true
|
|
178
|
+
exclude_also = [
|
|
179
|
+
"if TYPE_CHECKING:",
|
|
180
|
+
"raise NotImplementedError",
|
|
181
|
+
]
|
|
182
|
+
|
|
183
|
+
[tool.check-wheel-contents]
|
|
184
|
+
toplevel = ["survey_scribe"]
|