survey-scribe 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. survey_scribe-0.1.0/.gitignore +73 -0
  2. survey_scribe-0.1.0/CHANGELOG.md +47 -0
  3. survey_scribe-0.1.0/LICENSE +21 -0
  4. survey_scribe-0.1.0/PKG-INFO +200 -0
  5. survey_scribe-0.1.0/README.md +157 -0
  6. survey_scribe-0.1.0/pyproject.toml +184 -0
  7. survey_scribe-0.1.0/src/survey_scribe/__init__.py +146 -0
  8. survey_scribe-0.1.0/src/survey_scribe/cli.py +539 -0
  9. survey_scribe-0.1.0/src/survey_scribe/client.py +467 -0
  10. survey_scribe-0.1.0/src/survey_scribe/config.py +482 -0
  11. survey_scribe-0.1.0/src/survey_scribe/errors.py +196 -0
  12. survey_scribe-0.1.0/src/survey_scribe/models/__init__.py +79 -0
  13. survey_scribe-0.1.0/src/survey_scribe/models/routing.py +1252 -0
  14. survey_scribe-0.1.0/src/survey_scribe/models/svis.py +126 -0
  15. survey_scribe-0.1.0/src/survey_scribe/pipeline.py +1234 -0
  16. survey_scribe-0.1.0/src/survey_scribe/providers/__init__.py +26 -0
  17. survey_scribe-0.1.0/src/survey_scribe/providers/anthropic.py +98 -0
  18. survey_scribe-0.1.0/src/survey_scribe/providers/azure.py +278 -0
  19. survey_scribe-0.1.0/src/survey_scribe/providers/base.py +309 -0
  20. survey_scribe-0.1.0/src/survey_scribe/providers/capabilities.py +169 -0
  21. survey_scribe-0.1.0/src/survey_scribe/providers/openai_compatible.py +532 -0
  22. survey_scribe-0.1.0/src/survey_scribe/providers/testing.py +225 -0
  23. survey_scribe-0.1.0/src/survey_scribe/py.typed +0 -0
  24. survey_scribe-0.1.0/src/survey_scribe/results.py +311 -0
  25. survey_scribe-0.1.0/src/survey_scribe/routing/__init__.py +183 -0
  26. survey_scribe-0.1.0/src/survey_scribe/routing/algorithms.py +119 -0
  27. survey_scribe-0.1.0/src/survey_scribe/routing/config.py +5 -0
  28. survey_scribe-0.1.0/src/survey_scribe/routing/contracts.py +479 -0
  29. survey_scribe-0.1.0/src/survey_scribe/routing/diagnostics.py +147 -0
  30. survey_scribe-0.1.0/src/survey_scribe/routing/extraction.py +1241 -0
  31. survey_scribe-0.1.0/src/survey_scribe/routing/identity.py +519 -0
  32. survey_scribe-0.1.0/src/survey_scribe/routing/inventory.py +335 -0
  33. survey_scribe-0.1.0/src/survey_scribe/routing/native.py +432 -0
  34. survey_scribe-0.1.0/src/survey_scribe/routing/normalization.py +58 -0
  35. survey_scribe-0.1.0/src/survey_scribe/routing/pipeline.py +589 -0
  36. survey_scribe-0.1.0/src/survey_scribe/routing/prompts.py +544 -0
  37. survey_scribe-0.1.0/src/survey_scribe/routing/reconcile.py +1211 -0
  38. survey_scribe-0.1.0/src/survey_scribe/routing/review.py +309 -0
  39. survey_scribe-0.1.0/src/survey_scribe/routing/validate.py +1040 -0
  40. survey_scribe-0.1.0/src/survey_scribe/serialization/__init__.py +28 -0
  41. survey_scribe-0.1.0/src/survey_scribe/serialization/artifacts.py +1727 -0
  42. survey_scribe-0.1.0/src/survey_scribe/serialization/legacy.py +63 -0
  43. survey_scribe-0.1.0/src/survey_scribe/serialization/routing.py +158 -0
  44. survey_scribe-0.1.0/src/survey_scribe/sources/__init__.py +55 -0
  45. survey_scribe-0.1.0/src/survey_scribe/sources/base.py +649 -0
  46. survey_scribe-0.1.0/src/survey_scribe/sources/chunking.py +372 -0
  47. survey_scribe-0.1.0/src/survey_scribe/sources/docling.py +1384 -0
  48. survey_scribe-0.1.0/src/survey_scribe/sources/ocr.py +268 -0
  49. survey_scribe-0.1.0/src/survey_scribe/sources/registry.py +355 -0
  50. survey_scribe-0.1.0/src/survey_scribe/sources/tabular.py +338 -0
  51. survey_scribe-0.1.0/src/survey_scribe/sources/xlsform.py +1160 -0
  52. survey_scribe-0.1.0/uv.lock +4694 -0
@@ -0,0 +1,73 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.pyo
5
+ .venv/
6
+ venv/
7
+ env/
8
+ *.egg-info/
9
+ dist/
10
+ build/
11
+ site/
12
+
13
+ # Environment variables - NEVER commit these files
14
+ .env
15
+ .env.*
16
+ !.env.example
17
+
18
+ # Generated outputs
19
+ output/*
20
+ !output/.gitkeep
21
+ survey-scribe.toml
22
+ **/.survey-scribe/
23
+ *_svis.json
24
+ !tests/fixtures/**/*_svis.json
25
+ craft_mlt_25k.zip
26
+ english_g2.zip
27
+
28
+ # Test samples- questionnaire PDFs may be restricted data
29
+ tests/samples/*
30
+ !tests/samples/.gitkeep
31
+
32
+ # IDE
33
+ .vscode/
34
+ .idea/
35
+ *.swp
36
+
37
+ # OS
38
+ .DS_Store
39
+ Thumbs.db
40
+
41
+ # Jupyter
42
+ .ipynb_checkpoints/
43
+ *.ipynb
44
+
45
+ # Local uv configuration
46
+ uv.toml
47
+
48
+ # Test and tooling caches
49
+ .pytest_cache/
50
+ .ruff_cache/
51
+ .coverage
52
+ coverage.xml
53
+ htmlcov/
54
+ .cache/
55
+
56
+ # Quarto book
57
+ book/_book/
58
+ book/.quarto/
59
+
60
+ /.quarto/
61
+ **/*.quarto_ipynb
62
+
63
+ tmp/
64
+
65
+ # Compound GPID managed items (junctions + copied file - do not commit)
66
+ .compound-gpid/managed-files.json
67
+ .kilo/AGENTS.md
68
+ .kilo/agents
69
+ .kilo/commands
70
+ .kilo/instructions
71
+ .kilo/kilo.json
72
+ .kilo/shared
73
+ .kilo/skills
@@ -0,0 +1,47 @@
1
+ # Changelog
2
+
3
+ All notable changes to Survey Scribe are documented in this file. The format is
4
+ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the
5
+ project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [Unreleased]
8
+
9
+ ### Added
10
+
11
+ - PyPI-ready project metadata and MIT license declaration.
12
+ - MkDocs documentation with installation, usage, examples, and API reference.
13
+ - Coverage enforcement and release-artifact validation in CI.
14
+ - Additive routed SVIS models, deterministic directed-multigraph validation,
15
+ source-grounded evidence, and append-only discrepancy review.
16
+ - `QuestionnaireRouter` with native XLSForm routing and structured-provider integration.
17
+ - Deterministic routing-quality evaluation, routing-schema export, and routing documentation.
18
+ - Validated static metadata headers and per-attempt auxiliary secret headers for
19
+ direct `AzureOpenAIProvider` injection through compatible gateways.
20
+ - Extraction-first public guides for completed questionnaires, skip patterns,
21
+ Palantir Foundry, Microsoft Foundry, mAI Factory, and AI providers.
22
+
23
+ ### Changed
24
+
25
+ - Runtime dependencies now use compatible ranges while the committed `uv.lock`
26
+ retains exact engineering versions.
27
+
28
+ ### Fixed
29
+
30
+ - Native XLSForm `source_format="xlsform"` now binds to its validated XLSX
31
+ snapshot for provider-free questionnaire routing.
32
+ - Public installation guidance no longer presents an unavailable PyPI release or
33
+ a stale source revision as the primary installation path.
34
+
35
+ ## 0.1.0 - Unpublished
36
+
37
+ This version remains a source-tree milestone. No approved package-index or GitHub
38
+ release is available.
39
+
40
+ ### Added
41
+
42
+ - Installable `survey-scribe` package with typed SVIS Pydantic models.
43
+ - Bootstrap `survey-scribe` command.
44
+ - Legacy schema re-export and characterization suite.
45
+ - Cross-platform Python 3.11-3.13 CI and clean-wheel installation checks.
46
+
47
+ [Unreleased]: https://github.com/GMD-hub/survey-scribe/commits/main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Survey Scribe contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,200 @@
1
+ Metadata-Version: 2.4
2
+ Name: survey-scribe
3
+ Version: 0.1.0
4
+ Summary: Local-first questionnaire extraction to the Survey Variable Information Schema
5
+ Project-URL: Homepage, https://github.com/GMD-hub/survey-scribe
6
+ Project-URL: Documentation, https://gmd-hub.github.io/survey-scribe/
7
+ Project-URL: Repository, https://github.com/GMD-hub/survey-scribe.git
8
+ Project-URL: Issues, https://github.com/GMD-hub/survey-scribe/issues
9
+ Project-URL: Changelog, https://github.com/GMD-hub/survey-scribe/blob/main/CHANGELOG.md
10
+ Author: World Bank Global Monitoring Database team
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ Keywords: SVIS,data extraction,household surveys,metadata,pydantic,questionnaires,schema,structured data,survey harmonization,survey metadata
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Intended Audience :: Science/Research
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
23
+ Classifier: Typing :: Typed
24
+ Requires-Python: <3.14,>=3.11
25
+ Requires-Dist: defusedxml<1,>=0.7.1
26
+ Requires-Dist: pydantic<3,>=2.11.7
27
+ Provides-Extra: anthropic
28
+ Requires-Dist: anthropic<1,>=0.64.0; extra == 'anthropic'
29
+ Requires-Dist: instructor<2,>=1.10.0; extra == 'anthropic'
30
+ Provides-Extra: openai
31
+ Requires-Dist: instructor<2,>=1.10.0; extra == 'openai'
32
+ Requires-Dist: openai<2,>=1.99.9; extra == 'openai'
33
+ Requires-Dist: tenacity<10,>=9.1.2; extra == 'openai'
34
+ Requires-Dist: tiktoken<1,>=0.11.0; extra == 'openai'
35
+ Provides-Extra: pdf
36
+ Requires-Dist: docling<3,>=2.125.0; extra == 'pdf'
37
+ Requires-Dist: easyocr<2,>=1.7.2; extra == 'pdf'
38
+ Requires-Dist: lingua-language-detector<3,>=2.1.1; extra == 'pdf'
39
+ Requires-Dist: pymupdf<2,>=1.26.4; extra == 'pdf'
40
+ Provides-Extra: tabular
41
+ Requires-Dist: openpyxl<4,>=3.1.5; extra == 'tabular'
42
+ Description-Content-Type: text/markdown
43
+
44
+ # Survey Scribe
45
+
46
+ [![CI](https://github.com/GMD-hub/survey-scribe/actions/workflows/ci.yml/badge.svg)](https://github.com/GMD-hub/survey-scribe/actions/workflows/ci.yml)
47
+ [![Python 3.11-3.13](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue.svg)](https://www.python.org/)
48
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
49
+
50
+ Survey Scribe converts local survey questionnaires to the typed Survey Variable
51
+ Information Schema (SVIS). It provides synchronous, asynchronous, and batch APIs,
52
+ provider adapters, safe local source normalization, deterministic chunking,
53
+ questionnaire routing graphs, secure configuration, and versioned artifacts.
54
+
55
+ > **Alpha status:** The source tree declares version `0.1.0` and contains the
56
+ > public `SurveyScribe` API, conversion CLI, typed models, source and provider
57
+ > adapters, transactional artifacts, and `QuestionnaireRouter`. No approved PyPI
58
+ > release is currently available.
59
+
60
+ ## Features
61
+
62
+ - Typed SVIS models with Pydantic validation and JSON serialization.
63
+ - Stable top-level imports for schema consumers.
64
+ - Numeric, categorical, text, date, and other variable classifications.
65
+ - Missing-category, source-provenance, confidence, and review metadata.
66
+ - Local PDF, DOCX, XLSX, CSV, HTML, Markdown, and text normalization.
67
+ - Token-aware chunking with stable overlap and table provenance.
68
+ - Credential-safe configuration and versioned artifact manifests.
69
+ - Source-grounded directed routing multigraphs with separate evidence and audit history.
70
+ - Native XLSForm relevance and repeat routing without a provider call.
71
+ - Caller-defined structured pipelines for completed-form records.
72
+ - A PEP 561 `py.typed` marker for editor and type-checker support.
73
+ - An installed CLI for single and batch conversion, configuration checks,
74
+ provider listing, and deterministic routing-schema export.
75
+
76
+ The standard client extracts questionnaire instrument metadata. It does not
77
+ produce respondent microdata, classify skipped answers, execute routing, or
78
+ expand repeat instances.
79
+
80
+ ## Installation
81
+
82
+ After publication is approved, install the base schema package with:
83
+
84
+ ```console
85
+ pip install survey-scribe
86
+ ```
87
+
88
+ For development from this repository, use the locked environment:
89
+
90
+ ```console
91
+ uv sync --locked --python 3.11
92
+ ```
93
+
94
+ Optional dependency groups are available for provider and document adapters:
95
+
96
+ ```console
97
+ pip install "survey-scribe[openai]"
98
+ pip install "survey-scribe[anthropic]"
99
+ pip install "survey-scribe[pdf]"
100
+ pip install "survey-scribe[tabular]"
101
+ ```
102
+
103
+ The base package includes the CLI. Install provider and source extras required by
104
+ the selected conversion path.
105
+
106
+ ## Quick Start
107
+
108
+ ```python
109
+ from datetime import date
110
+
111
+ from survey_scribe.sources import SourceRegistry
112
+
113
+ conversion = SourceRegistry.default().convert_for_svis(
114
+ "questionnaire.xlsx",
115
+ extraction_date=date.today(),
116
+ )
117
+ if conversion.svis is None:
118
+ raise RuntimeError("The workbook is not a supported XLSForm")
119
+
120
+ review_codes = tuple(item.code for item in conversion.document.diagnostics)
121
+ if conversion.native is not None:
122
+ review_codes += tuple(item.code for item in conversion.native.diagnostics)
123
+ if review_codes:
124
+ raise RuntimeError(f"Review XLSForm diagnostics before use: {review_codes}")
125
+
126
+ survey = conversion.svis
127
+ ```
128
+
129
+ This native XLSForm path makes no provider call. PDF, DOCX, and other
130
+ questionnaire instruments use `SurveyScribe` with a configured provider.
131
+
132
+ Inspect the installed command without loading optional providers:
133
+
134
+ ```console
135
+ survey-scribe --help
136
+ survey-scribe --version
137
+ survey-scribe providers
138
+ survey-scribe config check
139
+ survey-scribe convert questionnaire.pdf --output-dir output
140
+ survey-scribe batch questionnaire-a.pdf questionnaire-b.xlsx --output-dir output
141
+ survey-scribe schema export routing > questionnaire-routing-graph-v1.0.json
142
+ ```
143
+
144
+ Set credentials with environment variables or use `--prompt-api-key` /
145
+ `--prompt-bearer-token`. The CLI writes sidecars and manifests by default,
146
+ refuses existing artifacts unless `--overwrite` is present, and supports
147
+ `--strict` when partial output must produce a nonzero exit. See the
148
+ [CLI guide](docs/cli.md) and [migration guide](docs/migration.md).
149
+
150
+ ## Documentation
151
+
152
+ The public website includes end-to-end questionnaire extraction, completed-form
153
+ contracts, skip-pattern examples, Palantir Foundry deployment, Microsoft Foundry,
154
+ mAI Factory, AI provider setup, artifacts, security, privacy, and generated API
155
+ references.
156
+
157
+ - [Documentation website](https://gmd-hub.github.io/survey-scribe/)
158
+ - [DeepWiki project guide](https://deepwiki.com/GMD-hub/survey-scribe)
159
+ - [Extraction guide](docs/guides/extraction.md)
160
+ - [Completed questionnaires](docs/guides/completed-questionnaires.md)
161
+ - [Skip patterns](docs/guides/skip-patterns.md)
162
+ - [Palantir Foundry](docs/platforms/palantir-foundry.md)
163
+ - [mAI Factory](docs/integrations/mai-factory.md)
164
+ - [AI providers](docs/guides/ai-providers.md)
165
+
166
+ ```console
167
+ uv run mkdocs serve
168
+ ```
169
+
170
+ The local site is available at `http://127.0.0.1:8000/` while the server runs.
171
+ Pushes to `main` build and deploy the strict static site to GitHub Pages through
172
+ `.github/workflows/deploy-docs.yml`.
173
+
174
+ ## Development
175
+
176
+ ```console
177
+ uv run ruff check .
178
+ uv run ruff format --check .
179
+ uv run pyright
180
+ uv run pytest tests/unit tests/characterization tests/test_schema.py \
181
+ --cov=survey_scribe --cov-branch --cov-report=term-missing
182
+ uv run mkdocs build --strict
183
+ uv build
184
+ uv run twine check --strict dist/*
185
+ ```
186
+
187
+ See [`CONTRIBUTING.md`](CONTRIBUTING.md) for fixture controls and pull request
188
+ requirements. Security reports follow [`SECURITY.md`](SECURITY.md).
189
+
190
+ ## Versioning
191
+
192
+ Survey Scribe uses PEP 440 and Semantic Versioning. The current static version is
193
+ declared once in `pyproject.toml`; the runtime `survey_scribe.__version__` value
194
+ is read from installed distribution metadata. Release changes are recorded in
195
+ [`CHANGELOG.md`](CHANGELOG.md).
196
+
197
+ ## License
198
+
199
+ Survey Scribe is licensed under the [MIT License](LICENSE). Package publication
200
+ is a separate operational decision and remains gated.
@@ -0,0 +1,157 @@
1
+ # Survey Scribe
2
+
3
+ [![CI](https://github.com/GMD-hub/survey-scribe/actions/workflows/ci.yml/badge.svg)](https://github.com/GMD-hub/survey-scribe/actions/workflows/ci.yml)
4
+ [![Python 3.11-3.13](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue.svg)](https://www.python.org/)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
6
+
7
+ Survey Scribe converts local survey questionnaires to the typed Survey Variable
8
+ Information Schema (SVIS). It provides synchronous, asynchronous, and batch APIs,
9
+ provider adapters, safe local source normalization, deterministic chunking,
10
+ questionnaire routing graphs, secure configuration, and versioned artifacts.
11
+
12
+ > **Alpha status:** The source tree declares version `0.1.0` and contains the
13
+ > public `SurveyScribe` API, conversion CLI, typed models, source and provider
14
+ > adapters, transactional artifacts, and `QuestionnaireRouter`. No approved PyPI
15
+ > release is currently available.
16
+
17
+ ## Features
18
+
19
+ - Typed SVIS models with Pydantic validation and JSON serialization.
20
+ - Stable top-level imports for schema consumers.
21
+ - Numeric, categorical, text, date, and other variable classifications.
22
+ - Missing-category, source-provenance, confidence, and review metadata.
23
+ - Local PDF, DOCX, XLSX, CSV, HTML, Markdown, and text normalization.
24
+ - Token-aware chunking with stable overlap and table provenance.
25
+ - Credential-safe configuration and versioned artifact manifests.
26
+ - Source-grounded directed routing multigraphs with separate evidence and audit history.
27
+ - Native XLSForm relevance and repeat routing without a provider call.
28
+ - Caller-defined structured pipelines for completed-form records.
29
+ - A PEP 561 `py.typed` marker for editor and type-checker support.
30
+ - An installed CLI for single and batch conversion, configuration checks,
31
+ provider listing, and deterministic routing-schema export.
32
+
33
+ The standard client extracts questionnaire instrument metadata. It does not
34
+ produce respondent microdata, classify skipped answers, execute routing, or
35
+ expand repeat instances.
36
+
37
+ ## Installation
38
+
39
+ After publication is approved, install the base schema package with:
40
+
41
+ ```console
42
+ pip install survey-scribe
43
+ ```
44
+
45
+ For development from this repository, use the locked environment:
46
+
47
+ ```console
48
+ uv sync --locked --python 3.11
49
+ ```
50
+
51
+ Optional dependency groups are available for provider and document adapters:
52
+
53
+ ```console
54
+ pip install "survey-scribe[openai]"
55
+ pip install "survey-scribe[anthropic]"
56
+ pip install "survey-scribe[pdf]"
57
+ pip install "survey-scribe[tabular]"
58
+ ```
59
+
60
+ The base package includes the CLI. Install provider and source extras required by
61
+ the selected conversion path.
62
+
63
+ ## Quick Start
64
+
65
+ ```python
66
+ from datetime import date
67
+
68
+ from survey_scribe.sources import SourceRegistry
69
+
70
+ conversion = SourceRegistry.default().convert_for_svis(
71
+ "questionnaire.xlsx",
72
+ extraction_date=date.today(),
73
+ )
74
+ if conversion.svis is None:
75
+ raise RuntimeError("The workbook is not a supported XLSForm")
76
+
77
+ review_codes = tuple(item.code for item in conversion.document.diagnostics)
78
+ if conversion.native is not None:
79
+ review_codes += tuple(item.code for item in conversion.native.diagnostics)
80
+ if review_codes:
81
+ raise RuntimeError(f"Review XLSForm diagnostics before use: {review_codes}")
82
+
83
+ survey = conversion.svis
84
+ ```
85
+
86
+ This native XLSForm path makes no provider call. PDF, DOCX, and other
87
+ questionnaire instruments use `SurveyScribe` with a configured provider.
88
+
89
+ Inspect the installed command without loading optional providers:
90
+
91
+ ```console
92
+ survey-scribe --help
93
+ survey-scribe --version
94
+ survey-scribe providers
95
+ survey-scribe config check
96
+ survey-scribe convert questionnaire.pdf --output-dir output
97
+ survey-scribe batch questionnaire-a.pdf questionnaire-b.xlsx --output-dir output
98
+ survey-scribe schema export routing > questionnaire-routing-graph-v1.0.json
99
+ ```
100
+
101
+ Set credentials with environment variables or use `--prompt-api-key` /
102
+ `--prompt-bearer-token`. The CLI writes sidecars and manifests by default,
103
+ refuses existing artifacts unless `--overwrite` is present, and supports
104
+ `--strict` when partial output must produce a nonzero exit. See the
105
+ [CLI guide](docs/cli.md) and [migration guide](docs/migration.md).
106
+
107
+ ## Documentation
108
+
109
+ The public website includes end-to-end questionnaire extraction, completed-form
110
+ contracts, skip-pattern examples, Palantir Foundry deployment, Microsoft Foundry,
111
+ mAI Factory, AI provider setup, artifacts, security, privacy, and generated API
112
+ references.
113
+
114
+ - [Documentation website](https://gmd-hub.github.io/survey-scribe/)
115
+ - [DeepWiki project guide](https://deepwiki.com/GMD-hub/survey-scribe)
116
+ - [Extraction guide](docs/guides/extraction.md)
117
+ - [Completed questionnaires](docs/guides/completed-questionnaires.md)
118
+ - [Skip patterns](docs/guides/skip-patterns.md)
119
+ - [Palantir Foundry](docs/platforms/palantir-foundry.md)
120
+ - [mAI Factory](docs/integrations/mai-factory.md)
121
+ - [AI providers](docs/guides/ai-providers.md)
122
+
123
+ ```console
124
+ uv run mkdocs serve
125
+ ```
126
+
127
+ The local site is available at `http://127.0.0.1:8000/` while the server runs.
128
+ Pushes to `main` build and deploy the strict static site to GitHub Pages through
129
+ `.github/workflows/deploy-docs.yml`.
130
+
131
+ ## Development
132
+
133
+ ```console
134
+ uv run ruff check .
135
+ uv run ruff format --check .
136
+ uv run pyright
137
+ uv run pytest tests/unit tests/characterization tests/test_schema.py \
138
+ --cov=survey_scribe --cov-branch --cov-report=term-missing
139
+ uv run mkdocs build --strict
140
+ uv build
141
+ uv run twine check --strict dist/*
142
+ ```
143
+
144
+ See [`CONTRIBUTING.md`](CONTRIBUTING.md) for fixture controls and pull request
145
+ requirements. Security reports follow [`SECURITY.md`](SECURITY.md).
146
+
147
+ ## Versioning
148
+
149
+ Survey Scribe uses PEP 440 and Semantic Versioning. The current static version is
150
+ declared once in `pyproject.toml`; the runtime `survey_scribe.__version__` value
151
+ is read from installed distribution metadata. Release changes are recorded in
152
+ [`CHANGELOG.md`](CHANGELOG.md).
153
+
154
+ ## License
155
+
156
+ Survey Scribe is licensed under the [MIT License](LICENSE). Package publication
157
+ is a separate operational decision and remains gated.
@@ -0,0 +1,184 @@
1
+ [build-system]
2
+ requires = ["hatchling==1.27.0"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "survey-scribe"
7
+ version = "0.1.0"
8
+ description = "Local-first questionnaire extraction to the Survey Variable Information Schema"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11,<3.14"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [
14
+ {name = "World Bank Global Monitoring Database team"},
15
+ ]
16
+ keywords = [
17
+ "data extraction",
18
+ "household surveys",
19
+ "metadata",
20
+ "pydantic",
21
+ "questionnaires",
22
+ "schema",
23
+ "structured data",
24
+ "survey harmonization",
25
+ "survey metadata",
26
+ "SVIS",
27
+ ]
28
+ classifiers = [
29
+ "Development Status :: 3 - Alpha",
30
+ "Intended Audience :: Developers",
31
+ "Intended Audience :: Science/Research",
32
+ "Operating System :: OS Independent",
33
+ "Programming Language :: Python :: 3",
34
+ "Programming Language :: Python :: 3.11",
35
+ "Programming Language :: Python :: 3.12",
36
+ "Programming Language :: Python :: 3.13",
37
+ "Topic :: Scientific/Engineering :: Information Analysis",
38
+ "Typing :: Typed",
39
+ ]
40
+ dependencies = [
41
+ "defusedxml>=0.7.1,<1",
42
+ "pydantic>=2.11.7,<3",
43
+ ]
44
+
45
+ [project.optional-dependencies]
46
+ openai = [
47
+ "instructor>=1.10.0,<2",
48
+ "openai>=1.99.9,<2",
49
+ "tenacity>=9.1.2,<10",
50
+ "tiktoken>=0.11.0,<1",
51
+ ]
52
+ anthropic = [
53
+ "anthropic>=0.64.0,<1",
54
+ "instructor>=1.10.0,<2",
55
+ ]
56
+ pdf = [
57
+ "docling>=2.125.0,<3",
58
+ "easyocr>=1.7.2,<2",
59
+ "lingua-language-detector>=2.1.1,<3",
60
+ "pymupdf>=1.26.4,<2",
61
+ ]
62
+ tabular = [
63
+ "openpyxl>=3.1.5,<4",
64
+ ]
65
+
66
+ [project.urls]
67
+ Homepage = "https://github.com/GMD-hub/survey-scribe"
68
+ Documentation = "https://gmd-hub.github.io/survey-scribe/"
69
+ Repository = "https://github.com/GMD-hub/survey-scribe.git"
70
+ Issues = "https://github.com/GMD-hub/survey-scribe/issues"
71
+ Changelog = "https://github.com/GMD-hub/survey-scribe/blob/main/CHANGELOG.md"
72
+
73
+ [[tool.uv.index]]
74
+ url = "https://pypi.org/simple"
75
+ default = true
76
+
77
+ [project.scripts]
78
+ survey-scribe = "survey_scribe.cli:main"
79
+
80
+ [dependency-groups]
81
+ dev = [
82
+ "axe-playwright-python==0.1.8",
83
+ "bandit==1.9.4",
84
+ "build==1.3.0",
85
+ "check-wheel-contents==0.6.3",
86
+ "cyclonedx-bom==7.3.1",
87
+ "detect-secrets==1.5.0",
88
+ "hatchling==1.27.0",
89
+ "loguru==0.7.3",
90
+ "mkdocs==1.6.1",
91
+ "mkdocs-material==9.7.7",
92
+ "mkdocstrings[python]==0.30.0",
93
+ "pip-audit==2.10.1",
94
+ "playwright==1.62.0",
95
+ "pyright==1.1.404",
96
+ "pytest==9.1.1",
97
+ "pytest-asyncio==1.4.0",
98
+ "pytest-cov==6.2.1",
99
+ "pytest-socket==0.8.1",
100
+ "linkchecker==10.6.0",
101
+ "pyyaml==6.0.3",
102
+ "ruff==0.12.8",
103
+ "tomli==2.2.1",
104
+ "twine==6.1.0",
105
+ ]
106
+
107
+ [tool.hatch.build.targets.wheel]
108
+ packages = ["src/survey_scribe"]
109
+
110
+ [tool.hatch.build.targets.sdist]
111
+ include = [
112
+ "/CHANGELOG.md",
113
+ "/LICENSE",
114
+ "/README.md",
115
+ "/pyproject.toml",
116
+ "/src/survey_scribe",
117
+ "/uv.lock",
118
+ ]
119
+ exclude = ["/.gitignore"]
120
+
121
+ [tool.pytest.ini_options]
122
+ testpaths = ["tests"]
123
+ addopts = "-ra --strict-markers --strict-config"
124
+ asyncio_mode = "auto"
125
+ asyncio_default_fixture_loop_scope = "function"
126
+ asyncio_default_test_loop_scope = "function"
127
+
128
+ [tool.ruff]
129
+ line-length = 100
130
+ target-version = "py311"
131
+
132
+ [tool.ruff.lint]
133
+ select = ["B", "E", "F", "I", "SIM", "UP", "W"]
134
+ ignore = ["E501"]
135
+
136
+ [tool.ruff.format]
137
+ quote-style = "double"
138
+
139
+ [tool.pyright]
140
+ pythonVersion = "3.11"
141
+ typeCheckingMode = "basic"
142
+ include = [
143
+ "src",
144
+ "tests/characterization",
145
+ "tests/cli",
146
+ "tests/compat",
147
+ "tests/contract",
148
+ "tests/architecture",
149
+ "tests/browser",
150
+ "tests/docs",
151
+ "tests/integration",
152
+ "tests/package",
153
+ "tests/quality",
154
+ "tests/security",
155
+ "tests/test_schema.py",
156
+ "tests/unit",
157
+ "scripts/check_workflow_policy.py",
158
+ "scripts/build_wheel_sbom.py",
159
+ "scripts/generate_docs_reference.py",
160
+ "scripts/evaluate_quality.py",
161
+ "scripts/evaluate_routing.py",
162
+ "scripts/run_security_gates.py",
163
+ "scripts/validate_ocr_artifacts.py",
164
+ "scripts/validate_routing_fixtures.py",
165
+ "scripts/cg_pr_preflight.py",
166
+ "docling_pipeline.py",
167
+ ]
168
+
169
+ [tool.coverage.run]
170
+ source = ["survey_scribe"]
171
+ branch = true
172
+
173
+ [tool.coverage.report]
174
+ fail_under = 95
175
+ precision = 1
176
+ show_missing = true
177
+ skip_covered = true
178
+ exclude_also = [
179
+ "if TYPE_CHECKING:",
180
+ "raise NotImplementedError",
181
+ ]
182
+
183
+ [tool.check-wheel-contents]
184
+ toplevel = ["survey_scribe"]