docintel-platform 1.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. docintel_platform-1.0.2/LICENSE +21 -0
  2. docintel_platform-1.0.2/MANIFEST.in +3 -0
  3. docintel_platform-1.0.2/PKG-INFO +607 -0
  4. docintel_platform-1.0.2/README.md +537 -0
  5. docintel_platform-1.0.2/pyproject.toml +106 -0
  6. docintel_platform-1.0.2/setup.cfg +4 -0
  7. docintel_platform-1.0.2/src/docintel/__init__.py +6 -0
  8. docintel_platform-1.0.2/src/docintel/app.py +45 -0
  9. docintel_platform-1.0.2/src/docintel/auth/__init__.py +12 -0
  10. docintel_platform-1.0.2/src/docintel/auth/api_keys.py +48 -0
  11. docintel_platform-1.0.2/src/docintel/auth/limiter.py +41 -0
  12. docintel_platform-1.0.2/src/docintel/auth/middleware.py +34 -0
  13. docintel_platform-1.0.2/src/docintel/auth/oidc.py +45 -0
  14. docintel_platform-1.0.2/src/docintel/cli.py +21 -0
  15. docintel_platform-1.0.2/src/docintel/client.py +193 -0
  16. docintel_platform-1.0.2/src/docintel/config.py +20 -0
  17. docintel_platform-1.0.2/src/docintel/jobs/__init__.py +16 -0
  18. docintel_platform-1.0.2/src/docintel/jobs/helpers.py +38 -0
  19. docintel_platform-1.0.2/src/docintel/jobs/models.py +78 -0
  20. docintel_platform-1.0.2/src/docintel/jobs/queue.py +75 -0
  21. docintel_platform-1.0.2/src/docintel/jobs/store.py +82 -0
  22. docintel_platform-1.0.2/src/docintel/jobs/tasks.py +173 -0
  23. docintel_platform-1.0.2/src/docintel/jobs/webhooks.py +32 -0
  24. docintel_platform-1.0.2/src/docintel/openapi/__init__.py +1 -0
  25. docintel_platform-1.0.2/src/docintel/openapi/openapi.yaml +380 -0
  26. docintel_platform-1.0.2/src/docintel/ops/__init__.py +1 -0
  27. docintel_platform-1.0.2/src/docintel/ops/logging.py +40 -0
  28. docintel_platform-1.0.2/src/docintel/ops/metrics.py +57 -0
  29. docintel_platform-1.0.2/src/docintel/ops/middleware.py +40 -0
  30. docintel_platform-1.0.2/src/docintel/routes/__init__.py +1 -0
  31. docintel_platform-1.0.2/src/docintel/routes/jobs.py +26 -0
  32. docintel_platform-1.0.2/src/docintel/routes/match.py +43 -0
  33. docintel_platform-1.0.2/src/docintel/routes/openapi_docs.py +57 -0
  34. docintel_platform-1.0.2/src/docintel/routes/ops.py +22 -0
  35. docintel_platform-1.0.2/src/docintel/routes/pdf.py +420 -0
  36. docintel_platform-1.0.2/src/docintel/routes/text.py +41 -0
  37. docintel_platform-1.0.2/src/docintel/services/__init__.py +1 -0
  38. docintel_platform-1.0.2/src/docintel/services/matching/__init__.py +6 -0
  39. docintel_platform-1.0.2/src/docintel/services/matching/models.py +19 -0
  40. docintel_platform-1.0.2/src/docintel/services/matching/scorer.py +64 -0
  41. docintel_platform-1.0.2/src/docintel/services/pdf/__init__.py +26 -0
  42. docintel_platform-1.0.2/src/docintel/services/pdf/annotator.py +188 -0
  43. docintel_platform-1.0.2/src/docintel/services/pdf/models.py +104 -0
  44. docintel_platform-1.0.2/src/docintel/services/pdf/ocr.py +130 -0
  45. docintel_platform-1.0.2/src/docintel/services/pdf/pii.py +105 -0
  46. docintel_platform-1.0.2/src/docintel/services/pdf/presets.py +26 -0
  47. docintel_platform-1.0.2/src/docintel/services/pdf/search.py +29 -0
  48. docintel_platform-1.0.2/src/docintel/services/pdf/sensitive.py +212 -0
  49. docintel_platform-1.0.2/src/docintel/services/pdf/structure.py +118 -0
  50. docintel_platform-1.0.2/src/docintel/services/pdf/structure_llm.py +136 -0
  51. docintel_platform-1.0.2/src/docintel/services/pdf/structure_render.py +136 -0
  52. docintel_platform-1.0.2/src/docintel/services/pdf/structure_schema.py +99 -0
  53. docintel_platform-1.0.2/src/docintel/services/summary/__init__.py +6 -0
  54. docintel_platform-1.0.2/src/docintel/services/summary/models.py +21 -0
  55. docintel_platform-1.0.2/src/docintel/services/summary/textrank.py +57 -0
  56. docintel_platform-1.0.2/src/docintel/ui.py +347 -0
  57. docintel_platform-1.0.2/src/docintel/wsgi.py +5 -0
  58. docintel_platform-1.0.2/src/docintel_platform.egg-info/PKG-INFO +607 -0
  59. docintel_platform-1.0.2/src/docintel_platform.egg-info/SOURCES.txt +81 -0
  60. docintel_platform-1.0.2/src/docintel_platform.egg-info/dependency_links.txt +1 -0
  61. docintel_platform-1.0.2/src/docintel_platform.egg-info/entry_points.txt +3 -0
  62. docintel_platform-1.0.2/src/docintel_platform.egg-info/requires.txt +52 -0
  63. docintel_platform-1.0.2/src/docintel_platform.egg-info/top_level.txt +1 -0
  64. docintel_platform-1.0.2/tests/test_auth.py +58 -0
  65. docintel_platform-1.0.2/tests/test_client.py +43 -0
  66. docintel_platform-1.0.2/tests/test_detect_sensitive_async.py +96 -0
  67. docintel_platform-1.0.2/tests/test_health.py +14 -0
  68. docintel_platform-1.0.2/tests/test_jobs.py +145 -0
  69. docintel_platform-1.0.2/tests/test_matching_routes.py +64 -0
  70. docintel_platform-1.0.2/tests/test_matching_service.py +59 -0
  71. docintel_platform-1.0.2/tests/test_oidc.py +49 -0
  72. docintel_platform-1.0.2/tests/test_openapi.py +22 -0
  73. docintel_platform-1.0.2/tests/test_ops.py +58 -0
  74. docintel_platform-1.0.2/tests/test_pdf_routes.py +112 -0
  75. docintel_platform-1.0.2/tests/test_pdf_sensitive.py +128 -0
  76. docintel_platform-1.0.2/tests/test_pdf_service.py +56 -0
  77. docintel_platform-1.0.2/tests/test_pdf_structure.py +184 -0
  78. docintel_platform-1.0.2/tests/test_pii_mask.py +26 -0
  79. docintel_platform-1.0.2/tests/test_structure_pii.py +51 -0
  80. docintel_platform-1.0.2/tests/test_summary_routes.py +48 -0
  81. docintel_platform-1.0.2/tests/test_summary_service.py +53 -0
  82. docintel_platform-1.0.2/tests/test_ui.py +17 -0
  83. docintel_platform-1.0.2/tests/test_webhooks.py +99 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Babandeep Singh
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,3 @@
1
+ include README.md
2
+ include LICENSE
3
+ recursive-include src/docintel/openapi *.yaml
@@ -0,0 +1,607 @@
1
+ Metadata-Version: 2.4
2
+ Name: docintel-platform
3
+ Version: 1.0.2
4
+ Summary: Document intelligence API and Python client for PDF OCR, PII detection, LLM structuring, matching, and summarization.
5
+ Author: Babandeep Singh
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/baban9/document-intelligence-platform
8
+ Project-URL: Repository, https://github.com/baban9/document-intelligence-platform
9
+ Project-URL: Documentation, https://github.com/baban9/document-intelligence-platform#readme
10
+ Project-URL: Issues, https://github.com/baban9/document-intelligence-platform/issues
11
+ Keywords: nlp,pdf,flask,document-ai,resume-matching,ocr,pii,presidio,openapi,document-intelligence
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.9
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Framework :: Flask
20
+ Classifier: Topic :: Text Processing
21
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.9
24
+ Description-Content-Type: text/markdown
25
+ Requires-Dist: flask>=3.0.3
26
+ Requires-Dist: werkzeug>=3.0.3
27
+ Requires-Dist: pymupdf>=1.24.10
28
+ Requires-Dist: scikit-learn>=1.5.2
29
+ Requires-Dist: networkx>=3.2.1
30
+ Requires-Dist: numpy>=1.26.4
31
+ Requires-Dist: gunicorn>=23.0.0
32
+ Requires-Dist: pyyaml>=6.0.2
33
+ Requires-Dist: requests>=2.32.3
34
+ Provides-Extra: dev
35
+ Requires-Dist: pytest>=8.3.3; extra == "dev"
36
+ Requires-Dist: build>=1.2.2; extra == "dev"
37
+ Requires-Dist: twine>=5.1.1; extra == "dev"
38
+ Requires-Dist: fakeredis>=2.26.2; extra == "dev"
39
+ Provides-Extra: ocr
40
+ Requires-Dist: easyocr>=1.7.2; extra == "ocr"
41
+ Requires-Dist: presidio-analyzer>=2.2.354; extra == "ocr"
42
+ Requires-Dist: spacy>=3.7.0; extra == "ocr"
43
+ Requires-Dist: opencv-python-headless>=4.10.0; extra == "ocr"
44
+ Requires-Dist: torch>=2.4.1; extra == "ocr"
45
+ Provides-Extra: ui
46
+ Requires-Dist: gradio>=4.44.0; extra == "ui"
47
+ Requires-Dist: requests>=2.32.3; extra == "ui"
48
+ Provides-Extra: llm
49
+ Requires-Dist: openai>=1.54.0; extra == "llm"
50
+ Provides-Extra: jobs
51
+ Requires-Dist: redis>=5.0.8; extra == "jobs"
52
+ Requires-Dist: rq>=1.16.2; extra == "jobs"
53
+ Provides-Extra: auth
54
+ Requires-Dist: flask-limiter>=3.8.0; extra == "auth"
55
+ Requires-Dist: PyJWT>=2.9.0; extra == "auth"
56
+ Requires-Dist: cryptography>=43.0.0; extra == "auth"
57
+ Provides-Extra: all
58
+ Requires-Dist: easyocr>=1.7.2; extra == "all"
59
+ Requires-Dist: presidio-analyzer>=2.2.354; extra == "all"
60
+ Requires-Dist: spacy>=3.7.0; extra == "all"
61
+ Requires-Dist: opencv-python-headless>=4.10.0; extra == "all"
62
+ Requires-Dist: torch>=2.4.1; extra == "all"
63
+ Requires-Dist: openai>=1.54.0; extra == "all"
64
+ Requires-Dist: redis>=5.0.8; extra == "all"
65
+ Requires-Dist: rq>=1.16.2; extra == "all"
66
+ Requires-Dist: flask-limiter>=3.8.0; extra == "all"
67
+ Requires-Dist: PyJWT>=2.9.0; extra == "all"
68
+ Requires-Dist: cryptography>=43.0.0; extra == "all"
69
+ Requires-Dist: gradio>=4.44.0; extra == "all"
70
+
71
+ # Document Intelligence Platform
72
+
73
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
74
+ [![Flask](https://img.shields.io/badge/flask-3.0+-green.svg)](https://flask.palletsprojects.com/)
75
+ [![Docker](https://img.shields.io/badge/docker-compose-ready-blue.svg)](docker-compose.yml)
76
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
77
+ [![Tests](https://img.shields.io/badge/tests-pytest-brightgreen.svg)](tests/)
78
+
79
+ Production-ready document AI: PDF annotation, scanned-document PII detection (EasyOCR + Presidio), LLM PDF structuring, resume matching, and extractive summarization. Ship as a REST API, a Gradio upload GUI, or both via Docker.
80
+
81
+ **Version:** 1.0.0
82
+
83
+ ---
84
+
85
+ ## Install from PyPI
86
+
87
+ ```bash
88
+ pip install docintel
89
+
90
+ # Full stack (OCR, LLM, jobs, auth, UI)
91
+ pip install "docintel[all]"
92
+ ```
93
+
94
+ **Python client:**
95
+
96
+ ```python
97
+ from docintel import DocintelClient
98
+
99
+ client = DocintelClient("http://127.0.0.1:5000", api_key="your-key")
100
+ result = client.match_resume(resume_text, job_description)
101
+ pdf_bytes = client.structure_pdf("scan.pdf", async_job=True)
102
+ ```
103
+
104
+ **Publish a release to PyPI** (maintainers): tag `v1.0.0` and push, or run `make publish-pypi` with `TWINE_USERNAME` / `TWINE_PASSWORD` or PyPI trusted publishing configured in GitHub Actions.
105
+
106
+ ---
107
+
108
+ ## Deploy in one command
109
+
110
+ No local Python setup required.
111
+
112
+ ```bash
113
+ git clone https://github.com/baban9/document-intelligence-platform.git
114
+ cd document-intelligence-platform
115
+ make docker-up
116
+ ```
117
+
118
+ | Service | URL | Use case |
119
+ |---------|-----|----------|
120
+ | **Gradio GUI** | http://127.0.0.1:7860 | Upload PDFs, no code |
121
+ | **REST API** | http://127.0.0.1:5000 | Integrations, curl, apps |
122
+ | Health | http://127.0.0.1:5000/health | Load balancer probe |
123
+ | API docs | http://127.0.0.1:5000/docs | Swagger UI (OpenAPI) |
124
+ | OpenAPI | http://127.0.0.1:5000/openapi.json | Machine-readable contract |
125
+ | Metrics | http://127.0.0.1:5000/metrics | Request counts and latency |
126
+
127
+ First startup can take a few minutes while EasyOCR and Presidio models download inside the container.
128
+
129
+ ```bash
130
+ make docker-logs # follow api + ui logs
131
+ make docker-down # stop services
132
+ ```
133
+
134
+ Optional overrides: copy `.env.example` to `.env` (ports, log level, worker count).
135
+
136
+ ---
137
+
138
+ ## Gradio upload GUI
139
+
140
+ Open http://127.0.0.1:7860 after `make docker-up` (or `make run-ui` locally).
141
+
142
+ | Tab | What it does |
143
+ |-----|--------------|
144
+ | **PDF regex annotate** | Search by pattern, highlight or redact |
145
+ | **Sensitive PDF (OCR + Presidio)** | Scanned docs: OCR, detect PII, annotate boxes |
146
+ | **PDF structure (LLM)** | Scanned or messy PDFs to curated structured PDF |
147
+ | **Resume matching** | Score resume vs job description |
148
+ | **Text summarization** | Extractive summary with TextRank |
149
+
150
+ The GUI calls the same REST API as external clients. Set `DOCINTEL_API_URL` if the API runs on a different host.
151
+
152
+ ---
153
+
154
+ ## What you get
155
+
156
+ | Capability | API | GUI |
157
+ |------------|-----|-----|
158
+ | PDF regex search and annotation | `POST /v1/pdf/annotate` | PDF regex annotate tab |
159
+ | Scanned PDF PII detection | `POST /v1/pdf/detect-sensitive` | Sensitive PDF tab |
160
+ | LLM PDF structuring | `POST /v1/pdf/structure` | PDF structure tab |
161
+ | Presidio entity catalog | `GET /v1/pdf/entities` | - |
162
+ | Resume vs job matching | `POST /v1/match/resume` | Resume matching tab |
163
+ | Extractive summarization | `POST /v1/text/summarize` | Text summarization tab |
164
+ | Health and metrics | `GET /health`, `GET /metrics` | - |
165
+
166
+ ---
167
+
168
+ ## Why this exists
169
+
170
+ HR, compliance, and research teams often maintain separate tools:
171
+
172
+ - a PDF highlighter or redaction script
173
+ - a resume keyword matcher
174
+ - a notebook for summarization
175
+
176
+ That split means duplicated config, no shared metrics, and broken workflows on **scanned PDFs** where text extraction returns empty. This platform unifies those flows behind one API and one upload GUI.
177
+
178
+ ---
179
+
180
+ ## Problems it solves
181
+
182
+ ### HR and recruiting
183
+
184
+ | Problem | Solution |
185
+ |---------|----------|
186
+ | Manual resume screening at scale | TF-IDF match score plus keyword overlap |
187
+ | Long ATS exports before phone screens | Extractive summary in seconds |
188
+ | Inconsistent reviewer shortlists | Same scoring logic every time |
189
+
190
+ ### Compliance and legal
191
+
192
+ | Problem | Solution |
193
+ |---------|----------|
194
+ | Regex search on digital contracts | `POST /v1/pdf/annotate` |
195
+ | **Scanned** contracts with no text layer | EasyOCR + Presidio on `POST /v1/pdf/detect-sensitive` |
196
+ | Redact SSN, email, phone before external share | Highlight or redact on exact bounding boxes |
197
+ | Audit trail | Structured JSON logs and `/metrics` |
198
+
199
+ ### Research intake
200
+
201
+ | Problem | Solution |
202
+ |---------|----------|
203
+ | Long reports need triage | TextRank summarization |
204
+ | Key terms buried in PDFs | Regex annotate or Presidio entity detection |
205
+
206
+ ### Before vs after
207
+
208
+ | Before | After |
209
+ |--------|-------|
210
+ | 3 scripts, 3 configs | 1 API + 1 GUI + 1 Docker deploy |
211
+ | Scanned PDFs fail regex tools | OCR fallback with Presidio PII boxes |
212
+ | Desktop-only redaction | Programmatic HTTP + downloadable output PDF |
213
+ | No observability | JSON logs, health check, metrics endpoint |
214
+
215
+ ---
216
+
217
+ ## Local development
218
+
219
+ ```bash
220
+ git clone https://github.com/baban9/document-intelligence-platform.git
221
+ cd document-intelligence-platform
222
+ make setup
223
+ make setup-ocr # EasyOCR + Presidio + spaCy en model
224
+ make setup-llm # OpenAI client for PDF structuring
225
+ make setup-ui # Gradio client
226
+ ```
227
+
228
+ **Terminal 1 (API):**
229
+
230
+ ```bash
231
+ make run
232
+ curl http://127.0.0.1:5000/health
233
+ ```
234
+
235
+ **Terminal 2 (GUI):**
236
+
237
+ ```bash
238
+ make run-ui
239
+ # open http://127.0.0.1:7860
240
+ ```
241
+
242
+ **Tests:**
243
+
244
+ ```bash
245
+ make test
246
+ ```
247
+
248
+ **Install extras:**
249
+
250
+ ```bash
251
+ pip install -e ".[dev]" # tests
252
+ pip install -e ".[ocr]" # scanned PDF pipeline
253
+ pip install -e ".[llm]" # LLM PDF structuring
254
+ pip install -e ".[ui]" # Gradio GUI
255
+ python -m spacy download en_core_web_sm
256
+ ```
257
+
258
+ ---
259
+
260
+ ## Architecture
261
+
262
+ Modular monolith: one Flask app, separate service modules, optional Gradio front end.
263
+
264
+ ```
265
+ Browser / curl Docker Compose
266
+ | |
267
+ v v
268
+ +-----------+ +-------+--------+
269
+ | Gradio | -- HTTP :5000 --> | Flask API |
270
+ | UI :7860 | | (Gunicorn) |
271
+ +-----------+ +-------+--------+
272
+ |
273
+ +-----------------------------+-----------------------------+
274
+ | | |
275
+ +-----v-----+ +-----v-----+ +-----v-----+
276
+ | PDF | | Matching | | Summary |
277
+ | service | | service | | service |
278
+ +-----------+ +-----------+ +-----------+
279
+ | | |
280
+ PyMuPDF regex TF-IDF cosine TextRank graph
281
+ EasyOCR (scanned) keyword overlap extractive output
282
+ Presidio PII boxes LLM PDF structure
283
+ ```
284
+
285
+ Decision records: [modular monolith](docs/adr/001-modular-monolith.md), [OCR + Presidio](docs/adr/002-ocr-presidio-pipeline.md)
286
+
287
+ ---
288
+
289
+ ## API reference
290
+
291
+ OpenAPI spec: `GET /openapi.json` | Interactive docs: `GET /docs`
292
+
293
+ ### Sensitive PDF detection (scanned + digital)
294
+
295
+ When native PDF text is empty, the service runs **EasyOCR (English)**, analyzes text with **Microsoft Presidio**, and returns a new PDF with highlights or redactions on bounding boxes. Optionally embeds an invisible text layer so the output stays searchable.
296
+
297
+ ```bash
298
+ curl -X POST http://127.0.0.1:5000/v1/pdf/detect-sensitive \
299
+ -F "file=@scanned_contract.pdf" \
300
+ -F "action=Highlight" \
301
+ -o marked_contract.pdf
302
+ ```
303
+
304
+ JSON report with findings:
305
+
306
+ ```bash
307
+ curl -X POST "http://127.0.0.1:5000/v1/pdf/detect-sensitive?format=json" \
308
+ -F "file=@scanned_contract.pdf" \
309
+ -F "action=Redact" \
310
+ -F "entities=EMAIL_ADDRESS,PHONE_NUMBER,US_SSN,CREDIT_CARD,PERSON"
311
+ ```
312
+
313
+ List Presidio entities (extend with [custom recognizers](https://microsoft.github.io/presidio/analyzer/adding_recognizers/)):
314
+
315
+ ```bash
316
+ curl http://127.0.0.1:5000/v1/pdf/entities
317
+ ```
318
+
319
+ | Field | Required | Description |
320
+ |-------|----------|-------------|
321
+ | `file` | Yes | PDF upload |
322
+ | `action` | No | `Highlight` (default), `Redact`, `Frame`, `Underline`, `Squiggly`, `Strikeout` |
323
+ | `entities` | No | Comma-separated Presidio types (default preset below) |
324
+ | `pattern` | No | Extra regex on top of Presidio |
325
+ | `force_ocr` | No | `true` to OCR every page |
326
+ | `add_text_layer` | No | `true` (default) adds searchable invisible text |
327
+ | `min_score` | No | Presidio confidence threshold (default `0.35`) |
328
+ | `async` | No | `true` queues the job (returns `202`); poll `GET /v1/jobs/<job_id>` |
329
+ | `callback_url` | No | Webhook URL when async job completes |
330
+
331
+ **Async mode:**
332
+
333
+ ```bash
334
+ curl -X POST "http://127.0.0.1:5000/v1/pdf/detect-sensitive?async=true" \
335
+ -H "Authorization: Bearer your-key" \
336
+ -F "file=@scanned_contract.pdf" \
337
+ -F "action=Highlight"
338
+ ```
339
+
340
+ **Default Presidio entities:** `EMAIL_ADDRESS`, `PHONE_NUMBER`, `US_SSN`, `CREDIT_CARD`, `US_BANK_NUMBER`, `US_DRIVER_LICENSE`, `US_ITIN`, `US_PASSPORT`, `PERSON`, `LOCATION`, `DATE_TIME`, `IP_ADDRESS`, `IBAN_CODE`, `MEDICAL_LICENSE`, `URL`.
341
+
342
+ ---
343
+
344
+ ### LLM PDF structuring (scanned to curated PDF)
345
+
346
+ Turn unstructured or scanned PDFs into a clean digital PDF. EasyOCR extracts text when the native layer is missing. An OpenAI-compatible LLM cleans and structures the content, then the service returns a curated typeset PDF or a searchable layer on the original pages.
347
+
348
+ ```bash
349
+ curl -X POST http://127.0.0.1:5000/v1/pdf/structure \
350
+ -F "file=@scanned_notes.pdf" \
351
+ -F "mode=curate" \
352
+ -o structured_notes.pdf
353
+ ```
354
+
355
+ | Field | Required | Description |
356
+ |-------|----------|-------------|
357
+ | `file` | Yes | PDF upload |
358
+ | `mode` | No | `curate` (default, new typeset PDF) or `searchable` (invisible text on original pages) |
359
+ | `force_ocr` | No | `true` to OCR every page |
360
+ | `redact_before_llm` | No | `true` masks Presidio PII before text is sent to the LLM |
361
+ | `callback_url` | No | Webhook URL notified when an async job completes or fails |
362
+ | `async` | No | `true` queues the job in Redis (returns `202`); `false` waits in the request (default) |
363
+
364
+ **Async mode (recommended for scanned PDFs):**
365
+
366
+ ```bash
367
+ # 1) Queue the job
368
+ curl -X POST "http://127.0.0.1:5000/v1/pdf/structure?async=true" \
369
+ -F "file=@scanned_notes.pdf" \
370
+ -F "mode=curate"
371
+
372
+ # 2) Poll until job_status is completed
373
+ curl http://127.0.0.1:5000/v1/jobs/<job_id>
374
+
375
+ # 3) Download from download_url in the poll response
376
+ ```
377
+
378
+ Start Redis and the worker locally: `make setup-jobs`, then `make run-worker` in a second terminal. Docker Compose starts `redis`, `api`, and `worker` automatically.
379
+
380
+ **Model used:** OpenAI **`gpt-4o-mini`** by default (set via `DOCINTEL_LLM_MODEL`). The service uses the official OpenAI Python client and any OpenAI-compatible endpoint if you set `DOCINTEL_LLM_BASE_URL`.
381
+
382
+ **Install LLM extras:**
383
+
384
+ ```bash
385
+ pip install -e ".[ocr,llm,jobs]"
386
+ ```
387
+
388
+ **Get an OpenAI API key**
389
+
390
+ Official guide: [OpenAI API quickstart](https://platform.openai.com/docs/quickstart)
391
+ Manage keys: [platform.openai.com/api-keys](https://platform.openai.com/api-keys)
392
+
393
+ 1. Create an account at [platform.openai.com](https://platform.openai.com) (or sign in).
394
+ 2. Open [API keys](https://platform.openai.com/api-keys) and click **Create new secret key**.
395
+ 3. Copy the key once (it is shown only at creation time).
396
+ 4. Add billing or credits on the OpenAI platform if required for your account.
397
+ 5. Export the key before starting the API:
398
+
399
+ ```bash
400
+ export DOCINTEL_LLM_API_KEY="sk-..."
401
+ export DOCINTEL_LLM_MODEL="gpt-4o-mini" # optional; this is the default
402
+ make run
403
+ ```
404
+
405
+ For Docker or persistent local use, copy `.env.example` to `.env` and set `DOCINTEL_LLM_API_KEY` there. Do not commit `.env` or share the key in git.
406
+
407
+ **Optional:** use another OpenAI-compatible provider by setting `DOCINTEL_LLM_BASE_URL` and the matching model name for that provider.
408
+
409
+ ---
410
+
411
+ ### PDF regex annotation
412
+
413
+ For digital PDFs with a text layer.
414
+
415
+ ```bash
416
+ curl -X POST http://127.0.0.1:5000/v1/pdf/annotate \
417
+ -F "file=@contract.pdf" \
418
+ -F "pattern=CONFIDENTIAL" \
419
+ -F "action=Redact" \
420
+ -o redacted_contract.pdf
421
+ ```
422
+
423
+ | Action | Description |
424
+ |--------|-------------|
425
+ | `Highlight` | Yellow highlight (default) |
426
+ | `Redact` | Black out matched text |
427
+ | `Frame` | Red bounding box |
428
+ | `Underline` / `Squiggly` / `Strikeout` | Text markup |
429
+ | `Remove` | Delete existing annotations |
430
+
431
+ Optional: `pages` (comma-separated, zero-based), `?format=json` for metadata + download URL.
432
+
433
+ ---
434
+
435
+ ### Resume matching
436
+
437
+ ```bash
438
+ curl -X POST http://127.0.0.1:5000/v1/match/resume \
439
+ -H "Content-Type: application/json" \
440
+ -d '{
441
+ "resume": "Python engineer with Flask, pytest, Docker, and NLP experience.",
442
+ "job_description": "Seeking Python developer with Flask, Docker, API, and NLP skills.",
443
+ "top_keywords": 10
444
+ }'
445
+ ```
446
+
447
+ ```json
448
+ {
449
+ "status": "ok",
450
+ "score": 42.15,
451
+ "matched_keywords": ["python", "flask", "docker", "nlp"],
452
+ "missing_keywords": ["developer", "api", "skills"]
453
+ }
454
+ ```
455
+
456
+ ---
457
+
458
+ ### Text summarization
459
+
460
+ ```bash
461
+ curl -X POST http://127.0.0.1:5000/v1/text/summarize \
462
+ -H "Content-Type: application/json" \
463
+ -d '{"text": "Your long document here...", "sentences": 3}'
464
+ ```
465
+
466
+ ---
467
+
468
+ ### Metrics
469
+
470
+ ```bash
471
+ curl http://127.0.0.1:5000/metrics
472
+ ```
473
+
474
+ Returns request counts, error counts, average latency, and per-endpoint breakdown. Metrics are per Gunicorn worker; use `WEB_CONCURRENCY=1` for OCR workloads (Docker default).
475
+
476
+ ---
477
+
478
+ ## Configuration
479
+
480
+ | Variable | Default | Purpose |
481
+ |----------|---------|---------|
482
+ | `DOCINTEL_HOST` | `127.0.0.1` | API bind address (`0.0.0.0` in Docker) |
483
+ | `DOCINTEL_PORT` | `5000` | API port |
484
+ | `DOCINTEL_UPLOAD_DIR` | `uploads` | PDF job storage |
485
+ | `DOCINTEL_LOG_LEVEL` | `INFO` | JSON log verbosity |
486
+ | `WEB_CONCURRENCY` | `1` | Gunicorn workers (keep at 1 for OCR) |
487
+ | `DOCINTEL_API_URL` | `http://127.0.0.1:5000` | Gradio UI backend URL |
488
+ | `DOCINTEL_LLM_API_KEY` | unset | OpenAI-compatible API key for `/v1/pdf/structure` |
489
+ | `DOCINTEL_LLM_MODEL` | `gpt-4o-mini` | Model name for structuring |
490
+ | `DOCINTEL_LLM_BASE_URL` | unset | Optional compatible API base URL |
491
+ | `DOCINTEL_API_KEYS` | unset | Comma-separated API keys (`Authorization: Bearer ...`) |
492
+ | `DOCINTEL_AUTH_REQUIRED` | `false` | Require auth on `/v1/*` when `true` or keys are set |
493
+ | `DOCINTEL_RATE_LIMIT_ENABLED` | `true` | Per-key rate limits via Redis |
494
+ | `DOCINTEL_OIDC_ISSUER` | unset | Optional OIDC issuer for JWT bearer tokens |
495
+ | `DOCINTEL_OIDC_AUDIENCE` | unset | Expected JWT audience |
496
+ | `DOCINTEL_OIDC_JWKS_URL` | unset | JWKS URL (defaults to issuer `/.well-known/jwks.json`) |
497
+ | `DOCINTEL_API_KEY` | unset | API key used by the Gradio UI client |
498
+ | `GRADIO_SERVER_NAME` | `127.0.0.1` | Gradio bind (`0.0.0.0` in Docker) |
499
+ | `GRADIO_SERVER_PORT` | `7860` | Gradio port |
500
+
501
+ ### API authentication
502
+
503
+ Protect `/v1/*` routes with API keys and optional OIDC JWTs.
504
+
505
+ ```bash
506
+ export DOCINTEL_API_KEYS="dev-key-1,dev-key-2"
507
+ export DOCINTEL_AUTH_REQUIRED=true
508
+
509
+ curl -H "Authorization: Bearer dev-key-1" \
510
+ http://127.0.0.1:5000/v1/pdf/entities
511
+ ```
512
+
513
+ **OIDC (enterprise SSO tokens):**
514
+
515
+ ```bash
516
+ export DOCINTEL_OIDC_ISSUER="https://your-idp.example.com"
517
+ export DOCINTEL_OIDC_AUDIENCE="docintel-api"
518
+ pip install -e ".[auth]"
519
+ ```
520
+
521
+ Send `Authorization: Bearer <jwt>` from your identity provider. API keys still work when both are configured.
522
+
523
+ Install auth extras: `pip install -e ".[auth]"` (Flask-Limiter + PyJWT).
524
+
525
+ ---
526
+
527
+ ## Project layout
528
+
529
+ ```
530
+ document-intelligence-platform/
531
+ src/docintel/
532
+ app.py Flask factory
533
+ ui.py Gradio upload GUI
534
+ wsgi.py Gunicorn entry
535
+ routes/ HTTP endpoints
536
+ services/
537
+ pdf/ PyMuPDF, EasyOCR, Presidio
538
+ matching/ TF-IDF resume scoring
539
+ summary/ TextRank summarizer
540
+ ops/ JSON logging, metrics
541
+ run.py Start API locally
542
+ run_ui.py Start Gradio locally
543
+ Dockerfile API + OCR stack image
544
+ docker-compose.yml api + ui services
545
+ docs/adr/ Architecture decisions
546
+ tests/ pytest suite
547
+ Makefile
548
+ pyproject.toml
549
+ ```
550
+
551
+ ---
552
+
553
+ ## Makefile commands
554
+
555
+ | Command | Description |
556
+ |---------|-------------|
557
+ | `make setup` | venv + core package |
558
+ | `make setup-ocr` | EasyOCR + Presidio + spaCy model |
559
+ | `make setup-llm` | OpenAI client for PDF structuring |
560
+ | `make build-dist` | Build PyPI wheel and sdist |
561
+ | `make publish-pypi` | Upload to PyPI with twine |
562
+ | `make setup-ui` | Gradio GUI dependencies |
563
+ | `make run` | Start API (:5000) |
564
+ | `make run-ui` | Start Gradio (:7860) |
565
+ | `make test` | Run pytest |
566
+ | `make docker-up` | Build and start API + UI containers |
567
+ | `make docker-down` | Stop containers |
568
+ | `make docker-logs` | Tail all service logs |
569
+
570
+ ---
571
+
572
+ ## Roadmap
573
+
574
+ | Milestone | Scope | Status |
575
+ |-----------|-------|--------|
576
+ | M1 | Project scaffold, health endpoint | Done |
577
+ | M2 | PDF regex annotation | Done |
578
+ | M3 | Resume matching | Done |
579
+ | M4 | Extractive summarization | Done |
580
+ | M5 | Docker, logging, metrics | Done |
581
+ | M5+ | OCR + Presidio scanned PDF pipeline | Done |
582
+ | M5+ | Gradio upload GUI | Done |
583
+ | M8 | LLM PDF structuring | Done |
584
+ | M6 | Offline eval harness | Planned |
585
+ | M7 | Production checklist | Planned |
586
+
587
+ Details: [docs/ROADMAP.md](docs/ROADMAP.md)
588
+
589
+ ---
590
+
591
+ ## Limits and notes
592
+
593
+ - OCR requests are CPU-heavy; expect higher latency on scanned PDFs.
594
+ - Presidio entity types are extensible; defaults cover common US PII.
595
+ - First EasyOCR run downloads models (~100MB+).
596
+ - LLM structuring sends page text to your configured model provider when native OCR text is used.
597
+ - Not intended for real-time collaborative editing or generative long-form writing.
598
+
599
+ ---
600
+
601
+ ## License
602
+
603
+ MIT. See [LICENSE](LICENSE).
604
+
605
+ ---
606
+
607
+ Built by [Babandeep Singh](https://github.com/baban9). Open an issue for bugs or feature requests.