varga 0.0.1__tar.gz

varga-0.0.1/.gitignore

@@ -0,0 +1,165 @@
+_docs/
+_proc/
+
+*.bak
+.gitattributes
+.last_checked
+.gitconfig
+*.bak
+*.log
+*~
+~*
+_tmp*
+tmp*
+tags
+*.pkg
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+conda/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.vscode
+*.swp
+
+# osx generated files
+.DS_Store
+.DS_Store?
+.Trashes
+ehthumbs.db
+Thumbs.db
+.idea
+
+# pytest
+.pytest_cache
+
+# tools/trust-doc-nbs
+docs_src/.last_checked
+
+# symlinks to fastai
+docs_src/fastai
+tools/fastai
+
+# link checker
+checklink/cookies.txt
+
+# .gitconfig is now autogenerated
+.gitconfig
+
+# Quarto installer
+.deb
+.pkg
+
+# Quarto
+.quarto
+
+# local vault + extracted assets
+*.db
+*.usearch
+# ...except the recorded model replies the notebook tests replay in CI. The sqlite side files are
+# transient, so they stay ignored.
+!nbs/chatcache/cache.db
+nbs/chatcache/*.db-wal
+nbs/chatcache/*.db-shm
+pdfs/
+assets/
+.venv/
+.kosha/

varga-0.0.1/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022, fastai
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

varga-0.0.1/PKG-INFO

@@ -0,0 +1,129 @@
+Metadata-Version: 2.5
+Name: varga
+Version: 0.0.1
+Summary: what kind of thing a document is, and the fields it holds: cue-scored doctypes and schema-constrained extraction
+Project-URL: Repository, https://github.com/vedicreader/varga
+Project-URL: Documentation, https://vedicreader.github.io/varga/
+Author-email: Karthik <karthik.rajgopal@hotmail.com>
+License: Apache-2.0
+License-File: LICENSE
+Keywords: doctype,document classification,information extraction,invoice,nbdev,schema,structured output
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.12
+Requires-Dist: fastcore>=1.12.31
+Requires-Dist: rahasya>=0.0.1
+Provides-Extra: keyphrase
+Requires-Dist: litesearch>=0.1.25; extra == 'keyphrase'
+Provides-Extra: llm
+Requires-Dist: rishi>=0.1.2; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# varga
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Two questions about one document. [`guess_type`](https://vedicreader.github.io/varga/core.html#guess_type) answers the first from cue phrases and
+countable evidence, with no model unless two kinds tie. [`as_schema`](https://vedicreader.github.io/varga/schema.html#as_schema) and [`structured`](https://vedicreader.github.io/varga/schema.html#structured) answer the
+second against a shape you name or invent.
+
+## Install
+
+``` sh
+uv add varga            # doctypes; no model, no network
+uv add 'varga[llm]'     # ...and `structured`, which asks one
+```
+
+## What is it
+
+``` python
+INVOICE = '''# INVOICE
+Invoice No: ACM-2024-0117
+Date: 2024-03-01
+Bill to: Contoso GmbH, Berlin
+From: Acme Supplies Ltd
+Total due: 1,240.00 EUR
+Payment terms: net 30
+'''
+r = guess_type(INVOICE)
+r.doctype, r.decisive, round(r.margin, 2)
+```
+
+    ('invoice', True, 0.4)
+
+`decisive` is the whole point: it says whether the cues settled it. When two kinds tie,
+`margin` is 0 and a caller knows to spend a model call. Nothing else here costs one.
+
+``` python
+signals(INVOICE).counts
+```
+
+    {'money': 1, 'date': 1, 'ref': 1, 'heading': 1, 'org': 2}
+
+## What is in it
+
+``` python
+from dataclasses import fields
+[f.name for f in fields(as_schema('invoice'))]
+```
+
+    ['number',
+     'date',
+     'due_date',
+     'vendor',
+     'vendor_tax_id',
+     'bill_to',
+     'ship_to',
+     'currency',
+     'subtotal',
+     'tax',
+     'total',
+     'payment_terms',
+     'items']
+
+Ten shapes are built in: `invoice`, `purchase_order`, `quote`, `receipt`, `catalogue`,
+`contract`, `resume`, `paper`, `meeting_notes`, `other`. Or write one on the spot:
+
+``` python
+[(f.name, f.type.__name__) for f in fields(as_schema('vendor:str, total:float, items:list'))]
+```
+
+    [('vendor', 'str'), ('total', 'float'), ('items', 'list')]
+
+`structured(chat, prompt, schema)` fills it. It prefers the model’s constrained mode and
+falls back to a JSON reply when that raises, so a model without tool calling still answers.
+
+## The doctypes
+
+``` python
+len(DOCTYPES), sorted(DOCTYPES)
+```
+
+Ten of them are work-product labels — `proposal`, `presentation`, `requirements_spec`,
+`technical_design`, `regulatory_guidance`, `procedure`, `qa_artifact`, `roadmap`, `claim`,
+`clinical_record` — and each is decisive on its own text without a model. Prose that merely
+borrows the vocabulary scores under 0.2 against every one of them.
+
+## What it depends on
+
+`fastcore` and [rahasya](https://github.com/vedicreader/rahasya), for the honorific-anchored
+name regex the entity leg shares with the privacy gate. One definition of what a name is.
+
+`rishi` is needed only by [`structured`](https://vedicreader.github.io/varga/schema.html#structured). `litesearch` adds keyphrases to `signals().ents` and is
+display only: no doctype score reads them, and [`signals`](https://vedicreader.github.io/varga/core.html#signals) works without it.
+
+## Development
+
+The notebooks in `nbs/` are the source; the modules are generated.
+
+``` sh
+pip install -e .
+nbdev_prepare
+```

varga-0.0.1/README.md

@@ -0,0 +1,102 @@
+# varga
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Two questions about one document. [`guess_type`](https://vedicreader.github.io/varga/core.html#guess_type) answers the first from cue phrases and
+countable evidence, with no model unless two kinds tie. [`as_schema`](https://vedicreader.github.io/varga/schema.html#as_schema) and [`structured`](https://vedicreader.github.io/varga/schema.html#structured) answer the
+second against a shape you name or invent.
+
+## Install
+
+``` sh
+uv add varga            # doctypes; no model, no network
+uv add 'varga[llm]'     # ...and `structured`, which asks one
+```
+
+## What is it
+
+``` python
+INVOICE = '''# INVOICE
+Invoice No: ACM-2024-0117
+Date: 2024-03-01
+Bill to: Contoso GmbH, Berlin
+From: Acme Supplies Ltd
+Total due: 1,240.00 EUR
+Payment terms: net 30
+'''
+r = guess_type(INVOICE)
+r.doctype, r.decisive, round(r.margin, 2)
+```
+
+    ('invoice', True, 0.4)
+
+`decisive` is the whole point: it says whether the cues settled it. When two kinds tie,
+`margin` is 0 and a caller knows to spend a model call. Nothing else here costs one.
+
+``` python
+signals(INVOICE).counts
+```
+
+    {'money': 1, 'date': 1, 'ref': 1, 'heading': 1, 'org': 2}
+
+## What is in it
+
+``` python
+from dataclasses import fields
+[f.name for f in fields(as_schema('invoice'))]
+```
+
+    ['number',
+     'date',
+     'due_date',
+     'vendor',
+     'vendor_tax_id',
+     'bill_to',
+     'ship_to',
+     'currency',
+     'subtotal',
+     'tax',
+     'total',
+     'payment_terms',
+     'items']
+
+Ten shapes are built in: `invoice`, `purchase_order`, `quote`, `receipt`, `catalogue`,
+`contract`, `resume`, `paper`, `meeting_notes`, `other`. Or write one on the spot:
+
+``` python
+[(f.name, f.type.__name__) for f in fields(as_schema('vendor:str, total:float, items:list'))]
+```
+
+    [('vendor', 'str'), ('total', 'float'), ('items', 'list')]
+
+`structured(chat, prompt, schema)` fills it. It prefers the model’s constrained mode and
+falls back to a JSON reply when that raises, so a model without tool calling still answers.
+
+## The doctypes
+
+``` python
+len(DOCTYPES), sorted(DOCTYPES)
+```
+
+Ten of them are work-product labels — `proposal`, `presentation`, `requirements_spec`,
+`technical_design`, `regulatory_guidance`, `procedure`, `qa_artifact`, `roadmap`, `claim`,
+`clinical_record` — and each is decisive on its own text without a model. Prose that merely
+borrows the vocabulary scores under 0.2 against every one of them.
+
+## What it depends on
+
+`fastcore` and [rahasya](https://github.com/vedicreader/rahasya), for the honorific-anchored
+name regex the entity leg shares with the privacy gate. One definition of what a name is.
+
+`rishi` is needed only by [`structured`](https://vedicreader.github.io/varga/schema.html#structured). `litesearch` adds keyphrases to `signals().ents` and is
+display only: no doctype score reads them, and [`signals`](https://vedicreader.github.io/varga/core.html#signals) works without it.
+
+## Development
+
+The notebooks in `nbs/` are the source; the modules are generated.
+
+``` sh
+pip install -e .
+nbdev_prepare
+```

varga-0.0.1/pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "varga"
+dynamic = ["version"]
+description = "what kind of thing a document is, and the fields it holds: cue-scored doctypes and schema-constrained extraction"
+readme = "README.md"
+requires-python = ">=3.12"
+license = {text = "Apache-2.0"}
+authors = [{name = "Karthik", email = "karthik.rajgopal@hotmail.com"}]
+keywords = ['nbdev', 'document classification', 'information extraction', 'structured output',
+            'invoice', 'doctype', 'schema']
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Text Processing",
+]
+dependencies = [
+    "fastcore>=1.12.31",
+    "rahasya>=0.0.1",
+]
+
+[project.optional-dependencies]
+# `structured` needs a model to ask; `signals` and `guess_type` need neither
+llm = ["rishi>=0.1.2"]
+# keyphrases in `signals().ents`; display only, no doctype depends on them
+keyphrase = ["litesearch>=0.1.25"]
+
+[project.urls]
+Repository = "https://github.com/vedicreader/varga"
+Documentation = "https://vedicreader.github.io/varga/"
+
+[project.entry-points.nbdev]
+varga = "varga._modidx:d"
+
+[tool.nbdev]
+
+[tool.hatch.build.targets.wheel]
+packages = ["varga"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/varga", "/README.md", "/pyproject.toml"]
+
+[tool.hatch.version]
+path = "varga/__init__.py"
+
+[dependency-groups]
+dev = ["ipykernel>=7.3.0", "nbdev>=3.3.4", "notebook>=7.6.1", "rishi>=0.1.2", "litesearch>=0.1.25"]

varga-0.0.1/varga/__init__.py

@@ -0,0 +1,3 @@
+__version__ = "0.0.1"
+from .core import *
+from .schema import *

varga-0.0.1/varga/_modidx.py

@@ -0,0 +1,27 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'main',
+                'doc_baseurl': '/varga',
+                'doc_host': 'https://vedicreader.github.io',
+                'git_url': 'https://github.com/vedicreader/varga',
+                'lib_path': 'varga'},
+  'syms': { 'varga.core': { 'varga.core._keyphrases': ('core.html#_keyphrases', 'varga/core.py'),
+                            'varga.core._noun_ents': ('core.html#_noun_ents', 'varga/core.py'),
+                            'varga.core.cue_scores': ('core.html#cue_scores', 'varga/core.py'),
+                            'varga.core.guess_type': ('core.html#guess_type', 'varga/core.py'),
+                            'varga.core.signals': ('core.html#signals', 'varga/core.py')},
+            'varga.schema': { 'varga.schema.Catalogue': ('schema.html#catalogue', 'varga/schema.py'),
+                              'varga.schema.Contract': ('schema.html#contract', 'varga/schema.py'),
+                              'varga.schema.Invoice': ('schema.html#invoice', 'varga/schema.py'),
+                              'varga.schema.MeetingNotes': ('schema.html#meetingnotes', 'varga/schema.py'),
+                              'varga.schema.Paper': ('schema.html#paper', 'varga/schema.py'),
+                              'varga.schema.Receipt': ('schema.html#receipt', 'varga/schema.py'),
+                              'varga.schema.Resume': ('schema.html#resume', 'varga/schema.py'),
+                              'varga.schema.Summary': ('schema.html#summary', 'varga/schema.py'),
+                              'varga.schema._json_reply': ('schema.html#_json_reply', 'varga/schema.py'),
+                              'varga.schema._norm': ('schema.html#_norm', 'varga/schema.py'),
+                              'varga.schema._split_reasoning': ('schema.html#_split_reasoning', 'varga/schema.py'),
+                              'varga.schema.as_schema': ('schema.html#as_schema', 'varga/schema.py'),
+                              'varga.schema.dyn_schema': ('schema.html#dyn_schema', 'varga/schema.py'),
+                              'varga.schema.schema_str': ('schema.html#schema_str', 'varga/schema.py'),
+                              'varga.schema.structured': ('schema.html#structured', 'varga/schema.py')}}}

varga-0.0.1/varga/core.py

@@ -0,0 +1,265 @@
+"""what kind of thing a document is, decided by cues before a model is asked
+
+Docs: https://vedicreader.github.io/varga/core.html.md"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
+
+# %% auto #0
+__all__ = ['SIGNALS', 'DOCTYPES', 'KIND_HINT', 'MIN_SCORE', 'MIN_MARGIN', 'KIND_BONUS', 'signals', 'cue_scores', 'guess_type']
+
+# %% ../nbs/00_core.ipynb #50d7feae
+import re
+from collections import Counter
+from fastcore.all import AttrDict, L
+from rahasya import NER_CHARS, PERSON_HON
+
+# %% ../nbs/00_core.ipynb #15f3fd04
+SIGNALS = dict(
+    # what a document *contains*, as cheaply as a regex can tell: the evidence a type is scored on
+    money    = r'[$€£¥₹]\s?\d[\d,]*(?:\.\d+)?|\b\d[\d,]*\.\d{2}\s?(?:usd|eur|gbp|inr|jpy|cad|aud)\b'
+               r'|\b(?:usd|eur|gbp|inr|jpy|cad|aud)\s?\d[\d,]*',
+    date     = r'\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}[/.]\d{1,2}[/.]\d{2,4}\b'
+               r'|\b\d{1,2} (?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]* \d{2,4}\b'
+               r'|\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]* \d{1,2},? \d{4}\b',
+    time     = r'\b\d{1,2}:\d{2}(?::\d{2})?\s?(?:am|pm)?\b',
+    ref      = r'\b(?:invoice|inv|order|p\.?o|ref(?:erence)?|receipt|bill|account|sku|part|item)\s*'
+               r'(?:no\.?|number|id)?\s*[:#]\s*[a-z0-9][a-z0-9\-/]{2,}\b',
+    qty      = r'\b(?:qty|quantity|units?|pcs?|nos?)\b\s*[:.]?\s*\d+|\b\d+\s?(?:x|×)\s?\d',
+    percent  = r'\b\d{1,3}(?:\.\d+)?\s?%',
+    tax      = r'\b(?:vat|gst|hsn|sac|tin|ein|abn|sales tax|tax id|withholding)\b',
+    email    = r'\b[\w.+-]+@[\w-]+\.[\w.]{2,}\b',
+    url      = r'https?://\S+',
+    citation = r'\[\d+\]|\bet al\.|\bdoi:|\barxiv:',
+    code     = r'^\s*(?:def|class|function|const|let|var|import|from|package|#include)\s+\w',
+    table    = r'^\s*\|.+\|\s*$',
+    heading  = r'^#{1,6} \S',
+    blank    = r'_{4,}|\[ ?\]',
+)
+_SIG = {k: re.compile(v, re.I|re.M) for k, v in SIGNALS.items()}
+# entity extraction runs on the head of a document (`NER_CHARS`, from rahasya): that is where
+# its type is decided
+
+# handrolled entity extraction: ORG by legal suffix, PERSON by honorific, LAW by instrument type
+_ORG_SUFF = re.compile(
+    r'(?<![a-z,\.])([A-Z][A-Za-z0-9\'\-& ]{1,50}?)\s+'
+    r'(?:Ltd\.?|Limited|Corp\.?|Corporation|Inc\.?|LLC|LLP|GmbH|AG|SA|NV|BV|Pty\.?|PLC'
+    r'|Group\b|Holdings?\b|Associates?\b|Partners?\b|Ventures?\b'
+    r'|University\b|College\b|Institute\b|Hospital\b|Foundation\b'
+    r'|Bank\b|Fund\b|Trust\b|Council\b|Authority\b|Commission\b'
+    r'|Ministry\b|Department\b|Agency\b|Bureau\b)\b'
+)
+# the PERSON arm is rahasya's, so a name means the same thing to the doctype scorer and to the
+# privacy gate. One regex, measured once.
+_PERSON_HON = PERSON_HON
+_LAW_INST = re.compile(
+    r'\b([A-Z][A-Za-z ]{2,50}?)\s+'
+    r'(?:Act(?:\s+\d{4})?|Regulations?\b|Directive\b|Ordinance\b|Statute\b|Treaty\b|Convention\b|Amendment\b)'
+)
+
+# %% ../nbs/00_core.ipynb #b9d6a10c
+def _noun_ents(text:str, limit:int=40) -> L:
+    'ORG, PERSON and LAW entities via regex; no model weight.'
+    seen, out = set(), L()
+    def _add(t, label):
+        n = re.sub(r'\s+', ' ', t.strip())[:60]
+        k = n.lower()
+        if n and k not in seen: seen.add(k); out.append((n, label))
+    persons = list(_PERSON_HON.finditer(text))
+    for m in _ORG_SUFF.finditer(text):
+        start = max((p.end() for p in persons if p.start() < m.end() and p.end() > m.start()), default=m.start())
+        org = re.sub(r'^\s*(?:of|at|from)\s+', '', text[start:m.end()])
+        _add(org, 'ORG')
+    for m in persons: _add(m.group(1), 'PERSON')
+    for m in _LAW_INST.finditer(text): _add(m.group(1).strip(), 'LAW')
+    return out[:limit]
+
+# %% ../nbs/00_core.ipynb #24e79db7
+def _keyphrases(text:str) -> tuple:
+    'litesearch keyphrases when it is installed, nothing when it is not. Display only: `cue_scores` reads the ORG/PERSON/LAW labels, never these.'
+    try: from litesearch import text_entities
+    except ImportError: return ()
+    return text_entities(text)
+
+# %% ../nbs/00_core.ipynb #ef3ce3c0
+def signals(text:str,            # the document text
+            ner:bool=True,       # run entity extraction (noun entities + keyphrases)
+            limit:int=20,        # entities kept
+) -> AttrDict:
+    'Countable evidence in one document: `counts` per signal, plus the entities it names.'
+    counts = {k: len(p.findall(text or '')) for k, p in _SIG.items()}
+    noun = L(_noun_ents((text or '')[:NER_CHARS]) if ner else ()).map(
+        lambda t: AttrDict(label=t[1], text=t[0]))
+    kws  = L(_keyphrases((text or '')[:NER_CHARS]) if ner else ()).map(
+        lambda t: AttrDict(label=(t[1] or 'KEYPHRASE').upper(), text=re.sub(r'\s+', ' ', t[0].strip())[:60])
+    ).filter(lambda e: e.text and e.label not in {'ORG', 'PERSON', 'LAW', 'PRODUCT', 'GPE'})
+    found = noun + kws
+    labels = dict(Counter(e.label.lower() for e in found))
+    # labels count everything found; limit caps display only (cue_scores reads labels)
+    if noun:
+        counts = {k: counts.get(k, 0) + (labels.get(k, 0) if k not in counts else 0)
+                  for k in (*counts, *(l for l in labels if l not in ('keyphrase', 'term')))}
+    return AttrDict(counts={k: v for k, v in counts.items() if v}, labels=labels,
+                    ents=found.sorted(key=lambda e: e.label in ('KEYPHRASE', 'TERM'))[:limit],
+                    method=('ner+regex' if noun else 'keyphrase+regex' if kws else 'regex'))
+
+# %% ../nbs/00_core.ipynb #e77bc6be
+DOCTYPES = {
+    # label: the cue phrases that are evidence for it, the regex signals it needs, and the
+    # entity labels it expects
+    'invoice': dict(needs=('money',), ents=('ORG',), cues=(
+        r'\b(?:tax )?invoice\b', r'\b(?:amount|balance|total) due\b', r'\bbill(?:ed)? to\b|\bremit\b',
+        r'\bpayment terms?\b|\bnet \d{1,3}\b', r'\bsub-?total\b', r'\b(?:vat|gst|sales tax)\b')),
+    'receipt': dict(needs=('money',), ents=('ORG',), cues=(
+        r'\breceipt\b', r'\bthank you for your (?:order|purchase|payment)\b',
+        r'\bcard\b[^\n]{0,16}\bending\b|\bcash tendered\b|\bchange due\b',
+        r'\btransaction (?:id|no)\b|\bauth(?:orisation|orization) code\b',
+        r'\bpaid\b|\bpayment received\b', r'\bmerchant\b|\bstore #\s?\d+\b')),
+    'purchase_order': dict(needs=('ref',), ents=('ORG',), cues=(
+        r'\bpurchase order\b|\bp\.?o\.?\s*(?:no|number|#)', r'\bship(?:[ -]?to)?\b',
+        r'\bdelivery date\b|\brequested delivery\b', r'\bvendor\b|\bsupplier\b',
+        r'\brequisition\b', r'\bunit price\b')),
+    'quote': dict(needs=('money',), ents=('ORG',), cues=(
+        r'\bquotation\b|\bquote\s*(?:no|number|#)|\bestimate\b', r'\bvalid (?:until|for|through)\b',
+        r'\bunit price\b', r'\bterms and conditions\b',
+        r'\bwe are pleased to (?:quote|offer)\b|\bno obligation\b')),
+    'catalogue': dict(needs=('money',), ents=(), cues=(
+        r'\bcatalog(?:ue)?\b|\bprice list\b|\bproduct list\b',
+        r'\bsku\b|\bmodel\s*(?:no|number)\b|\bpart\s*(?:no|number)\b',
+        r'\b(?:in|out of) stock\b|\bavailability\b', r'\bper (?:unit|pack|case|kg|litre|liter)\b',
+        r'\bspecifications?\b|\bdimensions\b', r'\badd to (?:cart|basket)\b')),
+    'contract': dict(needs=(), ents=('ORG', 'LAW'), cues=(
+        r'\bagreement\b|\bcontract\b', r'\bparties\b|\bby and between\b',
+        r'\bhereby\b|\bwhereas\b|\bhereinafter\b', r'\bshall\b',
+        r'\bgoverning law\b|\bjurisdiction\b|\btermination\b',
+        r'\bconfidential(?:ity)?\b|\bindemnif|\bliability\b')),
+    'resume': dict(needs=('email',), ents=('PERSON', 'ORG'), cues=(
+        r'\b(?:curriculum vitae|resum[eé])\b',
+        r'\bwork experience\b|\bemployment history\b|\bprofessional experience\b',
+        r'\beducation\b', r'\bskills\b', r'\bcertification',
+        r'\breferences available\b|linkedin\.com/in/')),
+    'paper': dict(needs=('citation',), ents=('ORG', 'PERSON'), cues=(
+        r'\babstract\b', r'\bintroduction\b', r'\brelated work\b|\bmethodology\b|\bexperiments?\b',
+        r'\breferences\b|\bbibliography\b', r'\bwe (?:propose|present|show|evaluate)\b',
+        r'\bdoi:|\barxiv:')),
+    'report': dict(needs=(), ents=('ORG',), cues=(
+        r'\bexecutive summary\b', r'\bfindings\b|\bconclusions?\b|\brecommendations?\b',
+        r'\bq[1-4] \d{4}\b|\bfiscal year\b|\bfy\d{2}\b|\bquarter\b',
+        r'\btable \d+\b|\bfigure \d+\b|\bappendix\b',
+        r'\byear[- ]on[- ]year\b|\brevenue\b|\bmargin\b')),
+    # --- work-product labels ---
+    'proposal': dict(needs=(), ents=('ORG',), cues=(
+        r'\bproposal\b|\brfp response\b|\brequest for proposal\b', r'\bproposed approach\b|\bour approach\b',
+        r'\bscope of work\b|\bwork ?plan\b', r'\bdeliverables?\b', r'\bfees?\b|\binvestment\b',
+        r'\bproject team\b|\bteam structure\b')),
+    'presentation': dict(needs=(), ents=(), cues=(
+        r'\bslide deck\b|\bthis (?:deck|presentation)\b|\bslide \d+\b',
+        r'\bkey takeaways?\b|\bkey messages?\b', r'\bfor discussion\b|\bquestions? for discussion\b',
+        r'\bexecutive update\b|\bsteering committee\b|\bboard update\b',
+        r'\bappendix slides?\b|\bback-?up slides?\b',
+        r'\bagenda for (?:today|this session)\b|\bwalk(?:ing)? through\b')),
+    'requirements_spec': dict(needs=(), ents=(), cues=(
+        r'\brequirements? specification\b|\bproduct requirements?\b', r'\bacceptance criteria\b',
+        r'\bfunctional requirements?\b|\bnon[- ]functional requirements?\b', r'\buser stor(?:y|ies)\b',
+        r'\brequirement id\b|\btraceability matrix\b', r'\bthe system shall\b|\bmust be able to\b')),
+    'technical_design': dict(needs=(), ents=(), cues=(
+        r'\bsolution design\b|\btechnical design\b|\bdesign document\b',
+        r'\bdata architecture\b|\bsystem architecture\b|\barchitecture (?:diagram|overview)\b',
+        r'\bdata flow\b|\bsequence diagram\b|\bcomponent (?:diagram|view)\b',
+        r'\bapplication hosting\b|\bnon[- ]functional\b|\bscalability\b',
+        r'\bintegration (?:points?|patterns?|layer)\b|\binterface (?:contract|spec)\b',
+        r'\bsubnet\b|\bapi gateway\b|\bdeployment (?:topology|diagram)\b')),
+    'regulatory_guidance': dict(needs=(), ents=('LAW', 'ORG'), cues=(
+        r'\bregulation \(?(?:eu|ec)\b|\bmedical device regulation\b', r'\bcompetent authority\b',
+        r'\bnotified bod(?:y|ies)\b|\bconformity assessment\b', r'\bauthori[sz]ed representative\b',
+        r'\btechnical documentation\b|\btechnical file\b', r'\barticle \d+\b|\bannex [ivx]+\b')),
+    'procedure': dict(needs=(), ents=('ORG',), cues=(
+        r'\bstandard operating procedure\b|\bs\.?o\.?p\.?\b|\bwork instruction\b',
+        r'\bprocess owner\b|\bresponsibilit(?:y|ies)\b', r'\bapproval workflow\b|\bapproval authority\b',
+        r'\bescalation\b|\bexception handling\b', r'\brevision history\b|\bchange control\b',
+        r'\bthis procedure (?:applies|describes|covers)\b|\bscope and purpose\b')),
+    'qa_artifact': dict(needs=(), ents=(), cues=(
+        r'\btest (?:case|plan|script|protocol)\b', r'\bexpected results?\b|\bactual results?\b',
+        r'\bsteps to reproduce\b|\bdefects?\b|\bbug report\b',
+        r'\bdeviation log\b|\bdiscrepanc(?:y|ies)\b',
+        r'\btemplate fidelity\b|\bvalidation report\b|\bqa audit\b',
+        r'\bpass/fail\b|\btest (?:results?|status|evidence)\b')),
+    'roadmap': dict(needs=(), ents=('ORG',), cues=(
+        r'\broadmap\b', r'\bmilestones?\b', r'\bphase [0-9a-z]+\b|\bnow[,/ ]+next[,/ ]+later\b',
+        r'\bnext (?:30|60|90) days\b|\bquarterly plan\b', r'\btarget date\b|\bplanned date\b',
+        r'\bproduct backlog\b|\bdelivery plan\b|\bthemes?\b')),
+    # claim vs clinical_record share a dated-person shape; cues must separate them
+    'claim': dict(needs=('date',), ents=('PERSON', 'ORG'), cues=(
+        r'\bclaim (?:number|no\.?|form|reference)\b|\bclaim id\b',
+        r'\bincident date\b|\bdate of loss\b', r'\bclaimant\b|\binsured\b|\bpolicyholder\b',
+        r'\bpolicy (?:number|no\.?|#)\b|\bcoverage\b', r'\b(?:loss )?adjuster\b|\bassessor\b',
+        r'\bsettlement\b|\bdeductible\b|\bexcess\b|\bliability accepted\b')),
+    'clinical_record': dict(needs=('date',), ents=('PERSON', 'ORG'), cues=(
+        r'\bpatient\b|\bencounter\b', r'\bdiagnos(?:is|es|ed)\b|\bicd-?10\b',
+        r'\bpresenting complaint\b|\bclinical (?:history|notes?)\b',
+        r'\bprescri(?:bed|ption)\b|\bmedication\b|\bdosage\b',
+        r'\bdischarge summary\b|\breferral letter\b|\bclinician\b|\bphysician\b',
+        r'\badverse events?\b|\bdevice malfunction\b|\bside effects?\b')),
+    'documentation': dict(needs=('heading',), ents=(), cues=(
+        r'\binstallation\b|\bgetting started\b|\bquick ?start\b',
+        r'\busage\b|\bexamples?\b|\bapi reference\b', r'\bparameters?\b|\breturns\b|\barguments?\b',
+        r'\bpip install\b|\bnpm install\b|\bdocker run\b', r'\bconfiguration\b|\bsee also\b',
+        r'^```')),
+    'code': dict(needs=('code',), ents=(), cues=(
+        r'^\s*(?:def|class)\s+\w+|^\s*(?:function|const|let|var)\s+\w+',
+        r'^\s*(?:import|from|#include|package|use)\s+\w', r'\breturn\b',
+        r'^\s*(?://|#|/\*)', r'\b(?:if|for|while)\s*\(', r'\b(?:public|private|static|async)\b')),
+    'email': dict(needs=('email',), ents=('PERSON',), cues=(
+        r'^(?:from|to|cc|bcc|subject|sent):', r'^\s*(?:dear|hi|hello)\b',
+        r'\b(?:best|kind) regards\b|\bsincerely\b|\bthanks,\b',
+        r'\bforwarded message\b|\bwrote:\s*$', r'\bunsubscribe\b')),
+    'meeting_notes': dict(needs=(), ents=('PERSON',), cues=(
+        r'\b(?:meeting|call) notes\b|\bminutes\b', r'\battendees\b|\bparticipants\b|\bpresent\b',
+        r'\bagenda\b', r'\baction items?\b|\bnext steps\b|\bfollow[- ]ups?\b',
+        r'\bdecided\b|\bdecisions?\b|\bowner\b|\bdue by\b')),
+    'form': dict(needs=('blank',), ents=(), cues=(
+        r'\bplease (?:complete|fill|print|sign)\b', r'\bfor office use only\b',
+        r'\bapplicant\b|\bapplication (?:form|for)\b', r'\bsignature\b|\bdate signed\b',
+        r'\b(?:full name|surname|given names?|date of birth|dob)\b', r'_{4,}')),
+    'transcript': dict(needs=('time',), ents=('PERSON',), cues=(
+        r'\btranscript\b', r'^\s*\[?\d{1,2}:\d{2}', r'^\s*(?:speaker \d|[A-Z][a-z]+\s?[A-Z]?[a-z]*):',
+        r'\b(?:um|uh|you know|i mean)\b', r'\bwelcome (?:back|to)\b|\bthanks for (?:watching|listening)\b')),
+    'article': dict(needs=(), ents=('PERSON', 'ORG'), cues=(
+        r'\bpublished\b|\bposted (?:on|by)\b', r'\bread more\b|\bshare this\b|\bcomments?\b',
+        r'\bsubscribe\b|\bnewsletter\b', r'\baccording to\b', r'\btags?:|\bcategor(?:y|ies):')),
+}
+_CUES = {l: [re.compile(c, re.I|re.M) for c in d['cues']] for l, d in DOCTYPES.items()}
+
+# arrival kind is strong evidence (youtube -> transcript)
+KIND_HINT = dict(youtube='transcript', arxiv='paper', code='code')
+MIN_SCORE, MIN_MARGIN, KIND_BONUS = 0.4, 0.12, 0.2
+
+def cue_scores(text:str, sig=None) -> dict:
+    'Every doctype scored against `text`, best first: the categorisation no model is needed for.'
+    sig = sig if sig is not None else signals(text)
+    ner, out = sig.method.startswith('ner'), {}
+    for lbl, d in DOCTYPES.items():
+        cues = [p for p in _CUES[lbl] if p.search(text or '')]
+        need = [s for s in d['needs'] if sig.counts.get(s)]
+        ents = [e for e in d['ents'] if sig.labels.get(e.lower())] if ner else []
+        legs = [(0.6, len(cues)/len(_CUES[lbl]))]
+        if d['needs']: legs.append((0.25, len(need)/len(d['needs'])))
+        if ner and d['ents']: legs.append((0.15, len(ents)/len(d['ents'])))
+        w = sum(x for x, _ in legs)
+        out[lbl] = round(sum(x*f for x, f in legs)/w, 3)
+    return dict(sorted(out.items(), key=lambda kv: -kv[1]))
+
+def guess_type(text:str,             # the document text
+               sig=None,             # a signals() result, computed if not given
+               kind:str=None,        # the vault kind that acquired it, if known
+               min_score:float=MIN_SCORE,    # below this the cues have not decided
+               min_margin:float=MIN_MARGIN,  # runner-up this close means they have not either
+) -> AttrDict:
+    'The best doctype for `text` from cues alone, and whether that guess is worth trusting.'
+    sig = sig if sig is not None else signals(text)
+    sc = cue_scores(text, sig)
+    if kind in KIND_HINT: sc[KIND_HINT[kind]] = round(min(1.0, sc[KIND_HINT[kind]] + KIND_BONUS), 3)
+    ranked = sorted(sc.items(), key=lambda kv: -kv[1])
+    (top, best), (_, second) = ranked[0], (ranked[1] if len(ranked) > 1 else ('', 0.0))
+    return AttrDict(doctype=top if best else 'other', score=best, margin=round(best-second, 3),
+                    decisive=best >= min_score and best-second >= min_margin,
+                    scores=dict(ranked[:5]), method=sig.method)

varga-0.0.1/varga/schema.py

@@ -0,0 +1,226 @@
+"""the shape an extraction fills, and the call that fills it
+
+Docs: https://vedicreader.github.io/varga/schema.html.md"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_schema.ipynb.
+
+# %% auto #0
+__all__ = ['SCHEMAS', 'FIELD_TYPES', 'Invoice', 'Receipt', 'Catalogue', 'Contract', 'Resume', 'Paper', 'MeetingNotes', 'Summary',
+           'dyn_schema', 'as_schema', 'schema_str', 'structured']
+
+# %% ../nbs/01_schema.ipynb #e423da84
+import json, re, warnings
+from dataclasses import dataclass, fields, is_dataclass, make_dataclass, asdict
+from typing import get_origin
+from fastcore.all import AttrDict, L
+
+# %% ../nbs/01_schema.ipynb #a125d2cb
+@dataclass
+class Invoice:
+    """An invoice, purchase order or quotation.
+
+    items: one entry per line, each {description, qty, unit_price, amount}.
+    Amounts are bare numbers with the currency in `currency`; dates are ISO (2024-03-01).
+    """
+    number:str = ''
+    date:str = ''
+    due_date:str = ''
+    vendor:str = ''
+    vendor_tax_id:str = ''
+    bill_to:str = ''
+    ship_to:str = ''
+    currency:str = ''
+    subtotal:float = 0
+    tax:float = 0
+    total:float = 0
+    payment_terms:str = ''
+    items:list[dict] = None
+
+# %% ../nbs/01_schema.ipynb #0bb49294
+@dataclass
+class Receipt:
+    """A receipt for a completed payment.
+
+    items: one entry per line, each {description, qty, amount}. `paid_with` is the tender or the
+    last digits of the card. Dates are ISO (2024-03-01)."""
+    merchant:str = ''
+    date:str = ''
+    transaction_id:str = ''
+    currency:str = ''
+    subtotal:float = 0
+    tax:float = 0
+    total:float = 0
+    paid_with:str = ''
+    items:list[dict] = None
+
+# %% ../nbs/01_schema.ipynb #5bb4dff8
+@dataclass
+class Catalogue:
+    """A product catalogue, price list or listing page.
+
+    products: one entry per product, each {sku, name, category, price, currency, availability, url}.
+    Prices are bare numbers."""
+    name:str = ''
+    vendor:str = ''
+    currency:str = ''
+    updated:str = ''
+    n_products:int = 0
+    products:list[dict] = None
+
+# %% ../nbs/01_schema.ipynb #57fbc6cd
+@dataclass
+class Contract:
+    """An agreement between parties.
+
+    parties: the legal names. obligations: what each side must do. Dates are ISO (2024-03-01)."""
+    title:str = ''
+    parties:list[str] = None
+    effective_date:str = ''
+    end_date:str = ''
+    term:str = ''
+    value:str = ''
+    governing_law:str = ''
+    termination:str = ''
+    obligations:list[str] = None
+
+# %% ../nbs/01_schema.ipynb #24e91acd
+@dataclass
+class Resume:
+    """One person's CV.
+
+    experience: one entry per role, each {employer, title, start, end, summary}.
+    education: one entry per qualification, each {institution, qualification, year}."""
+    name:str = ''
+    email:str = ''
+    phone:str = ''
+    location:str = ''
+    headline:str = ''
+    years_experience:float = 0
+    skills:list[str] = None
+    experience:list[dict] = None
+    education:list[dict] = None
+
+# %% ../nbs/01_schema.ipynb #e81c81f6
+@dataclass
+class Paper:
+    """An academic paper. authors: names in order. findings: the claims the paper actually makes."""
+    title:str = ''
+    authors:list[str] = None
+    venue:str = ''
+    year:str = ''
+    doi:str = ''
+    abstract:str = ''
+    method:str = ''
+    findings:list[str] = None
+    limitations:list[str] = None
+
+# %% ../nbs/01_schema.ipynb #17cbf009
+@dataclass
+class MeetingNotes:
+    """Notes from one meeting. actions: one entry per commitment, each {what, owner, due}."""
+    title:str = ''
+    date:str = ''
+    attendees:list[str] = None
+    topics:list[str] = None
+    decisions:list[str] = None
+    actions:list[dict] = None
+
+# %% ../nbs/01_schema.ipynb #70ed9b52
+@dataclass
+class Summary:
+    """What one document says, when no more specific shape fits.
+
+    entities: the organisations, people and places it names. dates: ISO where the document allows."""
+    title:str = ''
+    doctype:str = ''
+    about:str = ''
+    key_points:list[str] = None
+    entities:list[str] = None
+    dates:list[str] = None
+    numbers:list[str] = None
+    open_questions:list[str] = None
+
+# %% ../nbs/01_schema.ipynb #36192e1e
+# PO and quotation share the Invoice schema
+SCHEMAS = dict(invoice=Invoice, purchase_order=Invoice, quote=Invoice, receipt=Receipt,
+               catalogue=Catalogue, contract=Contract, resume=Resume, paper=Paper,
+               meeting_notes=MeetingNotes, other=Summary)
+
+FIELD_TYPES = {'str':str, 'text':str, 'int':int, 'float':float, 'number':float, 'num':float, 'bool':bool,
+               'list':list, 'dict':dict, 'strs':list[str], 'dicts':list[dict],
+               'list[str]':list[str], 'list[dict]':list[dict]}
+_DFLT = {str: '', int: 0, float: 0.0, bool: False}
+
+# %% ../nbs/01_schema.ipynb #7ab433ab
+def dyn_schema(spec,                    # 'vendor:str, total:float, items:dicts', or {'vendor':'str'}, or ['vendor']
+               name:str='Extracted',    # class name, which the model sees
+               doc:str=None,            # class docstring, which the model also sees
+) -> type:
+    'A dataclass built at runtime from a field spec: the shape of an answer, named in one string.'
+    if isinstance(spec, str): spec = [p for p in re.split(r'[,\n]', spec) if p.strip()]
+    if isinstance(spec, dict): spec = [f'{k}:{v}' for k, v in spec.items()]
+    flds = []
+    for p in spec:
+        nm, _, ty = (p if isinstance(p, str) else ':'.join(p)).partition(':')
+        t = FIELD_TYPES.get(ty.strip().lower() or 'str', str)
+        flds.append((re.sub(r'\W', '_', nm.strip()), t, _DFLT.get(t, None)))
+    if not flds: raise ValueError(f'no fields in schema spec {spec!r}')
+    return make_dataclass(re.sub(r'\W', '', name) or 'Extracted', flds,
+                          namespace=dict(__doc__=doc or 'The fields to pull out of the document.'))
+
+# %% ../nbs/01_schema.ipynb #86058550
+def as_schema(spec, name:str='Extracted', doc:str=None) -> type:
+    'Whatever names a shape, as a dataclass: a `SCHEMAS` key, a dataclass, or a `dyn_schema` spec.'
+    if is_dataclass(spec): return spec
+    if isinstance(spec, str) and spec.strip() in SCHEMAS: return SCHEMAS[spec.strip()]
+    return dyn_schema(spec, name=name, doc=doc)
+
+# %% ../nbs/01_schema.ipynb #393a81bb
+def _norm(obj, schema) -> dict:
+    "A structured reply as a plain dict, with the `None` a list field defaults to turned back into `[]`."
+    d = asdict(obj) if is_dataclass(obj) and not isinstance(obj, type) else dict(obj or {})
+    lists = {f.name for f in fields(schema) if f.type is list or get_origin(f.type) is list}
+    return {k: ([] if v is None and k in lists else v) for k, v in d.items()}
+
+# %% ../nbs/01_schema.ipynb #d62421fb
+def schema_str(schema) -> str:
+    "A schema written out for a model to read: its docstring, then `name: type` per field."
+    def ty(t):
+        if isinstance(t, str): return t
+        # `list[dict].__name__` is just 'list'; the parameter is the half the model needs
+        return str(t).replace('typing.', '') if get_origin(t) else getattr(t, '__name__', str(t))
+    return ((schema.__doc__ or '').strip() + '\n\n{\n'
+            + ',\n'.join(f'  "{f.name}": {ty(f.type)}' for f in fields(schema)) + '\n}')
+
+# %% ../nbs/01_schema.ipynb #3dae58c8
+def _split_reasoning(text:str) -> str:
+    "The answer with any reasoning stripped: rishi's `split_think`, plus the *closing*-only tag an MLX prefill leaves."
+    from rishi.core import split_think
+    text, _ = split_think(text)
+    if '</think>' in text: text = text.partition('</think>')[2]
+    return text.strip()
+
+# %% ../nbs/01_schema.ipynb #bf7b950f
+def _json_reply(ch, prompt:str, schema, sp:str='') -> object:
+    'Ask for the shape as a JSON object in prose and rebuild it, with the model left unconstrained.'
+    ch.hist = []
+    from rishi.core import resp_text
+    txt = resp_text(ch(f'{sp}\n\n{prompt}\n\nReply with only a JSON object in a ```json fence, with '
+                       f'exactly these keys and types:\n\n{schema_str(schema)}'))
+    from rishi.core import extract_fence
+    d = json.loads(extract_fence(_split_reasoning(txt), 'json'))
+    nms = {f.name for f in fields(schema)}
+    return schema(**{k: v for k, v in d.items() if k in nms})
+
+# %% ../nbs/01_schema.ipynb #875af6df
+def structured(ch,                # a `rishi.Chat`, e.g. from `new_chat`
+               prompt:str,        # the document and the instruction
+               schema,            # the dataclass the reply must fill
+               sp:str='',         # system prompt for the extraction
+) -> dict:
+    'A structured reply as a dict, retried as a plain JSON reply when the constrained call fails.'
+    try: return _norm(ch.structured(prompt, schema, sp=sp), schema)
+    except Exception as e:
+        warnings.warn(f'{type(e).__name__} on a constrained call for {schema.__name__} '
+                      f'({str(e)[:150]}); retrying as a JSON reply.')
+        return _norm(_json_reply(ch, prompt, schema, sp), schema)