contextgem 0.26.0__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextgem-1.0.0/.gitignore +84 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/LICENSE +3 -3
- {contextgem-0.26.0 → contextgem-1.0.0}/NOTICE +15 -17
- contextgem-1.0.0/PKG-INFO +80 -0
- contextgem-1.0.0/PYPI.md +24 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/__init__.py +24 -31
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/__init__.py +47 -55
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/__init__.py +16 -30
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/abstract.py +37 -25
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/aspects.py +87 -40
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/attrs.py +49 -59
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/concepts.py +139 -106
- contextgem-1.0.0/contextgem/internal/base/data_models.py +51 -0
- contextgem-1.0.0/contextgem/internal/base/decision_models.py +2211 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/documents.py +279 -95
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/examples.py +2 -17
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/images.py +2 -17
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/instances.py +5 -21
- contextgem-1.0.0/contextgem/internal/base/items.py +102 -0
- contextgem-1.0.0/contextgem/internal/base/llms.py +2451 -0
- contextgem-1.0.0/contextgem/internal/base/mixins.py +363 -0
- contextgem-0.26.0/contextgem/internal/base/llms.py → contextgem-1.0.0/contextgem/internal/base/models.py +2686 -2693
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/paras_and_sents.py +14 -40
- contextgem-1.0.0/contextgem/internal/base/pipelines.py +39 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/serialization.py +222 -116
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/base/utils.py +9 -22
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/data_models.py +144 -76
- contextgem-1.0.0/contextgem/internal/decisions.py +1004 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/decorators.py +10 -22
- contextgem-1.0.0/contextgem/internal/exceptions.py +239 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/items.py +62 -30
- contextgem-1.0.0/contextgem/internal/llm_output_structs/__init__.py +22 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/llm_output_structs/aspect_structs.py +7 -22
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/llm_output_structs/concept_structs.py +5 -20
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/llm_output_structs/utils.py +2 -17
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/loggers.py +13 -22
- contextgem-1.0.0/contextgem/internal/prompts/decision_question_aspect.j2 +14 -0
- contextgem-1.0.0/contextgem/internal/prompts/decision_question_concept.j2 +21 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/prompts/extract_aspect_items.j2 +7 -12
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/prompts/extract_concept_items.j2 +7 -15
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/registry.py +19 -20
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/suppressions.py +33 -27
- contextgem-1.0.0/contextgem/internal/tools/__init__.py +19 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/tools/docstring_parser.py +2 -17
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/tools/schema_generator.py +232 -42
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/typings/__init__.py +13 -26
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/typings/strings_to_types.py +2 -17
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/typings/typed_class_utils.py +3 -18
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/typings/types.py +39 -39
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/typings/types_normalization.py +2 -17
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/typings/types_to_strings.py +6 -21
- contextgem-1.0.0/contextgem/internal/typings/user_type_hints_validation.py +270 -0
- contextgem-1.0.0/contextgem/internal/typings/validators.py +111 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/utils.py +519 -120
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/__init__.py +26 -28
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/aspects.py +17 -24
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/concepts.py +158 -112
- contextgem-1.0.0/contextgem/public/data_models.py +41 -0
- contextgem-1.0.0/contextgem/public/decision_models.py +144 -0
- contextgem-1.0.0/contextgem/public/decorators.py +79 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/documents.py +7 -21
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/examples.py +4 -23
- contextgem-1.0.0/contextgem/public/exceptions.py +32 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/images.py +6 -23
- contextgem-1.0.0/contextgem/public/llms.py +225 -0
- contextgem-1.0.0/contextgem/public/models.py +53 -0
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/paragraphs.py +13 -21
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/pipelines.py +4 -43
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/sentences.py +11 -20
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/public/utils.py +2 -19
- {contextgem-0.26.0 → contextgem-1.0.0}/pyproject.toml +49 -41
- contextgem-0.26.0/.gitignore +0 -29
- contextgem-0.26.0/LICENSES.md +0 -115
- contextgem-0.26.0/PKG-INFO +0 -426
- contextgem-0.26.0/README.md +0 -366
- contextgem-0.26.0/contextgem/internal/base/data_models.py +0 -102
- contextgem-0.26.0/contextgem/internal/base/items.py +0 -55
- contextgem-0.26.0/contextgem/internal/base/md_text.py +0 -47
- contextgem-0.26.0/contextgem/internal/base/mixins.py +0 -85
- contextgem-0.26.0/contextgem/internal/base/pipelines.py +0 -75
- contextgem-0.26.0/contextgem/internal/converters/__init__.py +0 -25
- contextgem-0.26.0/contextgem/internal/converters/docx/__init__.py +0 -26
- contextgem-0.26.0/contextgem/internal/converters/docx/base.py +0 -2391
- contextgem-0.26.0/contextgem/internal/converters/docx/namespaces.py +0 -40
- contextgem-0.26.0/contextgem/internal/converters/docx/package.py +0 -293
- contextgem-0.26.0/contextgem/internal/converters/docx/utils.py +0 -129
- contextgem-0.26.0/contextgem/internal/exceptions.py +0 -170
- contextgem-0.26.0/contextgem/internal/llm_output_structs/__init__.py +0 -37
- contextgem-0.26.0/contextgem/internal/tools/__init__.py +0 -34
- contextgem-0.26.0/contextgem/internal/typings/user_type_hints_validation.py +0 -177
- contextgem-0.26.0/contextgem/internal/typings/validators.py +0 -131
- contextgem-0.26.0/contextgem/public/converters/__init__.py +0 -22
- contextgem-0.26.0/contextgem/public/converters/docx.py +0 -340
- contextgem-0.26.0/contextgem/public/data_models.py +0 -73
- contextgem-0.26.0/contextgem/public/decorators.py +0 -71
- contextgem-0.26.0/contextgem/public/llms.py +0 -195
- {contextgem-0.26.0 → contextgem-1.0.0}/contextgem/internal/system/default_system_message.j2 +0 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# uv.lock, package-lock.json and .python-version are committed. A leading
|
|
2
|
+
# slash anchors a path to the project's root.
|
|
3
|
+
|
|
4
|
+
# Python: bytecode, compiled extensions, and what a build writes
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
*.so
|
|
8
|
+
*.egg-info/
|
|
9
|
+
/build/
|
|
10
|
+
/dist/
|
|
11
|
+
/wheelhouse/
|
|
12
|
+
|
|
13
|
+
# uv's virtual environment
|
|
14
|
+
.venv/
|
|
15
|
+
|
|
16
|
+
# Tests and coverage
|
|
17
|
+
.pytest_cache/
|
|
18
|
+
.hypothesis/
|
|
19
|
+
.coverage
|
|
20
|
+
.coverage.*
|
|
21
|
+
coverage.xml
|
|
22
|
+
coverage.json
|
|
23
|
+
htmlcov/
|
|
24
|
+
|
|
25
|
+
# Linters
|
|
26
|
+
.ruff_cache/
|
|
27
|
+
|
|
28
|
+
# Documentation site: Sphinx's output, locally and in CI
|
|
29
|
+
/docs/build/
|
|
30
|
+
/docs/_build/
|
|
31
|
+
|
|
32
|
+
# Development tools that npm installs (package.json)
|
|
33
|
+
node_modules/
|
|
34
|
+
|
|
35
|
+
# Secrets: environment files are committed only as their .example templates
|
|
36
|
+
.env
|
|
37
|
+
.env.*
|
|
38
|
+
!.env.example
|
|
39
|
+
!.env.*.example
|
|
40
|
+
*.pem
|
|
41
|
+
*.key
|
|
42
|
+
credentials.json
|
|
43
|
+
token.json
|
|
44
|
+
|
|
45
|
+
# Agents. Claude Code's settings.json, hooks, skills, commands and rules are
|
|
46
|
+
# shared; its local settings, worktrees and plans belong to one machine.
|
|
47
|
+
.claude/settings.local.json
|
|
48
|
+
.claude/worktrees/
|
|
49
|
+
.claude/plans/
|
|
50
|
+
.cursor/plans/
|
|
51
|
+
# The licenseal review skill, installed from the locked licenseal version
|
|
52
|
+
.claude/skills/licenseal-review/
|
|
53
|
+
.agents/skills/licenseal-review/
|
|
54
|
+
# Temporary files that Claude Code can leave in the project
|
|
55
|
+
tmpclaude*
|
|
56
|
+
# What the agents' browser (Playwright MCP) saves
|
|
57
|
+
/.playwright-mcp/
|
|
58
|
+
|
|
59
|
+
# Editors. The extensions the project recommends are committed.
|
|
60
|
+
.idea/
|
|
61
|
+
.vscode/*
|
|
62
|
+
!.vscode/extensions.json
|
|
63
|
+
*.swp
|
|
64
|
+
*.swo
|
|
65
|
+
|
|
66
|
+
# Operating systems
|
|
67
|
+
.DS_Store
|
|
68
|
+
Thumbs.db
|
|
69
|
+
Desktop.ini
|
|
70
|
+
# Office's lock file beside an open document
|
|
71
|
+
~$*
|
|
72
|
+
# A file named nul, left when a Windows shell takes a redirect to /dev/null
|
|
73
|
+
# for a file name
|
|
74
|
+
nul
|
|
75
|
+
NUL
|
|
76
|
+
|
|
77
|
+
# ContextGem
|
|
78
|
+
# The serialized instances that the tests write and then remove
|
|
79
|
+
tests/instance_*.json
|
|
80
|
+
.cz.msg
|
|
81
|
+
*.tmp
|
|
82
|
+
/coverage_annotate/
|
|
83
|
+
# A local scratch file, which ruff, ty, deptry and interrogate leave out too
|
|
84
|
+
/code_sample.py
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
ContextGem -
|
|
2
|
-
|
|
1
|
+
ContextGem - Extract structured data and insights from documents using LLMs and decision models
|
|
2
|
+
================================================================================================
|
|
3
3
|
|
|
4
|
-
Copyright
|
|
5
|
-
All rights reserved
|
|
4
|
+
Copyright 2025 Shcherbak AI AS
|
|
6
5
|
Developed by Sergii Shcherbak
|
|
7
6
|
|
|
8
7
|
This software is licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -10,7 +9,7 @@ you may not use this file except in compliance with the License.
|
|
|
10
9
|
You may obtain a copy of the License at
|
|
11
10
|
|
|
12
11
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
13
|
-
|
|
12
|
+
|
|
14
13
|
Unless required by applicable law or agreed to in writing, software
|
|
15
14
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
16
15
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
@@ -31,15 +30,12 @@ Core Dependencies:
|
|
|
31
30
|
- genai-prices: LLM pricing data and utilities (by Pydantic) to automatically estimate costs
|
|
32
31
|
- Jinja2: Templating engine
|
|
33
32
|
- litellm: LLM interface library (this software uses only MIT-licensed portions of LiteLLM and does not utilize any components from the enterprise/ directory)
|
|
34
|
-
-
|
|
35
|
-
- onnxruntime: ONNX model inference runtime (pinned for Python 3.10/3.14 wheel compatibility; used transitively via wtpsplit-lite)
|
|
33
|
+
- onnxruntime: ONNX model inference runtime (constrained for Python 3.14 wheel compatibility; used transitively via wtpsplit-lite)
|
|
36
34
|
- openai: OpenAI Python client (pinned for stable compatibility with litellm)
|
|
37
35
|
- pillow: Image processing library for local model image handling
|
|
38
36
|
- pydantic: Data validation
|
|
39
37
|
- python-ulid: ULID generation
|
|
40
38
|
- tenacity: Retry library for Python
|
|
41
|
-
- tethered: Runtime network egress control; used to constrain SaT model downloads (HuggingFace) and genai-prices refresh (raw.githubusercontent.com) to known hosts, and to enforce egress policy in tests
|
|
42
|
-
- typing-extensions: Type annotation extensions
|
|
43
39
|
- wtpsplit-lite: Lightweight version of wtpsplit that only retains accelerated ONNX inference of SaT models with minimal dependencies
|
|
44
40
|
|
|
45
41
|
Development Dependencies:
|
|
@@ -50,11 +46,8 @@ Development Dependencies:
|
|
|
50
46
|
- fabric: SSH and command execution library used for development task automation
|
|
51
47
|
- hypothesis: Property-based testing library used for invariant tests of pure utility functions
|
|
52
48
|
- interrogate: Python docstring coverage checker
|
|
53
|
-
-
|
|
54
|
-
- nbformat: Notebook format utilities
|
|
55
|
-
- nest-asyncio: Patch for asyncio to allow nested event loops (used for VCR test compatibility)
|
|
49
|
+
- licenseal: License compatibility checker for the dependencies (used for the license audit in LICENSES.md)
|
|
56
50
|
- pre-commit: Pre-commit hooks
|
|
57
|
-
- pympler: Python memory analysis for object-level memory measurement
|
|
58
51
|
- ty: Static type checker for Python
|
|
59
52
|
- pytest: Testing framework
|
|
60
53
|
- pytest-cov: Coverage plugin for pytest
|
|
@@ -62,17 +55,22 @@ Development Dependencies:
|
|
|
62
55
|
- python-dotenv: Environment variable management
|
|
63
56
|
- ruff: Fast Python linter and formatter
|
|
64
57
|
- sphinx: Documentation generator
|
|
65
|
-
- sphinx-autobuild: Live-reloading docs builder for Sphinx
|
|
58
|
+
- sphinx-autobuild: Live-reloading docs builder for Sphinx
|
|
66
59
|
- sphinx-autodoc-typehints: Type annotation support for Sphinx
|
|
67
60
|
- sphinx-book-theme: Book-like theme for Sphinx
|
|
68
61
|
- sphinx-copybutton: Adds copy button to code blocks in Sphinx docs
|
|
69
62
|
- sphinx-design: Component library for Sphinx documentation
|
|
63
|
+
- sphinx-llm: Generates llms.txt and a Markdown copy of each page of the Sphinx documentation
|
|
70
64
|
- sphinx-sitemap: Generates XML sitemaps for Sphinx documentation
|
|
71
65
|
- sphinxext-opengraph: OpenGraph metadata support for Sphinx documentation
|
|
66
|
+
- tethered: Network egress control (used to enforce the egress policy of the tests)
|
|
72
67
|
- twine: PyPI package metadata validator (used by the package-check pre-commit hook)
|
|
73
|
-
- types-lxml: Type stubs for lxml
|
|
74
68
|
|
|
75
|
-
|
|
69
|
+
Development Tools (npm, package.json):
|
|
70
|
+
- @playwright/mcp: Browser automation server that gives AI coding assistants a browser (used to look at the README preview)
|
|
71
|
+
- github-markdown-css: GitHub's Markdown styles (used by the README preview)
|
|
72
|
+
|
|
73
|
+
Each of these components may have their own licenses. Users should refer to the
|
|
76
74
|
respective project repositories for detailed license information.
|
|
77
75
|
|
|
78
|
-
==============================================================================
|
|
76
|
+
==============================================================================
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: contextgem
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Extract structured data and insights from documents using LLMs and decision models
|
|
5
|
+
Project-URL: Homepage, https://github.com/shcherbak-ai/contextgem
|
|
6
|
+
Project-URL: Documentation, https://contextgem.shcherbak.ai/
|
|
7
|
+
Project-URL: Repository, https://github.com/shcherbak-ai/contextgem.git
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/shcherbak-ai/contextgem/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/shcherbak-ai/contextgem/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Shcherbak AI, https://shcherbak.ai/
|
|
11
|
+
Author-email: Sergii Shcherbak <sergii@shcherbak.ai>
|
|
12
|
+
Maintainer-email: Shcherbak AI <sergii@shcherbak.ai>
|
|
13
|
+
License: Apache-2.0
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
License-File: NOTICE
|
|
16
|
+
Keywords: artificial-intelligence,aspect-extraction,automated-prompting,concept-extraction,content-extraction,context-aware,contextgem,contract-analysis,contract-automation,contract-intelligence,contract-management,contract-parsing,contract-review,data-extraction,decision-models,document,document-analysis,document-extraction,document-intelligence,document-parsing,document-pipeline,document-processing,document-qa,document-understanding,entity-extraction,extraction-justifications,extraction-pipeline,fintech,generative-ai,information-extraction,insights-extraction,knowledge-extraction,large-language-models,legaltech,llm,llm-extraction,llm-framework,llm-library,llm-reasoning,low-code,machine-learning,model-routing,multilingual,multimodal,neural-segmentation,nlp,no-prompt-engineering,prompt-free,question-answering,reference-mapping,semantic-analysis,structured-data,structured-data-extraction,text-analysis,text-classification,text-processing,topic-extraction,unstructured-data,zero-shot
|
|
17
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
18
|
+
Classifier: Intended Audience :: Developers
|
|
19
|
+
Classifier: Intended Audience :: Education
|
|
20
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
21
|
+
Classifier: Intended Audience :: Information Technology
|
|
22
|
+
Classifier: Intended Audience :: Legal Industry
|
|
23
|
+
Classifier: Intended Audience :: Science/Research
|
|
24
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
25
|
+
Classifier: Operating System :: OS Independent
|
|
26
|
+
Classifier: Programming Language :: Python :: 3
|
|
27
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
28
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
29
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
30
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
31
|
+
Classifier: Topic :: Education
|
|
32
|
+
Classifier: Topic :: Office/Business
|
|
33
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
34
|
+
Classifier: Topic :: Scientific/Engineering :: Image Recognition
|
|
35
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
36
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
37
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
38
|
+
Classifier: Topic :: Text Processing :: General
|
|
39
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
40
|
+
Requires-Python: <3.15,>=3.11
|
|
41
|
+
Requires-Dist: aiolimiter<2.0.0,>=1.2.1
|
|
42
|
+
Requires-Dist: colorlog<7.0.0,>=6.9.0
|
|
43
|
+
Requires-Dist: docstring-parser<1.0.0,>=0.17.0
|
|
44
|
+
Requires-Dist: fastjsonschema<3.0.0,>=2.21.2
|
|
45
|
+
Requires-Dist: genai-prices==0.1.13
|
|
46
|
+
Requires-Dist: jinja2<4.0.0,>=3.1.5
|
|
47
|
+
Requires-Dist: litellm==1.105.1
|
|
48
|
+
Requires-Dist: onnxruntime>=1.24.0; python_version >= '3.14'
|
|
49
|
+
Requires-Dist: openai==2.54.0
|
|
50
|
+
Requires-Dist: pillow<13.0.0,>=11.3.0
|
|
51
|
+
Requires-Dist: pydantic<3.0.0,>=2.10.6
|
|
52
|
+
Requires-Dist: python-ulid<5.0.0,>=3.0.0
|
|
53
|
+
Requires-Dist: tenacity<10.0.0,>=9.1.2
|
|
54
|
+
Requires-Dist: wtpsplit-lite<0.3.0,>=0.2.0
|
|
55
|
+
Description-Content-Type: text/markdown
|
|
56
|
+
|
|
57
|
+
# ContextGem
|
|
58
|
+
|
|
59
|
+
Describe, don't prompt. ContextGem is a free, open-source framework that does the work around the models for you: you describe what to extract, and it handles the prompts, the data models, the source references and the justifications, with minimal code.
|
|
60
|
+
|
|
61
|
+
**Upgrading from 0.x?** ContextGem 1.0 renames and removes classes and parameters, changes some defaults and drops Python 3.10, so code and serialized data written for 0.x may need changes. Follow the [migration guide](https://contextgem.shcherbak.ai/migration_v1/).
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
uv add contextgem
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Or, with pip: `pip install -U contextgem`.
|
|
68
|
+
|
|
69
|
+
- Generates the prompts from your descriptions of what to extract: there are no prompts to write.
|
|
70
|
+
- Returns typed, validated values: text, numbers, dates, booleans, labels, ratings and JSON objects.
|
|
71
|
+
- Maps each extracted item back to the paragraphs or sentences of the document that it came from, and can give the reasoning behind it.
|
|
72
|
+
- Extracts in context: aspects, sub-aspects and concepts focus the model on the text that matters.
|
|
73
|
+
- Runs as one declarative pipeline, reusable and serializable, with model routing, fallbacks and cost tracking.
|
|
74
|
+
- Uses the right model for each question: cloud and local LLMs through LiteLLM where it takes reasoning, and decision models where a fast, low-cost answer is enough.
|
|
75
|
+
- Gives the probability of each possible answer of a decision model, so that you set the threshold.
|
|
76
|
+
- Is built for in-depth analysis of individual documents, not for cross-document querying or corpus-wide retrieval.
|
|
77
|
+
|
|
78
|
+
See the [documentation](https://contextgem.shcherbak.ai) for the [quickstart](https://contextgem.shcherbak.ai/quickstart/), [how it works](https://contextgem.shcherbak.ai/how_it_works/) and [decision models](https://contextgem.shcherbak.ai/decision_models/decision_models/), the [repository](https://github.com/shcherbak-ai/contextgem#readme) for the overview and the source, and the [changelog](https://github.com/shcherbak-ai/contextgem/blob/main/CHANGELOG.md) for the version history.
|
|
79
|
+
|
|
80
|
+
Licensed under [Apache-2.0](https://github.com/shcherbak-ai/contextgem/blob/main/LICENSE).
|
contextgem-1.0.0/PYPI.md
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# ContextGem
|
|
2
|
+
|
|
3
|
+
Describe, don't prompt. ContextGem is a free, open-source framework that does the work around the models for you: you describe what to extract, and it handles the prompts, the data models, the source references and the justifications, with minimal code.
|
|
4
|
+
|
|
5
|
+
**Upgrading from 0.x?** ContextGem 1.0 renames and removes classes and parameters, changes some defaults and drops Python 3.10, so code and serialized data written for 0.x may need changes. Follow the [migration guide](https://contextgem.shcherbak.ai/migration_v1/).
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
uv add contextgem
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Or, with pip: `pip install -U contextgem`.
|
|
12
|
+
|
|
13
|
+
- Generates the prompts from your descriptions of what to extract: there are no prompts to write.
|
|
14
|
+
- Returns typed, validated values: text, numbers, dates, booleans, labels, ratings and JSON objects.
|
|
15
|
+
- Maps each extracted item back to the paragraphs or sentences of the document that it came from, and can give the reasoning behind it.
|
|
16
|
+
- Extracts in context: aspects, sub-aspects and concepts focus the model on the text that matters.
|
|
17
|
+
- Runs as one declarative pipeline, reusable and serializable, with model routing, fallbacks and cost tracking.
|
|
18
|
+
- Uses the right model for each question: cloud and local LLMs through LiteLLM where it takes reasoning, and decision models where a fast, low-cost answer is enough.
|
|
19
|
+
- Gives the probability of each possible answer of a decision model, so that you set the threshold.
|
|
20
|
+
- Is built for in-depth analysis of individual documents, not for cross-document querying or corpus-wide retrieval.
|
|
21
|
+
|
|
22
|
+
See the [documentation](https://contextgem.shcherbak.ai) for the [quickstart](https://contextgem.shcherbak.ai/quickstart/), [how it works](https://contextgem.shcherbak.ai/how_it_works/) and [decision models](https://contextgem.shcherbak.ai/decision_models/decision_models/), the [repository](https://github.com/shcherbak-ai/contextgem#readme) for the overview and the source, and the [changelog](https://github.com/shcherbak-ai/contextgem/blob/main/CHANGELOG.md) for the version history.
|
|
23
|
+
|
|
24
|
+
Licensed under [Apache-2.0](https://github.com/shcherbak-ai/contextgem/blob/main/LICENSE).
|
|
@@ -1,49 +1,37 @@
|
|
|
1
|
-
#
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2025 Shcherbak AI AS. All rights reserved. Developed by Sergii Shcherbak.
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
1
|
+
# SPDX-FileCopyrightText: Shcherbak AI AS
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
18
3
|
|
|
19
4
|
"""
|
|
20
|
-
ContextGem -
|
|
5
|
+
ContextGem - Extract structured data and insights from documents using LLMs and decision models
|
|
21
6
|
"""
|
|
22
7
|
|
|
23
|
-
__version__ = "0.
|
|
8
|
+
__version__ = "1.0.0"
|
|
24
9
|
__author__ = "Shcherbak AI AS"
|
|
25
10
|
|
|
26
11
|
from contextgem.public import (
|
|
27
12
|
Aspect,
|
|
28
13
|
BooleanConcept,
|
|
29
14
|
ChatSession,
|
|
15
|
+
ContextGemError,
|
|
16
|
+
ContextGemWarning,
|
|
30
17
|
DateConcept,
|
|
31
18
|
Document,
|
|
19
|
+
DocumentDecisionModel,
|
|
32
20
|
DocumentLLM,
|
|
33
|
-
|
|
34
|
-
DocumentPipeline,
|
|
35
|
-
DocxConverter,
|
|
21
|
+
DocumentModelGroup,
|
|
36
22
|
ExtractionPipeline,
|
|
37
23
|
Image,
|
|
38
24
|
JsonObjectClassStruct,
|
|
39
25
|
JsonObjectConcept,
|
|
40
26
|
JsonObjectExample,
|
|
41
27
|
LabelConcept,
|
|
42
|
-
|
|
28
|
+
LLMToolLoopLimitError,
|
|
29
|
+
ModelAPIError,
|
|
30
|
+
ModelExtractionError,
|
|
31
|
+
ModelPricing,
|
|
43
32
|
NumericalConcept,
|
|
44
33
|
Paragraph,
|
|
45
34
|
RatingConcept,
|
|
46
|
-
RatingScale,
|
|
47
35
|
Sentence,
|
|
48
36
|
StringConcept,
|
|
49
37
|
StringExample,
|
|
@@ -54,7 +42,7 @@ from contextgem.public import (
|
|
|
54
42
|
)
|
|
55
43
|
|
|
56
44
|
|
|
57
|
-
__all__ = (
|
|
45
|
+
__all__ = ( # noqa: RUF022 - grouped by kind
|
|
58
46
|
# Aspects
|
|
59
47
|
"Aspect",
|
|
60
48
|
# Concepts
|
|
@@ -69,7 +57,6 @@ __all__ = (
|
|
|
69
57
|
"Document",
|
|
70
58
|
# Pipelines
|
|
71
59
|
"ExtractionPipeline",
|
|
72
|
-
"DocumentPipeline", # deprecated, will be removed in v1.0.0
|
|
73
60
|
# Paragraphs
|
|
74
61
|
"Paragraph",
|
|
75
62
|
# Sentences
|
|
@@ -82,17 +69,23 @@ __all__ = (
|
|
|
82
69
|
# LLMs
|
|
83
70
|
"ChatSession",
|
|
84
71
|
"DocumentLLM",
|
|
85
|
-
|
|
72
|
+
# Decision models
|
|
73
|
+
"DocumentDecisionModel",
|
|
74
|
+
# Model groups
|
|
75
|
+
"DocumentModelGroup",
|
|
86
76
|
# Data models
|
|
87
|
-
"
|
|
88
|
-
"RatingScale", # deprecated, will be removed in v1.0.0
|
|
77
|
+
"ModelPricing",
|
|
89
78
|
# Utils
|
|
90
79
|
"create_image",
|
|
91
80
|
"image_to_base64",
|
|
92
81
|
"reload_logger_settings",
|
|
93
82
|
"JsonObjectClassStruct",
|
|
94
|
-
# Converters
|
|
95
|
-
"DocxConverter",
|
|
96
83
|
# Decorators
|
|
97
84
|
"register_tool",
|
|
85
|
+
# Exceptions and warnings
|
|
86
|
+
"ContextGemError",
|
|
87
|
+
"ModelAPIError",
|
|
88
|
+
"ModelExtractionError",
|
|
89
|
+
"LLMToolLoopLimitError",
|
|
90
|
+
"ContextGemWarning",
|
|
98
91
|
)
|
|
@@ -1,20 +1,5 @@
|
|
|
1
|
-
#
|
|
2
|
-
#
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2025 Shcherbak AI AS. All rights reserved. Developed by Sergii Shcherbak.
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
1
|
+
# SPDX-FileCopyrightText: Shcherbak AI AS
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
18
3
|
|
|
19
4
|
from contextgem.internal.base import (
|
|
20
5
|
_COST_QUANT,
|
|
@@ -24,31 +9,29 @@ from contextgem.internal.base import (
|
|
|
24
9
|
_ChatSession,
|
|
25
10
|
_DateConcept,
|
|
26
11
|
_Document,
|
|
12
|
+
_DocumentDecisionModel,
|
|
27
13
|
_DocumentLLM,
|
|
28
|
-
|
|
29
|
-
_DocumentPipeline,
|
|
14
|
+
_DocumentModelGroup,
|
|
30
15
|
_ExtractionPipeline,
|
|
31
16
|
_Image,
|
|
32
17
|
_JsonObjectClassStruct,
|
|
33
18
|
_JsonObjectConcept,
|
|
34
19
|
_JsonObjectExample,
|
|
35
20
|
_LabelConcept,
|
|
36
|
-
|
|
21
|
+
_ModelPricing,
|
|
37
22
|
_NumericalConcept,
|
|
38
23
|
_Paragraph,
|
|
39
24
|
_RatingConcept,
|
|
40
|
-
_RatingScale,
|
|
41
25
|
_Sentence,
|
|
42
26
|
_StringConcept,
|
|
43
27
|
_StringExample,
|
|
44
28
|
)
|
|
45
|
-
from contextgem.internal.converters import _DocxConverterBase, _DocxPackage
|
|
46
29
|
from contextgem.internal.data_models import (
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
30
|
+
_ModelCall,
|
|
31
|
+
_ModelCost,
|
|
32
|
+
_ModelCostOutputContainer,
|
|
33
|
+
_ModelUsage,
|
|
34
|
+
_ModelUsageOutputContainer,
|
|
52
35
|
)
|
|
53
36
|
from contextgem.internal.decorators import (
|
|
54
37
|
_disable_direct_initialization,
|
|
@@ -57,10 +40,9 @@ from contextgem.internal.decorators import (
|
|
|
57
40
|
_timer_decorator,
|
|
58
41
|
)
|
|
59
42
|
from contextgem.internal.exceptions import (
|
|
60
|
-
DocxConverterError,
|
|
61
|
-
LLMAPIError,
|
|
62
|
-
LLMExtractionError,
|
|
63
43
|
LLMToolLoopLimitError,
|
|
44
|
+
ModelAPIError,
|
|
45
|
+
ModelExtractionError,
|
|
64
46
|
)
|
|
65
47
|
from contextgem.internal.items import (
|
|
66
48
|
_BooleanItem,
|
|
@@ -86,14 +68,13 @@ from contextgem.internal.typings import (
|
|
|
86
68
|
ExtractedInstanceType,
|
|
87
69
|
JustificationDepth,
|
|
88
70
|
LanguageRequirement,
|
|
89
|
-
|
|
90
|
-
|
|
71
|
+
Modality,
|
|
72
|
+
ModelRole,
|
|
91
73
|
NonEmptyStr,
|
|
92
74
|
ReasoningEffort,
|
|
93
75
|
ReferenceDepth,
|
|
94
76
|
SaTModelId,
|
|
95
77
|
StandardSaTModelId,
|
|
96
|
-
TextMode,
|
|
97
78
|
_deserialize_type_hint,
|
|
98
79
|
_dynamic_pydantic_model,
|
|
99
80
|
_format_dict_structure,
|
|
@@ -114,12 +95,21 @@ from contextgem.internal.utils import (
|
|
|
114
95
|
_clean_control_characters,
|
|
115
96
|
_clean_text_for_llm_prompt,
|
|
116
97
|
_contains_linebreaks,
|
|
98
|
+
_get_cache_token_counts,
|
|
99
|
+
_get_distinct_instances,
|
|
100
|
+
_get_llm_cache_token_counts,
|
|
101
|
+
_get_model_api_retry_delay,
|
|
102
|
+
_get_position_from_prompt_id,
|
|
103
|
+
_get_sentence_position_from_prompt_id,
|
|
117
104
|
_get_template,
|
|
118
105
|
_group_instances_by_fields,
|
|
119
106
|
_is_json_serializable,
|
|
107
|
+
_is_retryable_model_api_error,
|
|
120
108
|
_is_text_content_empty,
|
|
121
|
-
_llm_call_result_is_valid,
|
|
122
109
|
_load_sat_model,
|
|
110
|
+
_model_call_result_is_api_failure,
|
|
111
|
+
_model_call_result_is_output_truncated,
|
|
112
|
+
_model_call_result_is_valid,
|
|
123
113
|
_parse_llm_output_as_json,
|
|
124
114
|
_remove_thinking_content_from_llm_output,
|
|
125
115
|
_run_async_calls,
|
|
@@ -130,7 +120,7 @@ from contextgem.internal.utils import (
|
|
|
130
120
|
)
|
|
131
121
|
|
|
132
122
|
|
|
133
|
-
__all__ = (
|
|
123
|
+
__all__ = ( # noqa: RUF022 - grouped by kind
|
|
134
124
|
# Base
|
|
135
125
|
"_COST_QUANT",
|
|
136
126
|
"_LOCAL_MODEL_PROVIDERS",
|
|
@@ -139,20 +129,19 @@ __all__ = (
|
|
|
139
129
|
"_ChatSession",
|
|
140
130
|
"_DateConcept",
|
|
141
131
|
"_Document",
|
|
132
|
+
"_DocumentDecisionModel",
|
|
142
133
|
"_DocumentLLM",
|
|
143
|
-
"
|
|
144
|
-
"_DocumentPipeline",
|
|
134
|
+
"_DocumentModelGroup",
|
|
145
135
|
"_ExtractionPipeline",
|
|
146
136
|
"_Image",
|
|
147
137
|
"_JsonObjectClassStruct",
|
|
148
138
|
"_JsonObjectConcept",
|
|
149
139
|
"_JsonObjectExample",
|
|
150
140
|
"_LabelConcept",
|
|
151
|
-
"
|
|
141
|
+
"_ModelPricing",
|
|
152
142
|
"_NumericalConcept",
|
|
153
143
|
"_Paragraph",
|
|
154
144
|
"_RatingConcept",
|
|
155
|
-
"_RatingScale",
|
|
156
145
|
"_Sentence",
|
|
157
146
|
"_StringConcept",
|
|
158
147
|
"_StringExample",
|
|
@@ -162,8 +151,8 @@ __all__ = (
|
|
|
162
151
|
"_LabelConceptItemValueModel",
|
|
163
152
|
# Typings
|
|
164
153
|
"NonEmptyStr",
|
|
165
|
-
"
|
|
166
|
-
"
|
|
154
|
+
"ModelRole",
|
|
155
|
+
"Modality",
|
|
167
156
|
"AssignedInstancesAttrName",
|
|
168
157
|
"ExtractedInstanceType",
|
|
169
158
|
"DefaultPromptType",
|
|
@@ -175,7 +164,6 @@ __all__ = (
|
|
|
175
164
|
"AsyncCalsAndKwargs",
|
|
176
165
|
"DefaultDecimalField",
|
|
177
166
|
"ReasoningEffort",
|
|
178
|
-
"TextMode",
|
|
179
167
|
"_deserialize_type_hint",
|
|
180
168
|
"_is_json_serializable_type",
|
|
181
169
|
"_format_type",
|
|
@@ -190,11 +178,11 @@ __all__ = (
|
|
|
190
178
|
"_normalize_type_annotation",
|
|
191
179
|
"_validate_sequence_is_list",
|
|
192
180
|
# Data models
|
|
193
|
-
"
|
|
194
|
-
"
|
|
195
|
-
"
|
|
196
|
-
"
|
|
197
|
-
"
|
|
181
|
+
"_ModelCall",
|
|
182
|
+
"_ModelUsage",
|
|
183
|
+
"_ModelUsageOutputContainer",
|
|
184
|
+
"_ModelCost",
|
|
185
|
+
"_ModelCostOutputContainer",
|
|
198
186
|
# Decorators
|
|
199
187
|
"_post_init_method",
|
|
200
188
|
"_timer_decorator",
|
|
@@ -219,9 +207,18 @@ __all__ = (
|
|
|
219
207
|
"_async_multi_executor",
|
|
220
208
|
"_run_async_calls",
|
|
221
209
|
"_run_sync",
|
|
222
|
-
"
|
|
210
|
+
"_model_call_result_is_valid",
|
|
211
|
+
"_model_call_result_is_api_failure",
|
|
212
|
+
"_model_call_result_is_output_truncated",
|
|
213
|
+
"_is_retryable_model_api_error",
|
|
214
|
+
"_get_model_api_retry_delay",
|
|
215
|
+
"_get_cache_token_counts",
|
|
216
|
+
"_get_llm_cache_token_counts",
|
|
223
217
|
"_parse_llm_output_as_json",
|
|
218
|
+
"_get_position_from_prompt_id",
|
|
219
|
+
"_get_sentence_position_from_prompt_id",
|
|
224
220
|
"_validate_parsed_llm_output",
|
|
221
|
+
"_get_distinct_instances",
|
|
225
222
|
"_group_instances_by_fields",
|
|
226
223
|
"_is_json_serializable",
|
|
227
224
|
"_load_sat_model",
|
|
@@ -230,13 +227,8 @@ __all__ = (
|
|
|
230
227
|
"_clean_control_characters",
|
|
231
228
|
"_is_text_content_empty",
|
|
232
229
|
"_clean_text_for_llm_prompt",
|
|
233
|
-
# Converters
|
|
234
|
-
# DOCX
|
|
235
|
-
"_DocxConverterBase",
|
|
236
|
-
"_DocxPackage",
|
|
237
230
|
# Exceptions
|
|
238
|
-
"
|
|
239
|
-
"
|
|
231
|
+
"ModelExtractionError",
|
|
232
|
+
"ModelAPIError",
|
|
240
233
|
"LLMToolLoopLimitError",
|
|
241
|
-
"DocxConverterError",
|
|
242
234
|
)
|