mdmodels 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mdmodels-0.3.1 → mdmodels-0.4.0}/.github/workflows/lint.yml +1 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/PKG-INFO +3 -3
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/integrations/cli.mdx +6 -1
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/config.py +27 -4
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/library.py +17 -3
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/relations.py +6 -1
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/config.py +19 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/vector.py +13 -3
- {mdmodels-0.3.1 → mdmodels-0.4.0}/pyproject.toml +6 -2
- mdmodels-0.4.0/tests/unit/test_embedding_config.py +173 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/uv.lock +65 -65
- {mdmodels-0.3.1 → mdmodels-0.4.0}/.github/scripts/start_ollama.sh +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/.github/workflows/publish.yml +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/.github/workflows/test-publish.yml +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/.github/workflows/test.yml +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/.gitignore +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/Dockerfile +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/README.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/.gitignore +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/README.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/astro.config.mjs +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/package-lock.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/package.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/public/favicon.svg +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/assets/houston.webp +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/assets/logo.svg +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/config.ts +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/converting-formats.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/creating-data.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/index.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/library.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/multiple-models.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/parsing.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/querying.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/serialization.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/basic/validation.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/databases/graphdb.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/databases/sql.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/databases/vector.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/installation.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/integrations/fastapi.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/integrations/fastmcp.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content/docs/integrations/graphql.mdx +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/content.config.ts +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/src/styles/styles.css +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/docs/tsconfig.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/basic/BasicExample.ipynb +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/basic/model.jl +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/basic/model.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/basic/model.rs +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/basic/model.xsd +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/basic/uml_diagram.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/graph/Neo4JExample.ipynb +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/graph/model.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/graph/startn4j.sh +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/sql/basic/SQLDatabaseExample.ipynb +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/sql/basic/model.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/examples/sql/basic/projects.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/adder_method.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/cli.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/datamodel.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/git_utils.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/graph/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/graph/basenode.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/graph/connector.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/graph/create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/graph/relation.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/graphql/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/graphql/create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/llm/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/llm/handler.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/auth.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/config.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/annotation.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/constants.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/embedding.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/middleware.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/naming.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/ownership.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/relationships_types.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/runtime.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/_assets.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/aggregate.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/annotate.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/assets/data_entry_form.html +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/assets/downloader.html +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/assets/plot_viewer.html +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/assets/tabular_upload.html +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/download.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/form.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/form_store.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/plot.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/relationships.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/schema.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/select.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/tabular.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/upsert.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/sql/tools/vector_search.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/mcp/types.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/meta.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/path.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/reference.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/rest/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/rest/config.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/rest/create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/rest/search.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/aggregation.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/base.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/childref.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/connector.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/filter.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/groupby.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/insert.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/linked_type.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/migrations.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/upsert.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/sql/utils.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/templates.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/units/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/units/annotation.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/units/converter.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/units/mappings.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/units/unit_definition.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/mdmodels/utils.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/run-tests.sh +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/expected_json.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/expected_units.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/expected_units_complex.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/expected_wrapped_xml.xml +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/expected_xml.xml +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model_database.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model_graph.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model_json_schema.json +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model_multiple_types.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model_recursion.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model_units.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/fixtures/model_wrapped_xml.md +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/conftest.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/test_conversion.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/test_db.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/test_from_json_schema.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/test_graph.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/test_llm.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/integration/test_parse.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/unit/__init__.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/unit/test_create.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/unit/test_datamodel.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/unit/test_git_utils.py +0 -0
- {mdmodels-0.3.1 → mdmodels-0.4.0}/tests/unit/test_unit_definition.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: mdmodels
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Python package for the MDModels Rust crate
|
|
5
5
|
Author-email: Jan Range <range.jan@web.de>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -11,7 +11,7 @@ Requires-Dist: dotenv>=0.9.9
|
|
|
11
11
|
Requires-Dist: dotted-dict<2,>=1.1.3
|
|
12
12
|
Requires-Dist: fastexcel>=0.19.0
|
|
13
13
|
Requires-Dist: httpx<0.29,>=0.28
|
|
14
|
-
Requires-Dist: mdmodels-core<0.
|
|
14
|
+
Requires-Dist: mdmodels-core<0.4,>=0.3.0
|
|
15
15
|
Requires-Dist: nest-asyncio<2,>=1.6.0
|
|
16
16
|
Requires-Dist: pandas>=2.2.3
|
|
17
17
|
Requires-Dist: polars>=1.39.3
|
|
@@ -290,7 +290,7 @@ Embedding settings can be added under `[sql.tables.<TableName>.embedding]`:
|
|
|
290
290
|
|
|
291
291
|
- `column` (`string`, required when embedding is used)
|
|
292
292
|
- `model` (`string`, required)
|
|
293
|
-
- `provider` (`openai | huggingface`, required)
|
|
293
|
+
- `provider` (`openai | huggingface | fastembed`, required)
|
|
294
294
|
- `dimension` (`int`, optional)
|
|
295
295
|
|
|
296
296
|
Provider-specific subsections:
|
|
@@ -303,6 +303,11 @@ Provider-specific subsections:
|
|
|
303
303
|
- `device` (`string`, optional)
|
|
304
304
|
- `batch_size` (`int`, default `32`)
|
|
305
305
|
- `normalize_embeddings` (`bool`, default `true`)
|
|
306
|
+
- `trust_remote_code` (`bool`, default `false`)
|
|
307
|
+
|
|
308
|
+
- `[sql.tables.<TableName>.embedding.fastembed]`
|
|
309
|
+
- `cache_dir` (`string`, optional)
|
|
310
|
+
- `batch_size` (`int`, optional)
|
|
306
311
|
|
|
307
312
|
## `rest` Section
|
|
308
313
|
|
|
@@ -24,6 +24,7 @@ SQLDatabaseType = Literal[
|
|
|
24
24
|
EmbeddingProvider = Literal[
|
|
25
25
|
"openai",
|
|
26
26
|
"huggingface",
|
|
27
|
+
"fastembed",
|
|
27
28
|
]
|
|
28
29
|
AuthMethod = Literal["oidc"]
|
|
29
30
|
|
|
@@ -64,6 +65,14 @@ class HuggingFaceEmbeddingConfig(BaseModel):
|
|
|
64
65
|
device: Optional[str] = None
|
|
65
66
|
batch_size: int = 32
|
|
66
67
|
normalize_embeddings: bool = True
|
|
68
|
+
trust_remote_code: bool = False
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class FastembedEmbeddingConfig(BaseModel):
|
|
72
|
+
"""Provider-specific FastEmbed embedding settings."""
|
|
73
|
+
|
|
74
|
+
cache_dir: Optional[str] = None
|
|
75
|
+
batch_size: Optional[int] = None
|
|
67
76
|
|
|
68
77
|
|
|
69
78
|
class SQLEmbeddingConfig(BaseModel):
|
|
@@ -75,16 +84,30 @@ class SQLEmbeddingConfig(BaseModel):
|
|
|
75
84
|
dimension: Optional[int] = None
|
|
76
85
|
openai: Optional[OpenAIEmbeddingConfig] = None
|
|
77
86
|
huggingface: Optional[HuggingFaceEmbeddingConfig] = None
|
|
87
|
+
fastembed: Optional[FastembedEmbeddingConfig] = None
|
|
78
88
|
|
|
79
89
|
@model_validator(mode="after")
|
|
80
90
|
def validate_provider_settings(self) -> "SQLEmbeddingConfig":
|
|
81
|
-
if self.provider == "openai" and
|
|
91
|
+
if self.provider == "openai" and (
|
|
92
|
+
self.huggingface is not None or self.fastembed is not None
|
|
93
|
+
):
|
|
94
|
+
raise ValueError(
|
|
95
|
+
"embedding.huggingface and embedding.fastembed are not allowed "
|
|
96
|
+
"when embedding.provider='openai'"
|
|
97
|
+
)
|
|
98
|
+
if self.provider == "huggingface" and (
|
|
99
|
+
self.openai is not None or self.fastembed is not None
|
|
100
|
+
):
|
|
82
101
|
raise ValueError(
|
|
83
|
-
"embedding.
|
|
102
|
+
"embedding.openai and embedding.fastembed are not allowed "
|
|
103
|
+
"when embedding.provider='huggingface'"
|
|
84
104
|
)
|
|
85
|
-
if self.provider == "
|
|
105
|
+
if self.provider == "fastembed" and (
|
|
106
|
+
self.openai is not None or self.huggingface is not None
|
|
107
|
+
):
|
|
86
108
|
raise ValueError(
|
|
87
|
-
"embedding.openai
|
|
109
|
+
"embedding.openai and embedding.huggingface are not allowed "
|
|
110
|
+
"when embedding.provider='fastembed'"
|
|
88
111
|
)
|
|
89
112
|
return self
|
|
90
113
|
|
|
@@ -328,11 +328,17 @@ class Library(DottedDict, Generic[T]):
|
|
|
328
328
|
Returns:
|
|
329
329
|
Library[SQLModel]: A library containing SQLModel classes
|
|
330
330
|
"""
|
|
331
|
-
|
|
332
|
-
|
|
331
|
+
|
|
332
|
+
try:
|
|
333
|
+
from sqlmodel import SQLModel
|
|
334
|
+
except ImportError:
|
|
335
|
+
raise ImportError(
|
|
336
|
+
"SQLModel is not installed. Please install it with `pip install mdmodels[postgres]`."
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
from .sql import generate_sqlmodel
|
|
333
340
|
from mdmodels import DataModel
|
|
334
341
|
|
|
335
|
-
from .sql import generate_sqlmodel
|
|
336
342
|
|
|
337
343
|
# assert self.type == "DataModel", "Library must contain DataModel instances"
|
|
338
344
|
|
|
@@ -352,6 +358,14 @@ class Library(DottedDict, Generic[T]):
|
|
|
352
358
|
Returns:
|
|
353
359
|
Library[BaseNode]: A library containing BaseNode classes
|
|
354
360
|
"""
|
|
361
|
+
|
|
362
|
+
try:
|
|
363
|
+
import neomodel # noqa: F401 (availability check)
|
|
364
|
+
except ImportError:
|
|
365
|
+
raise ImportError(
|
|
366
|
+
"NeoModel is not installed. Please install it with `pip install mdmodels[neo4j]`."
|
|
367
|
+
)
|
|
368
|
+
|
|
355
369
|
from .datamodel import DataModel
|
|
356
370
|
from .graph import generate_neomodel
|
|
357
371
|
|
|
@@ -4,7 +4,6 @@ from collections import deque
|
|
|
4
4
|
from functools import lru_cache
|
|
5
5
|
from typing import Any, Iterable, Literal, Type, TypedDict, get_origin
|
|
6
6
|
|
|
7
|
-
from sqlalchemy.orm import RelationshipProperty
|
|
8
7
|
from pydantic import BaseModel
|
|
9
8
|
|
|
10
9
|
from mdmodels.utils import extract_dtype
|
|
@@ -26,6 +25,12 @@ class JoinEdge(TypedDict):
|
|
|
26
25
|
|
|
27
26
|
|
|
28
27
|
def _is_relationship_attr(attr: Any) -> bool:
|
|
28
|
+
# SQLAlchemy is only installed with the SQL extras; without it there are no relationships.
|
|
29
|
+
try:
|
|
30
|
+
from sqlalchemy.orm import RelationshipProperty
|
|
31
|
+
except ImportError:
|
|
32
|
+
return False
|
|
33
|
+
|
|
29
34
|
prop = getattr(attr, "property", None)
|
|
30
35
|
return isinstance(prop, RelationshipProperty)
|
|
31
36
|
|
|
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Dict, List, Literal, Optional
|
|
|
5
5
|
from pydantic import BaseModel, Field, model_validator
|
|
6
6
|
|
|
7
7
|
from mdmodels.sql.vector import (
|
|
8
|
+
FastembedTextEmbedding,
|
|
8
9
|
OpenAITextEmbedding,
|
|
9
10
|
SentenceTransformerEmbedding,
|
|
10
11
|
TextEmbedding,
|
|
@@ -256,6 +257,24 @@ class TableConfig(BaseModel):
|
|
|
256
257
|
normalize_embeddings=(
|
|
257
258
|
huggingface_cfg.normalize_embeddings if huggingface_cfg else True
|
|
258
259
|
),
|
|
260
|
+
trust_remote_code=(
|
|
261
|
+
huggingface_cfg.trust_remote_code if huggingface_cfg else False
|
|
262
|
+
),
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
if embedding.dimension is not None and embedding.dimension != model.dimension:
|
|
266
|
+
raise ValueError(
|
|
267
|
+
f"Configured embedding.dimension={embedding.dimension} does not "
|
|
268
|
+
f"match model dimension={model.dimension} for '{embedding.model}'"
|
|
269
|
+
)
|
|
270
|
+
return model
|
|
271
|
+
|
|
272
|
+
if embedding.provider == "fastembed":
|
|
273
|
+
fastembed_cfg = embedding.fastembed
|
|
274
|
+
model = FastembedTextEmbedding(
|
|
275
|
+
model_name=embedding.model,
|
|
276
|
+
cache_dir=fastembed_cfg.cache_dir if fastembed_cfg else None,
|
|
277
|
+
batch_size=fastembed_cfg.batch_size if fastembed_cfg else None,
|
|
259
278
|
)
|
|
260
279
|
|
|
261
280
|
if embedding.dimension is not None and embedding.dimension != model.dimension:
|
|
@@ -141,6 +141,7 @@ class SentenceTransformerEmbedding(TextEmbedding):
|
|
|
141
141
|
device: The device to run the model on (CPU, CUDA, etc.).
|
|
142
142
|
batch_size: The batch size for processing multiple texts.
|
|
143
143
|
normalize_embeddings: Whether to normalize the embedding vectors.
|
|
144
|
+
trust_remote_code: Whether to allow execution of remote model code.
|
|
144
145
|
model: The loaded SentenceTransformer model instance.
|
|
145
146
|
_dimension_cache: Cached dimension value for the model.
|
|
146
147
|
"""
|
|
@@ -153,6 +154,7 @@ class SentenceTransformerEmbedding(TextEmbedding):
|
|
|
153
154
|
device: str | None = None
|
|
154
155
|
batch_size: int = 32
|
|
155
156
|
normalize_embeddings: bool = True
|
|
157
|
+
trust_remote_code: bool = False
|
|
156
158
|
model: Any = None
|
|
157
159
|
_dimension_cache: int | None = None
|
|
158
160
|
|
|
@@ -163,6 +165,7 @@ class SentenceTransformerEmbedding(TextEmbedding):
|
|
|
163
165
|
device: str | None = None,
|
|
164
166
|
batch_size: int = 32,
|
|
165
167
|
normalize_embeddings: bool = True,
|
|
168
|
+
trust_remote_code: bool = False,
|
|
166
169
|
**kwargs: Any,
|
|
167
170
|
) -> None:
|
|
168
171
|
"""Initialize the SentenceTransformer embedding model.
|
|
@@ -172,6 +175,8 @@ class SentenceTransformerEmbedding(TextEmbedding):
|
|
|
172
175
|
device: The device to run the model on.
|
|
173
176
|
batch_size: The batch size for processing.
|
|
174
177
|
normalize_embeddings: Whether to normalize embeddings.
|
|
178
|
+
trust_remote_code: Whether to allow remote code execution for model
|
|
179
|
+
loading when required by the selected model.
|
|
175
180
|
**kwargs: Additional keyword arguments.
|
|
176
181
|
|
|
177
182
|
Raises:
|
|
@@ -188,7 +193,11 @@ class SentenceTransformerEmbedding(TextEmbedding):
|
|
|
188
193
|
"mdmodels[vector] extra."
|
|
189
194
|
) from exc
|
|
190
195
|
|
|
191
|
-
model = SentenceTransformer(
|
|
196
|
+
model = SentenceTransformer(
|
|
197
|
+
model_name,
|
|
198
|
+
device=device,
|
|
199
|
+
trust_remote_code=trust_remote_code,
|
|
200
|
+
)
|
|
192
201
|
dimension = model.get_sentence_embedding_dimension()
|
|
193
202
|
if dimension is None:
|
|
194
203
|
raise ValueError(f"Model {model_name} does not have a dimension")
|
|
@@ -201,10 +210,11 @@ class SentenceTransformerEmbedding(TextEmbedding):
|
|
|
201
210
|
device=device, # type: ignore[arg-type]
|
|
202
211
|
batch_size=batch_size, # type: ignore[arg-type]
|
|
203
212
|
normalize_embeddings=normalize_embeddings, # type: ignore[arg-type]
|
|
213
|
+
trust_remote_code=trust_remote_code, # type: ignore[arg-type]
|
|
204
214
|
model=model, # type: ignore[arg-type]
|
|
205
|
-
_dimension_cache=dimension, # type: ignore[arg-type]
|
|
206
215
|
**kwargs,
|
|
207
216
|
)
|
|
217
|
+
self._dimension_cache = dimension
|
|
208
218
|
|
|
209
219
|
def embed(self, text: str) -> EmbeddingVector:
|
|
210
220
|
"""Embed a single text string using SentenceTransformer.
|
|
@@ -328,13 +338,13 @@ class FastembedTextEmbedding(TextEmbedding):
|
|
|
328
338
|
cache_dir=cache_dir, # type: ignore[arg-type]
|
|
329
339
|
batch_size=batch_size, # type: ignore[arg-type]
|
|
330
340
|
model=model, # type: ignore[arg-type]
|
|
331
|
-
_dimension_cache=dimension, # type: ignore[arg-type]
|
|
332
341
|
**{
|
|
333
342
|
k: v
|
|
334
343
|
for k, v in kwargs.items()
|
|
335
344
|
if k not in ["model_name", "cache_dir", "batch_size"]
|
|
336
345
|
},
|
|
337
346
|
)
|
|
347
|
+
self._dimension_cache = dimension
|
|
338
348
|
|
|
339
349
|
def _ensure_dimension(self) -> None:
|
|
340
350
|
"""Infer and cache the embedding dimension lazily.
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "mdmodels"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "Python package for the MDModels Rust crate"
|
|
5
5
|
authors = [{ name = "Jan Range", email = "range.jan@web.de" }]
|
|
6
6
|
requires-python = ">=3.11"
|
|
7
7
|
readme = "README.md"
|
|
8
8
|
license = "MIT"
|
|
9
9
|
dependencies = [
|
|
10
|
-
"mdmodels-core>=0.
|
|
10
|
+
"mdmodels-core>=0.3.0,<0.4",
|
|
11
11
|
"pydantic-xml>=2.12.1,<3",
|
|
12
12
|
"bigtree>=0.21.1,<0.22",
|
|
13
13
|
"validators>=0.34.0,<0.35",
|
|
@@ -77,3 +77,7 @@ url = "https://test.pypi.org/simple/"
|
|
|
77
77
|
|
|
78
78
|
[tool.uv]
|
|
79
79
|
prerelease = "explicit"
|
|
80
|
+
|
|
81
|
+
[tool.ruff.lint]
|
|
82
|
+
# Ruff 0.16 widened its default rule set; keep the rules this code is linted against.
|
|
83
|
+
select = ["E4", "E7", "E9", "F"]
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import types
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from mdmodels.config import AppConfig, OpenAIEmbeddingConfig, SQLEmbeddingConfig
|
|
7
|
+
from mdmodels.sql.config import TableConfig
|
|
8
|
+
from mdmodels.sql.vector import SentenceTransformerEmbedding, TextEmbedding
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DummyTextEmbedding(TextEmbedding):
|
|
12
|
+
model_name: str
|
|
13
|
+
cache_dir: str | None = None
|
|
14
|
+
batch_size: int | None = None
|
|
15
|
+
trust_remote_code: bool = False
|
|
16
|
+
|
|
17
|
+
def embed(self, text: str):
|
|
18
|
+
return [0.0] * self.dimension
|
|
19
|
+
|
|
20
|
+
def embed_batch(self, texts):
|
|
21
|
+
return [[0.0] * self.dimension for _ in texts]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_sentence_transformer_embedding_accepts_dimension_cache_and_custom_code(
|
|
25
|
+
monkeypatch,
|
|
26
|
+
):
|
|
27
|
+
class FakeSentenceTransformer:
|
|
28
|
+
init_kwargs = {}
|
|
29
|
+
|
|
30
|
+
def __init__(self, model_name, device=None, trust_remote_code=False):
|
|
31
|
+
FakeSentenceTransformer.init_kwargs = {
|
|
32
|
+
"model_name": model_name,
|
|
33
|
+
"device": device,
|
|
34
|
+
"trust_remote_code": trust_remote_code,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
def get_sentence_embedding_dimension(self):
|
|
38
|
+
return 768
|
|
39
|
+
|
|
40
|
+
monkeypatch.setitem(
|
|
41
|
+
sys.modules,
|
|
42
|
+
"sentence_transformers",
|
|
43
|
+
types.SimpleNamespace(SentenceTransformer=FakeSentenceTransformer),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
embedding = SentenceTransformerEmbedding(
|
|
47
|
+
model_name="sentence-transformers/all-MiniLM-L6-v2",
|
|
48
|
+
trust_remote_code=True,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
assert embedding.dimension == 768
|
|
52
|
+
assert FakeSentenceTransformer.init_kwargs["trust_remote_code"] is True
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_sql_embedding_config_rejects_mixed_fastembed_provider_sections():
|
|
56
|
+
with pytest.raises(
|
|
57
|
+
ValueError,
|
|
58
|
+
match="embedding\\.openai and embedding\\.huggingface are not allowed",
|
|
59
|
+
):
|
|
60
|
+
SQLEmbeddingConfig(
|
|
61
|
+
column="description",
|
|
62
|
+
model="BAAI/bge-small-en-v1.5",
|
|
63
|
+
provider="fastembed",
|
|
64
|
+
openai=OpenAIEmbeddingConfig(),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_table_config_builds_fastembed_model(monkeypatch):
|
|
69
|
+
captured = {}
|
|
70
|
+
|
|
71
|
+
def fake_fastembed(model_name, cache_dir=None, batch_size=None):
|
|
72
|
+
captured.update(
|
|
73
|
+
{
|
|
74
|
+
"model_name": model_name,
|
|
75
|
+
"cache_dir": cache_dir,
|
|
76
|
+
"batch_size": batch_size,
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
return DummyTextEmbedding(
|
|
80
|
+
dimension=768,
|
|
81
|
+
model_name=model_name,
|
|
82
|
+
cache_dir=cache_dir,
|
|
83
|
+
batch_size=batch_size,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
monkeypatch.setattr("mdmodels.sql.config.FastembedTextEmbedding", fake_fastembed)
|
|
87
|
+
|
|
88
|
+
config = AppConfig.model_validate(
|
|
89
|
+
{
|
|
90
|
+
"model": {"path": "specifications/model.md"},
|
|
91
|
+
"sql": {
|
|
92
|
+
"type": "pgvector",
|
|
93
|
+
"tables": {
|
|
94
|
+
"StrendaBiocatalysis": {
|
|
95
|
+
"embedding": {
|
|
96
|
+
"column": "description",
|
|
97
|
+
"provider": "fastembed",
|
|
98
|
+
"model": "jinaai/jina-embeddings-v2-base-code",
|
|
99
|
+
"dimension": 768,
|
|
100
|
+
"fastembed": {
|
|
101
|
+
"cache_dir": "/tmp/emb-cache",
|
|
102
|
+
"batch_size": 16,
|
|
103
|
+
},
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
}
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
table_config = TableConfig.from_config(config, "StrendaBiocatalysis")
|
|
112
|
+
assert isinstance(table_config.embed_model, DummyTextEmbedding)
|
|
113
|
+
assert captured == {
|
|
114
|
+
"model_name": "jinaai/jina-embeddings-v2-base-code",
|
|
115
|
+
"cache_dir": "/tmp/emb-cache",
|
|
116
|
+
"batch_size": 16,
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_table_config_passes_trust_remote_code_to_huggingface(monkeypatch):
|
|
121
|
+
captured = {}
|
|
122
|
+
|
|
123
|
+
def fake_sentence_transformer(
|
|
124
|
+
model_name,
|
|
125
|
+
device=None,
|
|
126
|
+
batch_size=32,
|
|
127
|
+
normalize_embeddings=True,
|
|
128
|
+
trust_remote_code=False,
|
|
129
|
+
):
|
|
130
|
+
captured.update(
|
|
131
|
+
{
|
|
132
|
+
"model_name": model_name,
|
|
133
|
+
"device": device,
|
|
134
|
+
"batch_size": batch_size,
|
|
135
|
+
"normalize_embeddings": normalize_embeddings,
|
|
136
|
+
"trust_remote_code": trust_remote_code,
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
return DummyTextEmbedding(
|
|
140
|
+
dimension=768,
|
|
141
|
+
model_name=model_name,
|
|
142
|
+
trust_remote_code=trust_remote_code,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
monkeypatch.setattr(
|
|
146
|
+
"mdmodels.sql.config.SentenceTransformerEmbedding", fake_sentence_transformer
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
config = AppConfig.model_validate(
|
|
150
|
+
{
|
|
151
|
+
"model": {"path": "specifications/model.md"},
|
|
152
|
+
"sql": {
|
|
153
|
+
"type": "pgvector",
|
|
154
|
+
"tables": {
|
|
155
|
+
"StrendaBiocatalysis": {
|
|
156
|
+
"embedding": {
|
|
157
|
+
"column": "description",
|
|
158
|
+
"provider": "huggingface",
|
|
159
|
+
"model": "jinaai/jina-embeddings-v2-base-code",
|
|
160
|
+
"dimension": 768,
|
|
161
|
+
"huggingface": {
|
|
162
|
+
"trust_remote_code": True,
|
|
163
|
+
},
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
},
|
|
168
|
+
}
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
table_config = TableConfig.from_config(config, "StrendaBiocatalysis")
|
|
172
|
+
assert isinstance(table_config.embed_model, DummyTextEmbedding)
|
|
173
|
+
assert captured["trust_remote_code"] is True
|