inh-contracts 2.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inh_contracts-2.3.1/.gitignore +52 -0
- inh_contracts-2.3.1/PKG-INFO +60 -0
- inh_contracts-2.3.1/Readme.md +42 -0
- inh_contracts-2.3.1/pyproject.toml +96 -0
- inh_contracts-2.3.1/src/inh_contracts/__init__.py +100 -0
- inh_contracts-2.3.1/src/inh_contracts/conversation.py +53 -0
- inh_contracts-2.3.1/src/inh_contracts/defaults.py +70 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/__init__.py +67 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/batching.py +102 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/defaults.py +69 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/factory.py +75 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/identity.py +195 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/openai_provider.py +32 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/provider.py +156 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/retry.py +140 -0
- inh_contracts-2.3.1/src/inh_contracts/embedding/tei_provider.py +32 -0
- inh_contracts-2.3.1/src/inh_contracts/events.py +221 -0
- inh_contracts-2.3.1/src/inh_contracts/file_types.py +1238 -0
- inh_contracts-2.3.1/src/inh_contracts/naming.py +51 -0
- inh_contracts-2.3.1/tests/fixtures/openai_embeddings_response.json +17 -0
- inh_contracts-2.3.1/tests/fixtures/tei_embed_response.json +4 -0
- inh_contracts-2.3.1/tests/test_defaults.py +56 -0
- inh_contracts-2.3.1/tests/test_embedding_batching.py +121 -0
- inh_contracts-2.3.1/tests/test_embedding_factory.py +54 -0
- inh_contracts-2.3.1/tests/test_embedding_identity.py +162 -0
- inh_contracts-2.3.1/tests/test_embedding_retry.py +184 -0
- inh_contracts-2.3.1/tests/test_events.py +262 -0
- inh_contracts-2.3.1/tests/test_file_types.py +1325 -0
- inh_contracts-2.3.1/tests/test_naming.py +81 -0
- inh_contracts-2.3.1/tests/test_openai_provider.py +62 -0
- inh_contracts-2.3.1/tests/test_tei_provider.py +99 -0
- inh_contracts-2.3.1/uv.lock +920 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
.env
|
|
2
|
+
.env.*
|
|
3
|
+
!.env.example
|
|
4
|
+
|
|
5
|
+
node_modules/
|
|
6
|
+
.claude/
|
|
7
|
+
.grok/
|
|
8
|
+
.remember/
|
|
9
|
+
.venv/
|
|
10
|
+
venv/
|
|
11
|
+
__pycache__/
|
|
12
|
+
*.pyc
|
|
13
|
+
.pytest_cache/
|
|
14
|
+
.ruff_cache/
|
|
15
|
+
.mypy_cache/
|
|
16
|
+
.coverage
|
|
17
|
+
htmlcov/
|
|
18
|
+
dist/
|
|
19
|
+
build/
|
|
20
|
+
|
|
21
|
+
# Anchored to the repo root deliberately -- an earlier un-anchored `data/`
|
|
22
|
+
# here matched (and gitignored) infra/azure/modules/data/ too, silently
|
|
23
|
+
# masking that whole Terraform module from git (#47). The anchor also means
|
|
24
|
+
# the CLI's packaged release compose file (#280) needs no un-ignore rules,
|
|
25
|
+
# but they stay as a guard should the anchor ever be loosened again.
|
|
26
|
+
/data/
|
|
27
|
+
!services/inh-cli/src/inh_cli/data/
|
|
28
|
+
!services/inh-cli/src/inh_cli/data/docker-compose.release.yml
|
|
29
|
+
uploads/
|
|
30
|
+
playwright-report/
|
|
31
|
+
test-results/
|
|
32
|
+
eval-report.json
|
|
33
|
+
ingestion-benchmark-report.json
|
|
34
|
+
search-benchmark-report.json
|
|
35
|
+
.DS_Store
|
|
36
|
+
|
|
37
|
+
graphify-out/
|
|
38
|
+
|
|
39
|
+
# MkDocs build output
|
|
40
|
+
site/
|
|
41
|
+
.memory/
|
|
42
|
+
|
|
43
|
+
# Local agent tooling: skills vendored by the skills installer plus its
|
|
44
|
+
# lockfile. Machine-local and third-party, same category as .claude/ and
|
|
45
|
+
# .remember/ above (#347).
|
|
46
|
+
.agents/
|
|
47
|
+
skills-lock.json
|
|
48
|
+
|
|
49
|
+
# Terraform working dirs (any root/module) — state and providers never belong in git.
|
|
50
|
+
**/.terraform/
|
|
51
|
+
*.tfstate
|
|
52
|
+
*.tfstate.*
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: inh-contracts
|
|
3
|
+
Version: 2.3.1
|
|
4
|
+
Summary: Shared contracts (Weaviate naming, versioned event schemas, embedding provider abstraction) consumed by the ingestion and public-api services
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: httpx>=0.25.0
|
|
7
|
+
Requires-Dist: pydantic>=2
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
Requires-Dist: bandit[toml]==1.9.3; extra == 'dev'
|
|
10
|
+
Requires-Dist: black==24.10.0; extra == 'dev'
|
|
11
|
+
Requires-Dist: mypy==1.19.1; extra == 'dev'
|
|
12
|
+
Requires-Dist: pre-commit>=3.6.0; extra == 'dev'
|
|
13
|
+
Requires-Dist: pytest-asyncio==1.3.0; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest-cov==7.0.0; extra == 'dev'
|
|
15
|
+
Requires-Dist: pytest==9.0.2; extra == 'dev'
|
|
16
|
+
Requires-Dist: ruff==0.14.10; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# inh-contracts
|
|
20
|
+
|
|
21
|
+
Shared contracts package consumed by **both** the ingestion service
|
|
22
|
+
(`inh-ingestion-svc`) and the public API service (`inh-public-api-svc`).
|
|
23
|
+
|
|
24
|
+
This is the single source of truth for:
|
|
25
|
+
|
|
26
|
+
- **Weaviate naming** (`inh_contracts.naming`) — `get_workspace_collection_name`
|
|
27
|
+
and `get_user_tenant_name`. Both services derive collection/tenant names from
|
|
28
|
+
raw workspace/user ids and MUST agree byte-for-byte (#12).
|
|
29
|
+
- **Versioned event schemas** (`inh_contracts.events`) —
|
|
30
|
+
`DocumentUploadMessage` and `DocumentCompletionMessage`, the canonical
|
|
31
|
+
cross-service event contracts carrying `contract_version` (#17).
|
|
32
|
+
- **Embedding provider abstraction** (`inh_contracts.embedding`, #311) — the
|
|
33
|
+
`EmbeddingProvider` interface, the `TEIProvider` (default) and
|
|
34
|
+
`OpenAICompatibleProvider` wire adapters, `create_embedding_provider()`,
|
|
35
|
+
the shared retry/batching helpers, and the Weaviate collection
|
|
36
|
+
model-identity guard (`inh_contracts.embedding.identity`). Both services'
|
|
37
|
+
`embedder.py` are thin wrappers over this — see
|
|
38
|
+
`docs/reference/configuration.md`'s "Embedding provider &
|
|
39
|
+
model-identity guard" section for the operator-facing picture.
|
|
40
|
+
|
|
41
|
+
`CONTRACT_VERSION` pins the semantic version of these contracts.
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
Each service declares a uv path dependency in its `pyproject.toml`:
|
|
46
|
+
|
|
47
|
+
```toml
|
|
48
|
+
[project]
|
|
49
|
+
dependencies = ["inh-contracts", ...]
|
|
50
|
+
|
|
51
|
+
[tool.uv.sources]
|
|
52
|
+
inh-contracts = { path = "../inh-contracts" }
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
and imports from it:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from inh_contracts.naming import get_workspace_collection_name, get_user_tenant_name
|
|
59
|
+
from inh_contracts.events import DocumentUploadMessage, DocumentCompletionMessage
|
|
60
|
+
```
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# inh-contracts
|
|
2
|
+
|
|
3
|
+
Shared contracts package consumed by **both** the ingestion service
|
|
4
|
+
(`inh-ingestion-svc`) and the public API service (`inh-public-api-svc`).
|
|
5
|
+
|
|
6
|
+
This is the single source of truth for:
|
|
7
|
+
|
|
8
|
+
- **Weaviate naming** (`inh_contracts.naming`) — `get_workspace_collection_name`
|
|
9
|
+
and `get_user_tenant_name`. Both services derive collection/tenant names from
|
|
10
|
+
raw workspace/user ids and MUST agree byte-for-byte (#12).
|
|
11
|
+
- **Versioned event schemas** (`inh_contracts.events`) —
|
|
12
|
+
`DocumentUploadMessage` and `DocumentCompletionMessage`, the canonical
|
|
13
|
+
cross-service event contracts carrying `contract_version` (#17).
|
|
14
|
+
- **Embedding provider abstraction** (`inh_contracts.embedding`, #311) — the
|
|
15
|
+
`EmbeddingProvider` interface, the `TEIProvider` (default) and
|
|
16
|
+
`OpenAICompatibleProvider` wire adapters, `create_embedding_provider()`,
|
|
17
|
+
the shared retry/batching helpers, and the Weaviate collection
|
|
18
|
+
model-identity guard (`inh_contracts.embedding.identity`). Both services'
|
|
19
|
+
`embedder.py` are thin wrappers over this — see
|
|
20
|
+
`docs/reference/configuration.md`'s "Embedding provider &
|
|
21
|
+
model-identity guard" section for the operator-facing picture.
|
|
22
|
+
|
|
23
|
+
`CONTRACT_VERSION` pins the semantic version of these contracts.
|
|
24
|
+
|
|
25
|
+
## Usage
|
|
26
|
+
|
|
27
|
+
Each service declares a uv path dependency in its `pyproject.toml`:
|
|
28
|
+
|
|
29
|
+
```toml
|
|
30
|
+
[project]
|
|
31
|
+
dependencies = ["inh-contracts", ...]
|
|
32
|
+
|
|
33
|
+
[tool.uv.sources]
|
|
34
|
+
inh-contracts = { path = "../inh-contracts" }
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
and imports from it:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from inh_contracts.naming import get_workspace_collection_name, get_user_tenant_name
|
|
41
|
+
from inh_contracts.events import DocumentUploadMessage, DocumentCompletionMessage
|
|
42
|
+
```
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "inh-contracts"
|
|
3
|
+
version = "2.3.1"
|
|
4
|
+
description = "Shared contracts (Weaviate naming, versioned event schemas, embedding provider abstraction) consumed by the ingestion and public-api services"
|
|
5
|
+
readme = "Readme.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"pydantic>=2",
|
|
9
|
+
# Embedding provider HTTP adapters (#311) -- matches the floor already
|
|
10
|
+
# required by inh-ingestion-svc so the resolved version can't drift.
|
|
11
|
+
"httpx>=0.25.0",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[project.optional-dependencies]
|
|
15
|
+
# Dev tooling versions are normalized across all three services (M7 #21).
|
|
16
|
+
# See docs/developer/dependencies.md for the shared pin policy.
|
|
17
|
+
dev = [
|
|
18
|
+
"pytest==9.0.2",
|
|
19
|
+
"pytest-asyncio==1.3.0",
|
|
20
|
+
"pytest-cov==7.0.0",
|
|
21
|
+
"black==24.10.0",
|
|
22
|
+
"ruff==0.14.10",
|
|
23
|
+
"mypy==1.19.1",
|
|
24
|
+
"bandit[toml]==1.9.3",
|
|
25
|
+
"pre-commit>=3.6.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["hatchling"]
|
|
30
|
+
build-backend = "hatchling.build"
|
|
31
|
+
|
|
32
|
+
[tool.hatch.build.targets.wheel]
|
|
33
|
+
packages = ["src/inh_contracts"]
|
|
34
|
+
|
|
35
|
+
[tool.ruff]
|
|
36
|
+
line-length = 100
|
|
37
|
+
target-version = "py311"
|
|
38
|
+
|
|
39
|
+
[tool.ruff.lint]
|
|
40
|
+
select = ["E", "F", "I", "N", "W", "UP"]
|
|
41
|
+
ignore = ["E501"]
|
|
42
|
+
|
|
43
|
+
[tool.black]
|
|
44
|
+
line-length = 100
|
|
45
|
+
target-version = ["py311"]
|
|
46
|
+
|
|
47
|
+
[tool.mypy]
|
|
48
|
+
python_version = "3.11"
|
|
49
|
+
warn_return_any = true
|
|
50
|
+
warn_unused_configs = true
|
|
51
|
+
disallow_untyped_defs = false
|
|
52
|
+
ignore_missing_imports = true
|
|
53
|
+
|
|
54
|
+
# Mirrors inh-public-api-svc's [tool.bandit] verbatim so all three services
|
|
55
|
+
# gate on the SAME severity threshold -- CI invokes `bandit -c pyproject.toml
|
|
56
|
+
# -r src` everywhere. This package is clean at LOW and above with no
|
|
57
|
+
# suppressions; the config exists so it STAYS gated at that threshold rather
|
|
58
|
+
# than silently drifting to a laxer one.
|
|
59
|
+
[tool.bandit]
|
|
60
|
+
exclude_dirs = ["tests", "build", "dist"]
|
|
61
|
+
skips = ["B101"] # Skip assert_used check (used in tests)
|
|
62
|
+
|
|
63
|
+
[tool.bandit.assert_used]
|
|
64
|
+
skips = ["*_test.py", "test_*.py"]
|
|
65
|
+
|
|
66
|
+
[tool.pytest.ini_options]
|
|
67
|
+
testpaths = ["tests"]
|
|
68
|
+
python_files = ["test_*.py", "*_test.py"]
|
|
69
|
+
python_functions = ["test_*"]
|
|
70
|
+
# Mirrors the two services' pytest addopts (--cov=src) so `ci.yml`'s shared
|
|
71
|
+
# `--cov-fail-under=${{ matrix.cov_fail_under }}` Test step works the same
|
|
72
|
+
# way for this package (#132 item 4 -- this package previously ran in no
|
|
73
|
+
# automation at all, so a change to a shared constant like DEFAULT_S3_REGION
|
|
74
|
+
# was caught by nothing).
|
|
75
|
+
addopts = "--cov=src --cov-report=term-missing"
|
|
76
|
+
markers = [
|
|
77
|
+
"contract: contract/schema regression tests for shared event + naming surfaces",
|
|
78
|
+
"unit: fast isolated unit tests",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
[tool.coverage.report]
|
|
82
|
+
precision = 1
|
|
83
|
+
|
|
84
|
+
# Mirrors [project.optional-dependencies].dev so `uv sync --extra dev --group dev`
|
|
85
|
+
# resolves to the same normalized pins regardless of which flag is used (M7 #21).
|
|
86
|
+
[dependency-groups]
|
|
87
|
+
dev = [
|
|
88
|
+
"pytest==9.0.2",
|
|
89
|
+
"pytest-asyncio==1.3.0",
|
|
90
|
+
"pytest-cov==7.0.0",
|
|
91
|
+
"black==24.10.0",
|
|
92
|
+
"ruff==0.14.10",
|
|
93
|
+
"mypy==1.19.1",
|
|
94
|
+
"bandit[toml]==1.9.3",
|
|
95
|
+
"pre-commit>=3.6.0",
|
|
96
|
+
]
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""inh-contracts: shared contracts consumed by both Inherent services.
|
|
2
|
+
|
|
3
|
+
Single source of truth for Weaviate naming (#12), the versioned cross-service
|
|
4
|
+
event schemas (#17), shared configuration defaults (#132), the file-type
|
|
5
|
+
support registry (#117), and the embedding provider abstraction (#311). See
|
|
6
|
+
``inh_contracts.naming``, ``inh_contracts.events``, ``inh_contracts.defaults``,
|
|
7
|
+
``inh_contracts.file_types`` and ``inh_contracts.embedding``.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from inh_contracts.defaults import DEFAULT_MONGODB_URI, DEFAULT_S3_BUCKET, DEFAULT_S3_REGION
|
|
11
|
+
from inh_contracts.embedding import (
|
|
12
|
+
DEFAULT_EMBEDDING_PROVIDER,
|
|
13
|
+
EmbeddingIdentity,
|
|
14
|
+
EmbeddingIdentityAdoptionRequiredError,
|
|
15
|
+
EmbeddingIdentityMismatchError,
|
|
16
|
+
EmbeddingProvider,
|
|
17
|
+
OpenAICompatibleProvider,
|
|
18
|
+
TEIProvider,
|
|
19
|
+
create_embedding_provider,
|
|
20
|
+
decode_identity,
|
|
21
|
+
embed_batch_with_retry,
|
|
22
|
+
embed_single,
|
|
23
|
+
embed_texts_batched,
|
|
24
|
+
encode_identity,
|
|
25
|
+
is_transient_embed_error,
|
|
26
|
+
redact_url,
|
|
27
|
+
resolve_identity,
|
|
28
|
+
)
|
|
29
|
+
from inh_contracts.events import (
|
|
30
|
+
CONTRACT_VERSION,
|
|
31
|
+
ConversationTurnMessage,
|
|
32
|
+
DocumentCompletionMessage,
|
|
33
|
+
DocumentUploadMessage,
|
|
34
|
+
StorageBackend,
|
|
35
|
+
)
|
|
36
|
+
from inh_contracts.file_types import (
|
|
37
|
+
FILE_TYPE_REGISTRY,
|
|
38
|
+
ContentTypeMismatchError,
|
|
39
|
+
ExtensionMismatchError,
|
|
40
|
+
FileTypeSpec,
|
|
41
|
+
UnknownContentTypeError,
|
|
42
|
+
all_mime_types,
|
|
43
|
+
check_extension_consistency,
|
|
44
|
+
get_spec_by_key,
|
|
45
|
+
get_spec_for_extension,
|
|
46
|
+
get_spec_for_mime,
|
|
47
|
+
mcp_mime_types,
|
|
48
|
+
render_markdown_table,
|
|
49
|
+
sniff_content_type,
|
|
50
|
+
)
|
|
51
|
+
from inh_contracts.naming import (
|
|
52
|
+
USER_TENANT_PREFIX,
|
|
53
|
+
WORKSPACE_COLLECTION_PREFIX,
|
|
54
|
+
get_user_tenant_name,
|
|
55
|
+
get_workspace_collection_name,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
__all__ = [
|
|
59
|
+
"CONTRACT_VERSION",
|
|
60
|
+
"DocumentUploadMessage",
|
|
61
|
+
"DocumentCompletionMessage",
|
|
62
|
+
"ConversationTurnMessage",
|
|
63
|
+
"StorageBackend",
|
|
64
|
+
"get_workspace_collection_name",
|
|
65
|
+
"get_user_tenant_name",
|
|
66
|
+
"WORKSPACE_COLLECTION_PREFIX",
|
|
67
|
+
"USER_TENANT_PREFIX",
|
|
68
|
+
"DEFAULT_S3_REGION",
|
|
69
|
+
"DEFAULT_S3_BUCKET",
|
|
70
|
+
"DEFAULT_MONGODB_URI",
|
|
71
|
+
"FILE_TYPE_REGISTRY",
|
|
72
|
+
"FileTypeSpec",
|
|
73
|
+
"UnknownContentTypeError",
|
|
74
|
+
"ContentTypeMismatchError",
|
|
75
|
+
"ExtensionMismatchError",
|
|
76
|
+
"all_mime_types",
|
|
77
|
+
"mcp_mime_types",
|
|
78
|
+
"get_spec_for_mime",
|
|
79
|
+
"get_spec_for_extension",
|
|
80
|
+
"get_spec_by_key",
|
|
81
|
+
"sniff_content_type",
|
|
82
|
+
"check_extension_consistency",
|
|
83
|
+
"render_markdown_table",
|
|
84
|
+
"EmbeddingProvider",
|
|
85
|
+
"EmbeddingIdentity",
|
|
86
|
+
"EmbeddingIdentityMismatchError",
|
|
87
|
+
"EmbeddingIdentityAdoptionRequiredError",
|
|
88
|
+
"TEIProvider",
|
|
89
|
+
"OpenAICompatibleProvider",
|
|
90
|
+
"create_embedding_provider",
|
|
91
|
+
"redact_url",
|
|
92
|
+
"encode_identity",
|
|
93
|
+
"decode_identity",
|
|
94
|
+
"resolve_identity",
|
|
95
|
+
"embed_single",
|
|
96
|
+
"embed_texts_batched",
|
|
97
|
+
"embed_batch_with_retry",
|
|
98
|
+
"is_transient_embed_error",
|
|
99
|
+
"DEFAULT_EMBEDDING_PROVIDER",
|
|
100
|
+
]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Conversation-document contract: the synthetic identifiers a conversation's
|
|
2
|
+
``processed_documents`` row and Weaviate chunks carry (#306).
|
|
3
|
+
|
|
4
|
+
A conversation is not a file. It has no uploaded bytes, so
|
|
5
|
+
``ConversationMemoryWorkflow`` stamps synthetic-but-real values that satisfy
|
|
6
|
+
``processed_documents``' file-shaped NOT NULL/CHECK constraints (migration
|
|
7
|
+
001) while still being unambiguously identifiable as "not a file":
|
|
8
|
+
|
|
9
|
+
- ``content_type = CONVERSATION_CONTENT_TYPE`` -- written to
|
|
10
|
+
``processed_documents.content_type`` AND to every Weaviate chunk object's
|
|
11
|
+
``content_type`` property, so BOTH read paths (Postgres document row,
|
|
12
|
+
Weaviate search result) can tell a conversation chunk from a file chunk
|
|
13
|
+
without a second lookup.
|
|
14
|
+
- ``document_type = CONVERSATION_DOCUMENT_TYPE`` -- the Postgres-only column
|
|
15
|
+
migration 020 adds, used for row-scoped queries (external-id lookup,
|
|
16
|
+
conversation stats).
|
|
17
|
+
|
|
18
|
+
These live here rather than in either service because both sides need them:
|
|
19
|
+
``inh-ingestion-svc`` WRITES them (``conversation_memory.py``) and
|
|
20
|
+
``inh-public-api-svc`` READS them (freshness/staleness, conversation
|
|
21
|
+
queries). Same drift lesson as ``file_types.py`` -- a string literal
|
|
22
|
+
duplicated across two services is a bug waiting for one of them to be
|
|
23
|
+
edited alone.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from typing import Final
|
|
27
|
+
|
|
28
|
+
# The synthetic MIME type a conversation document/chunk carries. Deliberately
|
|
29
|
+
# in the `application/x-` vendor-extension space: it is NOT a real file format
|
|
30
|
+
# and must never match a FILE_TYPE_REGISTRY entry (see file_types.py).
|
|
31
|
+
CONVERSATION_CONTENT_TYPE: Final[str] = "application/x-inherent-conversation"
|
|
32
|
+
|
|
33
|
+
# `processed_documents.document_type` values (migration 020's CHECK
|
|
34
|
+
# constraint allows exactly these two). 'file' is the default every
|
|
35
|
+
# pre-existing row backfilled to.
|
|
36
|
+
CONVERSATION_DOCUMENT_TYPE: Final[str] = "conversation"
|
|
37
|
+
FILE_DOCUMENT_TYPE: Final[str] = "file"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def is_conversation(
|
|
41
|
+
*,
|
|
42
|
+
content_type: str | None = None,
|
|
43
|
+
document_type: str | None = None,
|
|
44
|
+
) -> bool:
|
|
45
|
+
"""Return True when either signal identifies a conversation document.
|
|
46
|
+
|
|
47
|
+
Both arguments are optional and independently sufficient, because the two
|
|
48
|
+
read paths carry different ones: a Weaviate search result has the chunk's
|
|
49
|
+
``content_type`` but no ``document_type`` (that column is Postgres-only),
|
|
50
|
+
while a ``processed_documents`` row has both. Unknown/absent values
|
|
51
|
+
(``None``) simply do not match -- this never guesses.
|
|
52
|
+
"""
|
|
53
|
+
return content_type == CONVERSATION_CONTENT_TYPE or document_type == CONVERSATION_DOCUMENT_TYPE
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Shared configuration defaults -- single source of truth (#132).
|
|
2
|
+
|
|
3
|
+
Both services accept an S3/object-store region override via an env var, but
|
|
4
|
+
each used to hardcode its OWN fallback default in its pydantic ``Settings``
|
|
5
|
+
class:
|
|
6
|
+
|
|
7
|
+
- ``inh-public-api-svc`` defaulted ``aws_s3_region`` to ``"eu-central-1"``.
|
|
8
|
+
- ``inh-ingestion-svc`` defaulted ``s3_region`` to ``"nbg1"`` (a Hetzner
|
|
9
|
+
Object Storage location code -- not even the same naming scheme as an
|
|
10
|
+
AWS-style region string).
|
|
11
|
+
|
|
12
|
+
A deployment that sets the region env var for only one service (or relies on
|
|
13
|
+
defaults during local dev / a bare ``uv run`` without docker-compose) silently
|
|
14
|
+
leaves the other service on its own, different default. Uploads then land in
|
|
15
|
+
one region/bucket while reads target another -- #132.
|
|
16
|
+
|
|
17
|
+
``DEFAULT_S3_REGION`` is now the ONE default both services' Settings classes
|
|
18
|
+
import, so the code-level fallback (used whenever the region env var is
|
|
19
|
+
unset) can never disagree between the two services again. This mirrors the
|
|
20
|
+
existing anti-drift pattern for Weaviate naming (see ``inh_contracts.naming``,
|
|
21
|
+
#12): put the single value here, have both services import it, and pin a
|
|
22
|
+
contract test on each side (``test_settings_region_contract.py``) so a future
|
|
23
|
+
hardcoded literal on either side fails CI instead of drifting silently.
|
|
24
|
+
|
|
25
|
+
The value matches the default already baked into the deployed stack
|
|
26
|
+
(``docker-compose.yml``, ``docker-compose.release.yml``, ``infra/server.tf``
|
|
27
|
+
and ``.env.example`` all default ``AWS_REGION`` to ``us-east-1``), so a
|
|
28
|
+
service now agrees with the documented, deployed default even when started
|
|
29
|
+
directly (e.g. tests, local ``uv run``) without compose's env injection.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
DEFAULT_S3_REGION = "us-east-1"
|
|
33
|
+
|
|
34
|
+
# ---------------------------------------------------------------------------
|
|
35
|
+
# #176 -- two more cross-service config defaults drifting the same way the
|
|
36
|
+
# S3 region did before #132. Both found while re-reading each service's
|
|
37
|
+
# Settings class during the #132 pattern sweep; scripts/validate_env.py
|
|
38
|
+
# already runtime-checks the resolved values for both, but that only warns
|
|
39
|
+
# once an operator has actually set the two env vars to *different* values --
|
|
40
|
+
# it does not catch the two services silently defaulting to different values
|
|
41
|
+
# when an operator sets neither.
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
# ``inh-ingestion-svc`` defaulted ``storage_bucket`` (env ``STORAGE_BUCKET``)
|
|
45
|
+
# to ``""`` while ``inh-public-api-svc`` defaulted ``aws_s3_bucket`` (env
|
|
46
|
+
# ``AWS_S3_BUCKET``) to ``"inherent-documents"``. An operator who sets
|
|
47
|
+
# neither env var got ingestion writing to an empty bucket name while
|
|
48
|
+
# public-api read from ``"inherent-documents"`` -- the exact #132 shape.
|
|
49
|
+
# Ingestion's primary upload path mostly uses the per-message
|
|
50
|
+
# ``storage_bucket`` from the event payload rather than this Settings
|
|
51
|
+
# default (see ``services/storage.py``'s ``default_bucket`` fallback), so the
|
|
52
|
+
# empty-string default was rarely the effective value -- but the same
|
|
53
|
+
# single-source-of-truth fix applies for defense in depth and consistency
|
|
54
|
+
# with the #132 precedent. Both services now default to this shared bucket
|
|
55
|
+
# name; the real deployed bucket is still fully operator-configurable via
|
|
56
|
+
# ``STORAGE_BUCKET`` / ``AWS_S3_BUCKET``.
|
|
57
|
+
DEFAULT_S3_BUCKET = "inherent-documents"
|
|
58
|
+
|
|
59
|
+
# Both services also defaulted ``mongodb_db_name`` to ``"main"`` and both
|
|
60
|
+
# select the database explicitly via that field when opening the Mongo
|
|
61
|
+
# client (``client[settings.mongodb_db_name]``, see
|
|
62
|
+
# ``inh-public-api-svc/src/services/mongo_client.py`` and
|
|
63
|
+
# ``inh-ingestion-svc/src/temporal/activities/audit_activities.py``) -- so
|
|
64
|
+
# the URI's own path segment is inert either way. Still, ``mongodb_uri``
|
|
65
|
+
# defaulted to two different literals (``.../27017`` vs ``.../27017/main``),
|
|
66
|
+
# the same "hardcoded twice, drifted slightly" shape #132 exists to prevent.
|
|
67
|
+
# Single-sourced here with no path segment -- the path is not the source of
|
|
68
|
+
# truth for database selection, ``mongodb_db_name`` is, so the shared
|
|
69
|
+
# default should not imply otherwise.
|
|
70
|
+
DEFAULT_MONGODB_URI = "mongodb://localhost:27017"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Shared embedding-provider abstraction (#311).
|
|
2
|
+
|
|
3
|
+
Single source of truth for both services' embedder modules: the
|
|
4
|
+
``EmbeddingProvider`` interface, the TEI (default) and OpenAI-compatible wire
|
|
5
|
+
adapters, the factory that selects one from ``EMBEDDING_PROVIDER``, the
|
|
6
|
+
retry/batching helpers that give both the ingestion write path and the
|
|
7
|
+
public-api query path identical behavior, and the Weaviate collection
|
|
8
|
+
model-identity guard. See ``inh_contracts.embedding.identity`` for the
|
|
9
|
+
identity-guard policy write-up.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from inh_contracts.embedding.batching import embed_single, embed_texts_batched
|
|
13
|
+
from inh_contracts.embedding.defaults import (
|
|
14
|
+
BATCH_RETRY_SLEEP_BUDGET_S,
|
|
15
|
+
DEFAULT_BATCH_MAX_RETRIES,
|
|
16
|
+
DEFAULT_BATCH_SIZE,
|
|
17
|
+
DEFAULT_EMBEDDING_PROVIDER,
|
|
18
|
+
DEFAULT_MAX_CONCURRENCY,
|
|
19
|
+
DEFAULT_QUERY_MAX_RETRIES,
|
|
20
|
+
DEFAULT_QUERY_TIMEOUT_S,
|
|
21
|
+
DEFAULT_TIMEOUT_S,
|
|
22
|
+
QUERY_RETRY_SLEEP_BUDGET_S,
|
|
23
|
+
)
|
|
24
|
+
from inh_contracts.embedding.factory import create_embedding_provider
|
|
25
|
+
from inh_contracts.embedding.identity import (
|
|
26
|
+
EmbeddingIdentityAdoptionRequiredError,
|
|
27
|
+
EmbeddingIdentityMismatchError,
|
|
28
|
+
decode_identity,
|
|
29
|
+
encode_identity,
|
|
30
|
+
resolve_identity,
|
|
31
|
+
)
|
|
32
|
+
from inh_contracts.embedding.openai_provider import OpenAICompatibleProvider
|
|
33
|
+
from inh_contracts.embedding.provider import EmbeddingIdentity, EmbeddingProvider, redact_url
|
|
34
|
+
from inh_contracts.embedding.retry import (
|
|
35
|
+
embed_batch_with_retry,
|
|
36
|
+
is_transient_embed_error,
|
|
37
|
+
max_wall_clock_s,
|
|
38
|
+
)
|
|
39
|
+
from inh_contracts.embedding.tei_provider import TEIProvider
|
|
40
|
+
|
|
41
|
+
__all__ = [
|
|
42
|
+
"EmbeddingProvider",
|
|
43
|
+
"EmbeddingIdentity",
|
|
44
|
+
"EmbeddingIdentityMismatchError",
|
|
45
|
+
"EmbeddingIdentityAdoptionRequiredError",
|
|
46
|
+
"TEIProvider",
|
|
47
|
+
"OpenAICompatibleProvider",
|
|
48
|
+
"create_embedding_provider",
|
|
49
|
+
"redact_url",
|
|
50
|
+
"encode_identity",
|
|
51
|
+
"decode_identity",
|
|
52
|
+
"resolve_identity",
|
|
53
|
+
"embed_single",
|
|
54
|
+
"embed_texts_batched",
|
|
55
|
+
"embed_batch_with_retry",
|
|
56
|
+
"is_transient_embed_error",
|
|
57
|
+
"max_wall_clock_s",
|
|
58
|
+
"DEFAULT_EMBEDDING_PROVIDER",
|
|
59
|
+
"DEFAULT_BATCH_SIZE",
|
|
60
|
+
"DEFAULT_MAX_CONCURRENCY",
|
|
61
|
+
"DEFAULT_TIMEOUT_S",
|
|
62
|
+
"DEFAULT_BATCH_MAX_RETRIES",
|
|
63
|
+
"BATCH_RETRY_SLEEP_BUDGET_S",
|
|
64
|
+
"DEFAULT_QUERY_TIMEOUT_S",
|
|
65
|
+
"DEFAULT_QUERY_MAX_RETRIES",
|
|
66
|
+
"QUERY_RETRY_SLEEP_BUDGET_S",
|
|
67
|
+
]
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Zero-vector shortcuts, TEI-sized batching, and bounded concurrency (#311).
|
|
2
|
+
|
|
3
|
+
Ported verbatim (behaviorally) from ``inh-ingestion-svc``'s ``embedder.py``,
|
|
4
|
+
which is the side that originally had this logic; ``embed_query`` on the
|
|
5
|
+
public-api side gets it for the first time via this shared module.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
11
|
+
|
|
12
|
+
from inh_contracts.embedding.defaults import (
|
|
13
|
+
BATCH_RETRY_SLEEP_BUDGET_S,
|
|
14
|
+
DEFAULT_BATCH_MAX_RETRIES,
|
|
15
|
+
DEFAULT_BATCH_SIZE,
|
|
16
|
+
DEFAULT_MAX_CONCURRENCY,
|
|
17
|
+
)
|
|
18
|
+
from inh_contracts.embedding.provider import EmbeddingProvider
|
|
19
|
+
from inh_contracts.embedding.retry import embed_batch_with_retry
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def embed_single(
|
|
23
|
+
provider: EmbeddingProvider,
|
|
24
|
+
text: str,
|
|
25
|
+
*,
|
|
26
|
+
max_retries: int = DEFAULT_BATCH_MAX_RETRIES,
|
|
27
|
+
retry_budget_s: float = BATCH_RETRY_SLEEP_BUDGET_S,
|
|
28
|
+
) -> list[float]:
|
|
29
|
+
"""Return a normalized embedding for one text.
|
|
30
|
+
|
|
31
|
+
Empty / whitespace-only input returns a zero vector -- those chunks/
|
|
32
|
+
queries shouldn't surface in semantic search results anyway, and we
|
|
33
|
+
avoid a network round-trip.
|
|
34
|
+
"""
|
|
35
|
+
if not text or not text.strip():
|
|
36
|
+
return [0.0] * provider.dimension
|
|
37
|
+
vecs = embed_batch_with_retry(
|
|
38
|
+
provider, [text], max_retries=max_retries, retry_budget_s=retry_budget_s
|
|
39
|
+
)
|
|
40
|
+
return vecs[0]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def embed_texts_batched(
|
|
44
|
+
provider: EmbeddingProvider,
|
|
45
|
+
texts: list[str],
|
|
46
|
+
*,
|
|
47
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
48
|
+
max_concurrency: int = DEFAULT_MAX_CONCURRENCY,
|
|
49
|
+
max_retries: int = DEFAULT_BATCH_MAX_RETRIES,
|
|
50
|
+
retry_budget_s: float = BATCH_RETRY_SLEEP_BUDGET_S,
|
|
51
|
+
) -> list[list[float]]:
|
|
52
|
+
"""Batched embedding with bounded parallel dispatch (#231 phase 1).
|
|
53
|
+
|
|
54
|
+
Empty strings still get zero vectors (preserved per-position), and only
|
|
55
|
+
the non-empty positions go over the wire. Batches run concurrently up to
|
|
56
|
+
``max_concurrency`` so a large document is
|
|
57
|
+
``ceil(n_batches / concurrency)`` round-trips instead of ``n_batches``
|
|
58
|
+
serial ones.
|
|
59
|
+
"""
|
|
60
|
+
dim = provider.dimension
|
|
61
|
+
if not texts:
|
|
62
|
+
return []
|
|
63
|
+
keep_idx = [i for i, t in enumerate(texts) if t and t.strip()]
|
|
64
|
+
if not keep_idx:
|
|
65
|
+
return [[0.0] * dim for _ in texts]
|
|
66
|
+
|
|
67
|
+
# Chunk into batches under the provider's max batch size to avoid a
|
|
68
|
+
# payload-too-large response. A 535-chunk PDF was failing with one giant
|
|
69
|
+
# POST against TEI; batching keeps every request comfortably sized.
|
|
70
|
+
batch = max(1, batch_size)
|
|
71
|
+
keep_texts = [texts[i] for i in keep_idx]
|
|
72
|
+
batches: list[tuple[int, list[str]]] = []
|
|
73
|
+
for offset in range(0, len(keep_texts), batch):
|
|
74
|
+
batches.append((offset, keep_texts[offset : offset + batch]))
|
|
75
|
+
|
|
76
|
+
# index-in-keep_texts -> vector
|
|
77
|
+
results: dict[int, list[float]] = {}
|
|
78
|
+
concurrency = min(max(1, max_concurrency), len(batches))
|
|
79
|
+
|
|
80
|
+
def _run_batch(item: tuple[int, list[str]]) -> tuple[int, list[list[float]]]:
|
|
81
|
+
offset, inputs = item
|
|
82
|
+
return offset, embed_batch_with_retry(
|
|
83
|
+
provider, inputs, max_retries=max_retries, retry_budget_s=retry_budget_s
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
if concurrency == 1:
|
|
87
|
+
for item in batches:
|
|
88
|
+
offset, vecs = _run_batch(item)
|
|
89
|
+
for j, vec in enumerate(vecs):
|
|
90
|
+
results[offset + j] = vec
|
|
91
|
+
else:
|
|
92
|
+
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
|
93
|
+
futures = [pool.submit(_run_batch, item) for item in batches]
|
|
94
|
+
for fut in as_completed(futures):
|
|
95
|
+
offset, vecs = fut.result()
|
|
96
|
+
for j, vec in enumerate(vecs):
|
|
97
|
+
results[offset + j] = vec
|
|
98
|
+
|
|
99
|
+
out: list[list[float]] = [[0.0] * dim for _ in texts]
|
|
100
|
+
for j, i in enumerate(keep_idx):
|
|
101
|
+
out[i] = results[j]
|
|
102
|
+
return out
|