genorbex-dynamic-search 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- genorbex_dynamic_search-0.1.0/.gitignore +84 -0
- genorbex_dynamic_search-0.1.0/PKG-INFO +195 -0
- genorbex_dynamic_search-0.1.0/README.md +163 -0
- genorbex_dynamic_search-0.1.0/alembic/env.py +61 -0
- genorbex_dynamic_search-0.1.0/alembic/versions/0001_search_documents.py +81 -0
- genorbex_dynamic_search-0.1.0/alembic.ini +38 -0
- genorbex_dynamic_search-0.1.0/pyproject.toml +56 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/__init__.py +25 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/api/__init__.py +2 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/api/router.py +201 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/config.py +57 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/exceptions.py +21 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/__init__.py +20 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/agents.py +17 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/apps.py +18 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/audit_events.py +19 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/base.py +139 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/connectors.py +17 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/documents.py +31 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/executions.py +19 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/workflows.py +13 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/jobs/__init__.py +2 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/jobs/backfill.py +121 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/jobs/queue.py +96 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/models.py +190 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/__init__.py +2 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/base.py +27 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/opensearch.py +265 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/postgres.py +297 -0
- genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/service.py +175 -0
- genorbex_dynamic_search-0.1.0/tests/test_api_and_jobs.py +75 -0
- genorbex_dynamic_search-0.1.0/tests/test_contracts.py +78 -0
- genorbex_dynamic_search-0.1.0/tests/test_opensearch.py +108 -0
- genorbex_dynamic_search-0.1.0/tests/test_postgres_integration.py +77 -0
- genorbex_dynamic_search-0.1.0/tests/test_postgres_provider.py +152 -0
- genorbex_dynamic_search-0.1.0/tests/test_service_and_indexers.py +114 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
|
2
|
+
|
|
3
|
+
# dependencies
|
|
4
|
+
node_modules/
|
|
5
|
+
/.pnp
|
|
6
|
+
.pnp.*
|
|
7
|
+
.yarn/*
|
|
8
|
+
!.yarn/patches
|
|
9
|
+
!.yarn/plugins
|
|
10
|
+
!.yarn/releases
|
|
11
|
+
!.yarn/versions
|
|
12
|
+
|
|
13
|
+
# testing
|
|
14
|
+
/coverage
|
|
15
|
+
|
|
16
|
+
# next.js
|
|
17
|
+
/.next/
|
|
18
|
+
/.next-*/
|
|
19
|
+
/out/
|
|
20
|
+
|
|
21
|
+
# production
|
|
22
|
+
/build
|
|
23
|
+
/desktop-app
|
|
24
|
+
/desktop-dist
|
|
25
|
+
/public/downloads/*.dmg
|
|
26
|
+
/public/downloads/*.zip
|
|
27
|
+
/public/downloads/*.blockmap
|
|
28
|
+
|
|
29
|
+
# AI Code Builder preview
|
|
30
|
+
/.preview
|
|
31
|
+
/.streamlit-runtime/
|
|
32
|
+
|
|
33
|
+
# AI Code Builder persistent projects (local development only)
|
|
34
|
+
/ai-code-projects
|
|
35
|
+
|
|
36
|
+
# misc
|
|
37
|
+
.DS_Store
|
|
38
|
+
*.pem
|
|
39
|
+
|
|
40
|
+
# debug
|
|
41
|
+
npm-debug.log*
|
|
42
|
+
yarn-debug.log*
|
|
43
|
+
yarn-error.log*
|
|
44
|
+
.pnpm-debug.log*
|
|
45
|
+
|
|
46
|
+
# build logs
|
|
47
|
+
build.log
|
|
48
|
+
@build.log
|
|
49
|
+
|
|
50
|
+
# env files (can opt-in for committing if needed)
|
|
51
|
+
.env*
|
|
52
|
+
!.env.orbexdb-pg-wire.example
|
|
53
|
+
|
|
54
|
+
# vercel
|
|
55
|
+
.vercel
|
|
56
|
+
|
|
57
|
+
# typescript
|
|
58
|
+
*.tsbuildinfo
|
|
59
|
+
next-env.d.ts
|
|
60
|
+
|
|
61
|
+
/src/generated/prisma
|
|
62
|
+
.env.local
|
|
63
|
+
.env
|
|
64
|
+
.env.development
|
|
65
|
+
.env.production
|
|
66
|
+
.env.test
|
|
67
|
+
.env.development.local
|
|
68
|
+
.env.production.local
|
|
69
|
+
.env.test.local
|
|
70
|
+
..env.production.backup.
|
|
71
|
+
|
|
72
|
+
# AI Code Builder deployments
|
|
73
|
+
/deployments/
|
|
74
|
+
|
|
75
|
+
# local Prisma dev state
|
|
76
|
+
/.prisma-home
|
|
77
|
+
/.emka-storage-local
|
|
78
|
+
/.orbex-whatsapp-web/
|
|
79
|
+
|
|
80
|
+
# zlog
|
|
81
|
+
zlog
|
|
82
|
+
|
|
83
|
+
# local Genorbex security SDK reports
|
|
84
|
+
/sdks/python-security/reports/
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: genorbex-dynamic-search
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tenant-isolated dynamic search service for Genorbex applications
|
|
5
|
+
Author: Genorbex
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: genorbex,multi-tenant,opensearch,postgresql,search
|
|
8
|
+
Requires-Python: >=3.12
|
|
9
|
+
Requires-Dist: asyncpg>=0.29
|
|
10
|
+
Requires-Dist: fastapi>=0.110
|
|
11
|
+
Requires-Dist: pydantic-settings>=2.2
|
|
12
|
+
Requires-Dist: pydantic>=2.7
|
|
13
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: alembic>=1.13; extra == 'dev'
|
|
16
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
17
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
18
|
+
Requires-Dist: opensearch-py[async]>=2.5; extra == 'dev'
|
|
19
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
20
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
21
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
22
|
+
Provides-Extra: migrations
|
|
23
|
+
Requires-Dist: alembic>=1.13; extra == 'migrations'
|
|
24
|
+
Provides-Extra: opensearch
|
|
25
|
+
Requires-Dist: opensearch-py[async]>=2.5; extra == 'opensearch'
|
|
26
|
+
Provides-Extra: test
|
|
27
|
+
Requires-Dist: aiosqlite>=0.20; extra == 'test'
|
|
28
|
+
Requires-Dist: httpx>=0.27; extra == 'test'
|
|
29
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
|
|
30
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# genorbex-dynamic-search
|
|
34
|
+
|
|
35
|
+
An async, multi-tenant search service package for Genorbex resources. PostgreSQL full-text
|
|
36
|
+
search is the default provider; OpenSearch is an optional adapter with the same response model.
|
|
37
|
+
The package is intentionally an installable service library, not a new web deployment or
|
|
38
|
+
database infrastructure definition.
|
|
39
|
+
|
|
40
|
+
## Install and configure
|
|
41
|
+
|
|
42
|
+
Python 3.12 or newer is required.
|
|
43
|
+
|
|
44
|
+
```sh
|
|
45
|
+
python -m pip install -e '.[migrations,test]'
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
PostgreSQL is the default. Use the application database or a dedicated PostgreSQL database and
|
|
49
|
+
run the package migration before mounting the API:
|
|
50
|
+
|
|
51
|
+
```env
|
|
52
|
+
SEARCH_PROVIDER=postgres
|
|
53
|
+
SEARCH_DATABASE_URL=postgresql://user:password@host:5432/genorbex
|
|
54
|
+
SEARCH_ENVIRONMENT=dev
|
|
55
|
+
SEARCH_QUERY_RATE_LIMIT=60
|
|
56
|
+
SEARCH_QUERY_RATE_WINDOW_SECONDS=60
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The migration is package-owned Alembic SQL. This repository's main web application currently
|
|
64
|
+
uses Prisma migrations and has no Python FastAPI host. Keep migration ownership explicit: either
|
|
65
|
+
run this Alembic migration as part of the Python-service release or port the SQL into the host's
|
|
66
|
+
Prisma migration workflow before sharing deployment ownership of the table.
|
|
67
|
+
|
|
68
|
+
## Use the service
|
|
69
|
+
|
|
70
|
+
Application code depends on `DynamicSearchService`, not provider classes:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from datetime import datetime, timezone
|
|
74
|
+
|
|
75
|
+
from genorbex_dynamic_search import ResourceType, SearchDocument, SearchRequest
|
|
76
|
+
from genorbex_dynamic_search.config import SearchSettings
|
|
77
|
+
from genorbex_dynamic_search.service import create_search_service
|
|
78
|
+
|
|
79
|
+
service = create_search_service(SearchSettings())
|
|
80
|
+
await service.initialize()
|
|
81
|
+
await service.index(SearchDocument(
|
|
82
|
+
id="workflow:flow-123",
|
|
83
|
+
tenant_id="org-123", # trusted organization ID; use user ID for personal tenants
|
|
84
|
+
resource_type=ResourceType.WORKFLOW,
|
|
85
|
+
title="Customer onboarding",
|
|
86
|
+
content="Verify the account and provision the customer workspace.",
|
|
87
|
+
source_id="flow-123",
|
|
88
|
+
created_at=datetime.now(timezone.utc),
|
|
89
|
+
updated_at=datetime.now(timezone.utc),
|
|
90
|
+
))
|
|
91
|
+
response = await service.search(SearchRequest(tenant_id="org-123", query="onboarding"))
|
|
92
|
+
await service.aclose()
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
`SearchRequest` supports resource type, project, tag, creator, date, sort, cursor, and page-size
|
|
96
|
+
filters. Results include ranked hits, PostgreSQL/OpenSearch highlights, type facets, total count,
|
|
97
|
+
latency, and an opaque `next_cursor`. Cursors are tied to their sort order; pass the same sort
|
|
98
|
+
on the next request. Treat highlight strings as untrusted display text and escape them in HTML.
|
|
99
|
+
|
|
100
|
+
## FastAPI integration and authentication
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from fastapi import FastAPI
|
|
104
|
+
from genorbex_dynamic_search.api.router import create_search_router
|
|
105
|
+
from genorbex_dynamic_search.service import create_search_service
|
|
106
|
+
|
|
107
|
+
app = FastAPI()
|
|
108
|
+
search_service = create_search_service()
|
|
109
|
+
app.include_router(create_search_router(search_service, identity_dependency=genorbex_identity))
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`genorbex_identity` must adapt the authenticated principal established by the host's middleware
|
|
113
|
+
to `SearchIdentity(user_id=..., tenant_id=..., role=..., permissions=...)`. The default adapter
|
|
114
|
+
reads `request.state.user` or `request.state.current_user`; it fails closed if no trusted identity
|
|
115
|
+
is present and uses `organizationId` as the tenant, falling back to the authenticated user ID for
|
|
116
|
+
personal workspaces. The package never reads a tenant from query parameters or request headers.
|
|
117
|
+
Manual index/delete endpoints require an admin/owner role or an explicit search-admin permission.
|
|
118
|
+
The GET endpoint applies a configurable per-tenant rate limit and max query length. Its built-in
|
|
119
|
+
limiter is process-local; inject a shared limiter implementation for multi-worker deployments.
|
|
120
|
+
|
|
121
|
+
The current Genorbex web application is Next.js/Prisma, not FastAPI, so its authentication
|
|
122
|
+
middleware cannot be imported directly into a Python process. Mount the router only in a host
|
|
123
|
+
that supplies that trusted identity adapter.
|
|
124
|
+
|
|
125
|
+
## Resource indexing and queue lifecycle
|
|
126
|
+
|
|
127
|
+
Indexers are provided for workflows, executions, agents, generated apps, connectors, documents,
|
|
128
|
+
knowledge chunks, and audit events. Call `enqueue_upsert` after create/update and `enqueue_delete`
|
|
129
|
+
after delete, passing the host's durable `SearchJobQueue` adapter. That adapter's `enqueue` method
|
|
130
|
+
must persist the serialized job before returning. A worker deserializes `SearchIndexJob` and calls
|
|
131
|
+
`process_search_job(service, job)`; failures use bounded exponential retries and are logged with
|
|
132
|
+
structured context. Upserts and deletes are idempotent.
|
|
133
|
+
|
|
134
|
+
The repository has a Prisma-backed message queue in the TypeScript application, but no Python
|
|
135
|
+
consumer/adapter contract exists yet. The package therefore defines the queue protocol and worker
|
|
136
|
+
handler without creating another queue, Redis service, or cloud resource. Connect the protocol to
|
|
137
|
+
the existing queue consumer at the application boundary.
|
|
138
|
+
|
|
139
|
+
Indexers deliberately avoid workflow definitions, prompts, execution input/output blobs, connector
|
|
140
|
+
settings, and credential fields. SearchDocument additionally redacts common credential assignments
|
|
141
|
+
and JWT-shaped strings, and metadata is filtered through an explicit allowlist. Review extracted
|
|
142
|
+
document text before indexing if it may contain regulated or confidential data.
|
|
143
|
+
|
|
144
|
+
## Backfill
|
|
145
|
+
|
|
146
|
+
The CLI requires exactly one scope. `--all-tenants` is an explicit opt-in and never the default:
|
|
147
|
+
|
|
148
|
+
```sh
|
|
149
|
+
GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
|
|
150
|
+
python -m genorbex_dynamic_search.jobs.backfill --tenant-id org-123
|
|
151
|
+
|
|
152
|
+
GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
|
|
153
|
+
python -m genorbex_dynamic_search.jobs.backfill --all-tenants
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
The source factory returns an adapter implementing `authorize_backfill(tenant_id, all_tenants)`,
|
|
157
|
+
`list_tenant_ids()`, and async `iter_documents(tenant_id)`. The authorization method must reject
|
|
158
|
+
operators who are not allowed to backfill the requested scope; it runs before any tenant records
|
|
159
|
+
are fetched. Every yielded document is checked against the tenant being backfilled. This adapter is
|
|
160
|
+
where host-specific Prisma/ORM reads and the package indexers belong.
|
|
161
|
+
|
|
162
|
+
## OpenSearch later
|
|
163
|
+
|
|
164
|
+
OpenSearch is optional and is never started or provisioned by this package:
|
|
165
|
+
|
|
166
|
+
```sh
|
|
167
|
+
python -m pip install -e '.[opensearch]'
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
```env
|
|
171
|
+
SEARCH_PROVIDER=opensearch
|
|
172
|
+
SEARCH_OPENSEARCH_URL=https://search.example.internal
|
|
173
|
+
SEARCH_OPENSEARCH_USERNAME=...
|
|
174
|
+
SEARCH_OPENSEARCH_PASSWORD=...
|
|
175
|
+
SEARCH_ENVIRONMENT=prod
|
|
176
|
+
# Defaults to genorbex-search-prod; override with SEARCH_OPENSEARCH_INDEX if needed.
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
The provider creates an index with text, keyword, timestamp, tenant, and metadata mappings when
|
|
180
|
+
`await service.initialize()` runs. Every query includes a mandatory tenant filter. No embedding
|
|
181
|
+
service or vector index is required; vector/semantic search can be added later without changing
|
|
182
|
+
the `SearchResponse` contract.
|
|
183
|
+
|
|
184
|
+
## Migrations, tests, and observability
|
|
185
|
+
|
|
186
|
+
```sh
|
|
187
|
+
alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
|
|
188
|
+
python -m unittest discover -s packages/genorbex-dynamic-search/tests
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Set `SEARCH_TEST_DATABASE_URL` to a disposable PostgreSQL database with the migration applied to
|
|
192
|
+
enable integration tests. Unit tests mock OpenSearch. The service emits structured query latency,
|
|
193
|
+
result-count, index failure, retry, and provider-health logs. Optionally inject `SearchMetrics` to
|
|
194
|
+
export query latency/result counts, index failures, and provider health; inject `SearchJobMetrics`
|
|
195
|
+
into the worker to export job retries and failures.
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# genorbex-dynamic-search
|
|
2
|
+
|
|
3
|
+
An async, multi-tenant search service package for Genorbex resources. PostgreSQL full-text
|
|
4
|
+
search is the default provider; OpenSearch is an optional adapter with the same response model.
|
|
5
|
+
The package is intentionally an installable service library, not a new web deployment or
|
|
6
|
+
database infrastructure definition.
|
|
7
|
+
|
|
8
|
+
## Install and configure
|
|
9
|
+
|
|
10
|
+
Python 3.12 or newer is required.
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
python -m pip install -e '.[migrations,test]'
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
PostgreSQL is the default. Use the application database or a dedicated PostgreSQL database and
|
|
17
|
+
run the package migration before mounting the API:
|
|
18
|
+
|
|
19
|
+
```env
|
|
20
|
+
SEARCH_PROVIDER=postgres
|
|
21
|
+
SEARCH_DATABASE_URL=postgresql://user:password@host:5432/genorbex
|
|
22
|
+
SEARCH_ENVIRONMENT=dev
|
|
23
|
+
SEARCH_QUERY_RATE_LIMIT=60
|
|
24
|
+
SEARCH_QUERY_RATE_WINDOW_SECONDS=60
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```sh
|
|
28
|
+
alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The migration is package-owned Alembic SQL. This repository's main web application currently
|
|
32
|
+
uses Prisma migrations and has no Python FastAPI host. Keep migration ownership explicit: either
|
|
33
|
+
run this Alembic migration as part of the Python-service release or port the SQL into the host's
|
|
34
|
+
Prisma migration workflow before sharing deployment ownership of the table.
|
|
35
|
+
|
|
36
|
+
## Use the service
|
|
37
|
+
|
|
38
|
+
Application code depends on `DynamicSearchService`, not provider classes:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from datetime import datetime, timezone
|
|
42
|
+
|
|
43
|
+
from genorbex_dynamic_search import ResourceType, SearchDocument, SearchRequest
|
|
44
|
+
from genorbex_dynamic_search.config import SearchSettings
|
|
45
|
+
from genorbex_dynamic_search.service import create_search_service
|
|
46
|
+
|
|
47
|
+
service = create_search_service(SearchSettings())
|
|
48
|
+
await service.initialize()
|
|
49
|
+
await service.index(SearchDocument(
|
|
50
|
+
id="workflow:flow-123",
|
|
51
|
+
tenant_id="org-123", # trusted organization ID; use user ID for personal tenants
|
|
52
|
+
resource_type=ResourceType.WORKFLOW,
|
|
53
|
+
title="Customer onboarding",
|
|
54
|
+
content="Verify the account and provision the customer workspace.",
|
|
55
|
+
source_id="flow-123",
|
|
56
|
+
created_at=datetime.now(timezone.utc),
|
|
57
|
+
updated_at=datetime.now(timezone.utc),
|
|
58
|
+
))
|
|
59
|
+
response = await service.search(SearchRequest(tenant_id="org-123", query="onboarding"))
|
|
60
|
+
await service.aclose()
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`SearchRequest` supports resource type, project, tag, creator, date, sort, cursor, and page-size
|
|
64
|
+
filters. Results include ranked hits, PostgreSQL/OpenSearch highlights, type facets, total count,
|
|
65
|
+
latency, and an opaque `next_cursor`. Cursors are tied to their sort order; pass the same sort
|
|
66
|
+
on the next request. Treat highlight strings as untrusted display text and escape them in HTML.
|
|
67
|
+
|
|
68
|
+
## FastAPI integration and authentication
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from fastapi import FastAPI
|
|
72
|
+
from genorbex_dynamic_search.api.router import create_search_router
|
|
73
|
+
from genorbex_dynamic_search.service import create_search_service
|
|
74
|
+
|
|
75
|
+
app = FastAPI()
|
|
76
|
+
search_service = create_search_service()
|
|
77
|
+
app.include_router(create_search_router(search_service, identity_dependency=genorbex_identity))
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`genorbex_identity` must adapt the authenticated principal established by the host's middleware
|
|
81
|
+
to `SearchIdentity(user_id=..., tenant_id=..., role=..., permissions=...)`. The default adapter
|
|
82
|
+
reads `request.state.user` or `request.state.current_user`; it fails closed if no trusted identity
|
|
83
|
+
is present and uses `organizationId` as the tenant, falling back to the authenticated user ID for
|
|
84
|
+
personal workspaces. The package never reads a tenant from query parameters or request headers.
|
|
85
|
+
Manual index/delete endpoints require an admin/owner role or an explicit search-admin permission.
|
|
86
|
+
The GET endpoint applies a configurable per-tenant rate limit and max query length. Its built-in
|
|
87
|
+
limiter is process-local; inject a shared limiter implementation for multi-worker deployments.
|
|
88
|
+
|
|
89
|
+
The current Genorbex web application is Next.js/Prisma, not FastAPI, so its authentication
|
|
90
|
+
middleware cannot be imported directly into a Python process. Mount the router only in a host
|
|
91
|
+
that supplies that trusted identity adapter.
|
|
92
|
+
|
|
93
|
+
## Resource indexing and queue lifecycle
|
|
94
|
+
|
|
95
|
+
Indexers are provided for workflows, executions, agents, generated apps, connectors, documents,
|
|
96
|
+
knowledge chunks, and audit events. Call `enqueue_upsert` after create/update and `enqueue_delete`
|
|
97
|
+
after delete, passing the host's durable `SearchJobQueue` adapter. That adapter's `enqueue` method
|
|
98
|
+
must persist the serialized job before returning. A worker deserializes `SearchIndexJob` and calls
|
|
99
|
+
`process_search_job(service, job)`; failures use bounded exponential retries and are logged with
|
|
100
|
+
structured context. Upserts and deletes are idempotent.
|
|
101
|
+
|
|
102
|
+
The repository has a Prisma-backed message queue in the TypeScript application, but no Python
|
|
103
|
+
consumer/adapter contract exists yet. The package therefore defines the queue protocol and worker
|
|
104
|
+
handler without creating another queue, Redis service, or cloud resource. Connect the protocol to
|
|
105
|
+
the existing queue consumer at the application boundary.
|
|
106
|
+
|
|
107
|
+
Indexers deliberately avoid workflow definitions, prompts, execution input/output blobs, connector
|
|
108
|
+
settings, and credential fields. SearchDocument additionally redacts common credential assignments
|
|
109
|
+
and JWT-shaped strings, and metadata is filtered through an explicit allowlist. Review extracted
|
|
110
|
+
document text before indexing if it may contain regulated or confidential data.
|
|
111
|
+
|
|
112
|
+
## Backfill
|
|
113
|
+
|
|
114
|
+
The CLI requires exactly one scope. `--all-tenants` is an explicit opt-in and never the default:
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
|
|
118
|
+
python -m genorbex_dynamic_search.jobs.backfill --tenant-id org-123
|
|
119
|
+
|
|
120
|
+
GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
|
|
121
|
+
python -m genorbex_dynamic_search.jobs.backfill --all-tenants
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The source factory returns an adapter implementing `authorize_backfill(tenant_id, all_tenants)`,
|
|
125
|
+
`list_tenant_ids()`, and async `iter_documents(tenant_id)`. The authorization method must reject
|
|
126
|
+
operators who are not allowed to backfill the requested scope; it runs before any tenant records
|
|
127
|
+
are fetched. Every yielded document is checked against the tenant being backfilled. This adapter is
|
|
128
|
+
where host-specific Prisma/ORM reads and the package indexers belong.
|
|
129
|
+
|
|
130
|
+
## OpenSearch later
|
|
131
|
+
|
|
132
|
+
OpenSearch is optional and is never started or provisioned by this package:
|
|
133
|
+
|
|
134
|
+
```sh
|
|
135
|
+
python -m pip install -e '.[opensearch]'
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
```env
|
|
139
|
+
SEARCH_PROVIDER=opensearch
|
|
140
|
+
SEARCH_OPENSEARCH_URL=https://search.example.internal
|
|
141
|
+
SEARCH_OPENSEARCH_USERNAME=...
|
|
142
|
+
SEARCH_OPENSEARCH_PASSWORD=...
|
|
143
|
+
SEARCH_ENVIRONMENT=prod
|
|
144
|
+
# Defaults to genorbex-search-prod; override with SEARCH_OPENSEARCH_INDEX if needed.
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
The provider creates an index with text, keyword, timestamp, tenant, and metadata mappings when
|
|
148
|
+
`await service.initialize()` runs. Every query includes a mandatory tenant filter. No embedding
|
|
149
|
+
service or vector index is required; vector/semantic search can be added later without changing
|
|
150
|
+
the `SearchResponse` contract.
|
|
151
|
+
|
|
152
|
+
## Migrations, tests, and observability
|
|
153
|
+
|
|
154
|
+
```sh
|
|
155
|
+
alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
|
|
156
|
+
python -m unittest discover -s packages/genorbex-dynamic-search/tests
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
Set `SEARCH_TEST_DATABASE_URL` to a disposable PostgreSQL database with the migration applied to
|
|
160
|
+
enable integration tests. Unit tests mock OpenSearch. The service emits structured query latency,
|
|
161
|
+
result-count, index failure, retry, and provider-health logs. Optionally inject `SearchMetrics` to
|
|
162
|
+
export query latency/result counts, index failures, and provider health; inject `SearchJobMetrics`
|
|
163
|
+
into the worker to export job retries and failures.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Async Alembic environment for the package-owned search table."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import os
|
|
7
|
+
from logging.config import fileConfig
|
|
8
|
+
|
|
9
|
+
from alembic import context
|
|
10
|
+
from sqlalchemy import Connection, pool
|
|
11
|
+
from sqlalchemy.ext.asyncio import async_engine_from_config
|
|
12
|
+
|
|
13
|
+
config = context.config
|
|
14
|
+
if config.config_file_name is not None:
|
|
15
|
+
fileConfig(config.config_file_name)
|
|
16
|
+
|
|
17
|
+
database_url = os.getenv("SEARCH_DATABASE_URL") or os.getenv("DATABASE_URL")
|
|
18
|
+
if database_url:
|
|
19
|
+
if database_url.startswith("postgres://"):
|
|
20
|
+
database_url = "postgresql+asyncpg://" + database_url.removeprefix("postgres://")
|
|
21
|
+
elif database_url.startswith("postgresql://"):
|
|
22
|
+
database_url = "postgresql+asyncpg://" + database_url.removeprefix("postgresql://")
|
|
23
|
+
config.set_main_option("sqlalchemy.url", database_url.replace("%", "%%"))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def run_migrations_offline() -> None:
|
|
27
|
+
context.configure(
|
|
28
|
+
url=config.get_main_option("sqlalchemy.url"),
|
|
29
|
+
literal_binds=True,
|
|
30
|
+
dialect_opts={"paramstyle": "named"},
|
|
31
|
+
compare_type=True,
|
|
32
|
+
)
|
|
33
|
+
with context.begin_transaction():
|
|
34
|
+
context.run_migrations()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def do_run_migrations(connection: Connection) -> None:
|
|
38
|
+
context.configure(connection=connection, compare_type=True)
|
|
39
|
+
with context.begin_transaction():
|
|
40
|
+
context.run_migrations()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
async def run_async_migrations() -> None:
|
|
44
|
+
connectable = async_engine_from_config(
|
|
45
|
+
config.get_section(config.config_ini_section, {}),
|
|
46
|
+
prefix="sqlalchemy.",
|
|
47
|
+
poolclass=pool.NullPool,
|
|
48
|
+
)
|
|
49
|
+
async with connectable.connect() as connection:
|
|
50
|
+
await connection.run_sync(do_run_migrations)
|
|
51
|
+
await connectable.dispose()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def run_migrations_online() -> None:
|
|
55
|
+
asyncio.run(run_async_migrations())
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if context.is_offline_mode():
|
|
59
|
+
run_migrations_offline()
|
|
60
|
+
else:
|
|
61
|
+
run_migrations_online()
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Create tenant-scoped search documents and weighted full-text index."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from alembic import op
|
|
6
|
+
import sqlalchemy as sa
|
|
7
|
+
from sqlalchemy.dialects import postgresql
|
|
8
|
+
|
|
9
|
+
revision = "0001_search_documents"
|
|
10
|
+
down_revision = None
|
|
11
|
+
branch_labels = None
|
|
12
|
+
depends_on = None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def upgrade() -> None:
|
|
16
|
+
op.create_table(
|
|
17
|
+
"genorbex_search_documents",
|
|
18
|
+
sa.Column("id", sa.Text(), nullable=False),
|
|
19
|
+
sa.Column("tenant_id", sa.Text(), nullable=False),
|
|
20
|
+
sa.Column("project_id", sa.Text(), nullable=True),
|
|
21
|
+
sa.Column("resource_type", sa.Text(), nullable=False),
|
|
22
|
+
sa.Column("title", sa.Text(), nullable=False),
|
|
23
|
+
sa.Column("content", sa.Text(), nullable=False, server_default=""),
|
|
24
|
+
sa.Column("summary", sa.Text(), nullable=True),
|
|
25
|
+
sa.Column("tags", postgresql.ARRAY(sa.Text()), nullable=False, server_default="{}"),
|
|
26
|
+
sa.Column("metadata", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")),
|
|
27
|
+
sa.Column("source_id", sa.Text(), nullable=False),
|
|
28
|
+
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
29
|
+
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
|
30
|
+
sa.Column("created_by", sa.Text(), nullable=True),
|
|
31
|
+
sa.Column(
|
|
32
|
+
"search_vector",
|
|
33
|
+
postgresql.TSVECTOR(),
|
|
34
|
+
sa.Computed(
|
|
35
|
+
"setweight(to_tsvector('english', coalesce(title, '')), 'A') || "
|
|
36
|
+
"setweight(to_tsvector('english', coalesce(summary, '')), 'B') || "
|
|
37
|
+
"setweight(to_tsvector('english', coalesce(content, '')), 'C') || "
|
|
38
|
+
"setweight(to_tsvector('english', coalesce(array_to_string(tags, ' '), '')), 'D')",
|
|
39
|
+
persisted=True,
|
|
40
|
+
),
|
|
41
|
+
nullable=True,
|
|
42
|
+
),
|
|
43
|
+
sa.PrimaryKeyConstraint("id", name="pk_genorbex_search_documents"),
|
|
44
|
+
sa.UniqueConstraint(
|
|
45
|
+
"tenant_id", "resource_type", "source_id", name="uq_genorbex_search_tenant_resource_source"
|
|
46
|
+
),
|
|
47
|
+
sa.CheckConstraint(
|
|
48
|
+
"resource_type IN ('workflow','execution','agent','app','connector','document','knowledge_chunk','audit_event')",
|
|
49
|
+
name="ck_genorbex_search_resource_type",
|
|
50
|
+
),
|
|
51
|
+
)
|
|
52
|
+
op.create_index(
|
|
53
|
+
"ix_genorbex_search_vector",
|
|
54
|
+
"genorbex_search_documents",
|
|
55
|
+
["search_vector"],
|
|
56
|
+
postgresql_using="gin",
|
|
57
|
+
)
|
|
58
|
+
op.create_index(
|
|
59
|
+
"ix_genorbex_search_tenant_created",
|
|
60
|
+
"genorbex_search_documents",
|
|
61
|
+
["tenant_id", "created_at", "id"],
|
|
62
|
+
)
|
|
63
|
+
op.create_index(
|
|
64
|
+
"ix_genorbex_search_tenant_project",
|
|
65
|
+
"genorbex_search_documents",
|
|
66
|
+
["tenant_id", "project_id"],
|
|
67
|
+
)
|
|
68
|
+
op.create_index(
|
|
69
|
+
"ix_genorbex_search_tenant_type_created",
|
|
70
|
+
"genorbex_search_documents",
|
|
71
|
+
["tenant_id", "resource_type", "created_at"],
|
|
72
|
+
)
|
|
73
|
+
op.create_index(
|
|
74
|
+
"ix_genorbex_search_tenant_creator",
|
|
75
|
+
"genorbex_search_documents",
|
|
76
|
+
["tenant_id", "created_by"],
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def downgrade() -> None:
|
|
81
|
+
op.drop_table("genorbex_search_documents")
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[alembic]
|
|
2
|
+
script_location = %(here)s/alembic
|
|
3
|
+
prepend_sys_path = %(here)s/src
|
|
4
|
+
sqlalchemy.url = postgresql+asyncpg://unused:unused@localhost/unused
|
|
5
|
+
|
|
6
|
+
[loggers]
|
|
7
|
+
keys = root,sqlalchemy,alembic
|
|
8
|
+
|
|
9
|
+
[handlers]
|
|
10
|
+
keys = console
|
|
11
|
+
|
|
12
|
+
[formatters]
|
|
13
|
+
keys = generic
|
|
14
|
+
|
|
15
|
+
[logger_root]
|
|
16
|
+
level = WARN
|
|
17
|
+
handlers = console
|
|
18
|
+
qualname =
|
|
19
|
+
|
|
20
|
+
[logger_sqlalchemy]
|
|
21
|
+
level = WARN
|
|
22
|
+
handlers =
|
|
23
|
+
qualname = sqlalchemy.engine
|
|
24
|
+
|
|
25
|
+
[logger_alembic]
|
|
26
|
+
level = INFO
|
|
27
|
+
handlers =
|
|
28
|
+
qualname = alembic
|
|
29
|
+
|
|
30
|
+
[handler_console]
|
|
31
|
+
class = StreamHandler
|
|
32
|
+
args = (sys.stderr,)
|
|
33
|
+
level = NOTSET
|
|
34
|
+
formatter = generic
|
|
35
|
+
|
|
36
|
+
[formatter_generic]
|
|
37
|
+
format = %(levelname)-5.5s [%(name)s] %(message)s
|
|
38
|
+
datefmt = %H:%M:%S
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.24"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "genorbex-dynamic-search"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Tenant-isolated dynamic search service for Genorbex applications"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Genorbex" }]
|
|
13
|
+
keywords = ["genorbex", "search", "postgresql", "opensearch", "multi-tenant"]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"asyncpg>=0.29",
|
|
16
|
+
"fastapi>=0.110",
|
|
17
|
+
"pydantic>=2.7",
|
|
18
|
+
"pydantic-settings>=2.2",
|
|
19
|
+
"sqlalchemy[asyncio]>=2.0",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
opensearch = ["opensearch-py[async]>=2.5"]
|
|
24
|
+
migrations = ["alembic>=1.13"]
|
|
25
|
+
test = ["httpx>=0.27", "pytest>=8", "pytest-asyncio>=0.23", "aiosqlite>=0.20"]
|
|
26
|
+
dev = [
|
|
27
|
+
"alembic>=1.13",
|
|
28
|
+
"httpx>=0.27",
|
|
29
|
+
"mypy>=1.10",
|
|
30
|
+
"opensearch-py[async]>=2.5",
|
|
31
|
+
"pytest>=8",
|
|
32
|
+
"pytest-asyncio>=0.23",
|
|
33
|
+
"ruff>=0.5",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.scripts]
|
|
37
|
+
genorbex-search-backfill = "genorbex_dynamic_search.jobs.backfill:main"
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.wheel]
|
|
40
|
+
packages = ["src/genorbex_dynamic_search"]
|
|
41
|
+
core-metadata-version = "2.4"
|
|
42
|
+
|
|
43
|
+
[tool.hatch.build.targets.sdist]
|
|
44
|
+
core-metadata-version = "2.4"
|
|
45
|
+
|
|
46
|
+
[tool.pytest.ini_options]
|
|
47
|
+
asyncio_mode = "auto"
|
|
48
|
+
testpaths = ["tests"]
|
|
49
|
+
addopts = "-ra"
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
target-version = "py312"
|
|
53
|
+
line-length = 100
|
|
54
|
+
|
|
55
|
+
[tool.ruff.lint]
|
|
56
|
+
select = ["E", "F", "I", "UP", "B"]
|