genorbex-dynamic-search 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. genorbex_dynamic_search-0.1.0/.gitignore +84 -0
  2. genorbex_dynamic_search-0.1.0/PKG-INFO +195 -0
  3. genorbex_dynamic_search-0.1.0/README.md +163 -0
  4. genorbex_dynamic_search-0.1.0/alembic/env.py +61 -0
  5. genorbex_dynamic_search-0.1.0/alembic/versions/0001_search_documents.py +81 -0
  6. genorbex_dynamic_search-0.1.0/alembic.ini +38 -0
  7. genorbex_dynamic_search-0.1.0/pyproject.toml +56 -0
  8. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/__init__.py +25 -0
  9. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/api/__init__.py +2 -0
  10. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/api/router.py +201 -0
  11. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/config.py +57 -0
  12. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/exceptions.py +21 -0
  13. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/__init__.py +20 -0
  14. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/agents.py +17 -0
  15. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/apps.py +18 -0
  16. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/audit_events.py +19 -0
  17. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/base.py +139 -0
  18. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/connectors.py +17 -0
  19. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/documents.py +31 -0
  20. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/executions.py +19 -0
  21. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/indexers/workflows.py +13 -0
  22. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/jobs/__init__.py +2 -0
  23. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/jobs/backfill.py +121 -0
  24. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/jobs/queue.py +96 -0
  25. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/models.py +190 -0
  26. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/__init__.py +2 -0
  27. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/base.py +27 -0
  28. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/opensearch.py +265 -0
  29. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/providers/postgres.py +297 -0
  30. genorbex_dynamic_search-0.1.0/src/genorbex_dynamic_search/service.py +175 -0
  31. genorbex_dynamic_search-0.1.0/tests/test_api_and_jobs.py +75 -0
  32. genorbex_dynamic_search-0.1.0/tests/test_contracts.py +78 -0
  33. genorbex_dynamic_search-0.1.0/tests/test_opensearch.py +108 -0
  34. genorbex_dynamic_search-0.1.0/tests/test_postgres_integration.py +77 -0
  35. genorbex_dynamic_search-0.1.0/tests/test_postgres_provider.py +152 -0
  36. genorbex_dynamic_search-0.1.0/tests/test_service_and_indexers.py +114 -0
@@ -0,0 +1,84 @@
1
+ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2
+
3
+ # dependencies
4
+ node_modules/
5
+ /.pnp
6
+ .pnp.*
7
+ .yarn/*
8
+ !.yarn/patches
9
+ !.yarn/plugins
10
+ !.yarn/releases
11
+ !.yarn/versions
12
+
13
+ # testing
14
+ /coverage
15
+
16
+ # next.js
17
+ /.next/
18
+ /.next-*/
19
+ /out/
20
+
21
+ # production
22
+ /build
23
+ /desktop-app
24
+ /desktop-dist
25
+ /public/downloads/*.dmg
26
+ /public/downloads/*.zip
27
+ /public/downloads/*.blockmap
28
+
29
+ # AI Code Builder preview
30
+ /.preview
31
+ /.streamlit-runtime/
32
+
33
+ # AI Code Builder persistent projects (local development only)
34
+ /ai-code-projects
35
+
36
+ # misc
37
+ .DS_Store
38
+ *.pem
39
+
40
+ # debug
41
+ npm-debug.log*
42
+ yarn-debug.log*
43
+ yarn-error.log*
44
+ .pnpm-debug.log*
45
+
46
+ # build logs
47
+ build.log
48
+ @build.log
49
+
50
+ # env files (can opt-in for committing if needed)
51
+ .env*
52
+ !.env.orbexdb-pg-wire.example
53
+
54
+ # vercel
55
+ .vercel
56
+
57
+ # typescript
58
+ *.tsbuildinfo
59
+ next-env.d.ts
60
+
61
+ /src/generated/prisma
62
+ .env.local
63
+ .env
64
+ .env.development
65
+ .env.production
66
+ .env.test
67
+ .env.development.local
68
+ .env.production.local
69
+ .env.test.local
70
+ ..env.production.backup.
71
+
72
+ # AI Code Builder deployments
73
+ /deployments/
74
+
75
+ # local Prisma dev state
76
+ /.prisma-home
77
+ /.emka-storage-local
78
+ /.orbex-whatsapp-web/
79
+
80
+ # zlog
81
+ zlog
82
+
83
+ # local Genorbex security SDK reports
84
+ /sdks/python-security/reports/
@@ -0,0 +1,195 @@
1
+ Metadata-Version: 2.4
2
+ Name: genorbex-dynamic-search
3
+ Version: 0.1.0
4
+ Summary: Tenant-isolated dynamic search service for Genorbex applications
5
+ Author: Genorbex
6
+ License: MIT
7
+ Keywords: genorbex,multi-tenant,opensearch,postgresql,search
8
+ Requires-Python: >=3.12
9
+ Requires-Dist: asyncpg>=0.29
10
+ Requires-Dist: fastapi>=0.110
11
+ Requires-Dist: pydantic-settings>=2.2
12
+ Requires-Dist: pydantic>=2.7
13
+ Requires-Dist: sqlalchemy[asyncio]>=2.0
14
+ Provides-Extra: dev
15
+ Requires-Dist: alembic>=1.13; extra == 'dev'
16
+ Requires-Dist: httpx>=0.27; extra == 'dev'
17
+ Requires-Dist: mypy>=1.10; extra == 'dev'
18
+ Requires-Dist: opensearch-py[async]>=2.5; extra == 'dev'
19
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
20
+ Requires-Dist: pytest>=8; extra == 'dev'
21
+ Requires-Dist: ruff>=0.5; extra == 'dev'
22
+ Provides-Extra: migrations
23
+ Requires-Dist: alembic>=1.13; extra == 'migrations'
24
+ Provides-Extra: opensearch
25
+ Requires-Dist: opensearch-py[async]>=2.5; extra == 'opensearch'
26
+ Provides-Extra: test
27
+ Requires-Dist: aiosqlite>=0.20; extra == 'test'
28
+ Requires-Dist: httpx>=0.27; extra == 'test'
29
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
30
+ Requires-Dist: pytest>=8; extra == 'test'
31
+ Description-Content-Type: text/markdown
32
+
33
+ # genorbex-dynamic-search
34
+
35
+ An async, multi-tenant search service package for Genorbex resources. PostgreSQL full-text
36
+ search is the default provider; OpenSearch is an optional adapter with the same response model.
37
+ The package is intentionally an installable service library, not a new web deployment or
38
+ database infrastructure definition.
39
+
40
+ ## Install and configure
41
+
42
+ Python 3.12 or newer is required.
43
+
44
+ ```sh
45
+ python -m pip install -e '.[migrations,test]'
46
+ ```
47
+
48
+ PostgreSQL is the default. Use the application database or a dedicated PostgreSQL database and
49
+ run the package migration before mounting the API:
50
+
51
+ ```env
52
+ SEARCH_PROVIDER=postgres
53
+ SEARCH_DATABASE_URL=postgresql://user:password@host:5432/genorbex
54
+ SEARCH_ENVIRONMENT=dev
55
+ SEARCH_QUERY_RATE_LIMIT=60
56
+ SEARCH_QUERY_RATE_WINDOW_SECONDS=60
57
+ ```
58
+
59
+ ```sh
60
+ alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
61
+ ```
62
+
63
+ The migration is package-owned Alembic SQL. This repository's main web application currently
64
+ uses Prisma migrations and has no Python FastAPI host. Keep migration ownership explicit: either
65
+ run this Alembic migration as part of the Python-service release or port the SQL into the host's
66
+ Prisma migration workflow before sharing deployment ownership of the table.
67
+
68
+ ## Use the service
69
+
70
+ Application code depends on `DynamicSearchService`, not provider classes:
71
+
72
+ ```python
73
+ from datetime import datetime, timezone
74
+
75
+ from genorbex_dynamic_search import ResourceType, SearchDocument, SearchRequest
76
+ from genorbex_dynamic_search.config import SearchSettings
77
+ from genorbex_dynamic_search.service import create_search_service
78
+
79
+ service = create_search_service(SearchSettings())
80
+ await service.initialize()
81
+ await service.index(SearchDocument(
82
+ id="workflow:flow-123",
83
+ tenant_id="org-123", # trusted organization ID; use user ID for personal tenants
84
+ resource_type=ResourceType.WORKFLOW,
85
+ title="Customer onboarding",
86
+ content="Verify the account and provision the customer workspace.",
87
+ source_id="flow-123",
88
+ created_at=datetime.now(timezone.utc),
89
+ updated_at=datetime.now(timezone.utc),
90
+ ))
91
+ response = await service.search(SearchRequest(tenant_id="org-123", query="onboarding"))
92
+ await service.aclose()
93
+ ```
94
+
95
+ `SearchRequest` supports resource type, project, tag, creator, date, sort, cursor, and page-size
96
+ filters. Results include ranked hits, PostgreSQL/OpenSearch highlights, type facets, total count,
97
+ latency, and an opaque `next_cursor`. Cursors are tied to their sort order; pass the same sort
98
+ on the next request. Treat highlight strings as untrusted display text and escape them in HTML.
99
+
100
+ ## FastAPI integration and authentication
101
+
102
+ ```python
103
+ from fastapi import FastAPI
104
+ from genorbex_dynamic_search.api.router import create_search_router
105
+ from genorbex_dynamic_search.service import create_search_service
106
+
107
+ app = FastAPI()
108
+ search_service = create_search_service()
109
+ app.include_router(create_search_router(search_service, identity_dependency=genorbex_identity))
110
+ ```
111
+
112
+ `genorbex_identity` must adapt the authenticated principal established by the host's middleware
113
+ to `SearchIdentity(user_id=..., tenant_id=..., role=..., permissions=...)`. The default adapter
114
+ reads `request.state.user` or `request.state.current_user`; it fails closed if no trusted identity
115
+ is present and uses `organizationId` as the tenant, falling back to the authenticated user ID for
116
+ personal workspaces. The package never reads a tenant from query parameters or request headers.
117
+ Manual index/delete endpoints require an admin/owner role or an explicit search-admin permission.
118
+ The GET endpoint applies a configurable per-tenant rate limit and max query length. Its built-in
119
+ limiter is process-local; inject a shared limiter implementation for multi-worker deployments.
120
+
121
+ The current Genorbex web application is Next.js/Prisma, not FastAPI, so its authentication
122
+ middleware cannot be imported directly into a Python process. Mount the router only in a host
123
+ that supplies that trusted identity adapter.
124
+
125
+ ## Resource indexing and queue lifecycle
126
+
127
+ Indexers are provided for workflows, executions, agents, generated apps, connectors, documents,
128
+ knowledge chunks, and audit events. Call `enqueue_upsert` after create/update and `enqueue_delete`
129
+ after delete, passing the host's durable `SearchJobQueue` adapter. That adapter's `enqueue` method
130
+ must persist the serialized job before returning. A worker deserializes `SearchIndexJob` and calls
131
+ `process_search_job(service, job)`; failures use bounded exponential retries and are logged with
132
+ structured context. Upserts and deletes are idempotent.
133
+
134
+ The repository has a Prisma-backed message queue in the TypeScript application, but no Python
135
+ consumer/adapter contract exists yet. The package therefore defines the queue protocol and worker
136
+ handler without creating another queue, Redis service, or cloud resource. Connect the protocol to
137
+ the existing queue consumer at the application boundary.
138
+
139
+ Indexers deliberately avoid workflow definitions, prompts, execution input/output blobs, connector
140
+ settings, and credential fields. SearchDocument additionally redacts common credential assignments
141
+ and JWT-shaped strings, and metadata is filtered through an explicit allowlist. Review extracted
142
+ document text before indexing if it may contain regulated or confidential data.
143
+
144
+ ## Backfill
145
+
146
+ The CLI requires exactly one scope. `--all-tenants` is an explicit opt-in and never the default:
147
+
148
+ ```sh
149
+ GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
150
+ python -m genorbex_dynamic_search.jobs.backfill --tenant-id org-123
151
+
152
+ GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
153
+ python -m genorbex_dynamic_search.jobs.backfill --all-tenants
154
+ ```
155
+
156
+ The source factory returns an adapter implementing `authorize_backfill(tenant_id, all_tenants)`,
157
+ `list_tenant_ids()`, and async `iter_documents(tenant_id)`. The authorization method must reject
158
+ operators who are not allowed to backfill the requested scope; it runs before any tenant records
159
+ are fetched. Every yielded document is checked against the tenant being backfilled. This adapter is
160
+ where host-specific Prisma/ORM reads and the package indexers belong.
161
+
162
+ ## OpenSearch later
163
+
164
+ OpenSearch is optional and is never started or provisioned by this package:
165
+
166
+ ```sh
167
+ python -m pip install -e '.[opensearch]'
168
+ ```
169
+
170
+ ```env
171
+ SEARCH_PROVIDER=opensearch
172
+ SEARCH_OPENSEARCH_URL=https://search.example.internal
173
+ SEARCH_OPENSEARCH_USERNAME=...
174
+ SEARCH_OPENSEARCH_PASSWORD=...
175
+ SEARCH_ENVIRONMENT=prod
176
+ # Defaults to genorbex-search-prod; override with SEARCH_OPENSEARCH_INDEX if needed.
177
+ ```
178
+
179
+ The provider creates an index with text, keyword, timestamp, tenant, and metadata mappings when
180
+ `await service.initialize()` runs. Every query includes a mandatory tenant filter. No embedding
181
+ service or vector index is required; vector/semantic search can be added later without changing
182
+ the `SearchResponse` contract.
183
+
184
+ ## Migrations, tests, and observability
185
+
186
+ ```sh
187
+ alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
188
+ python -m unittest discover -s packages/genorbex-dynamic-search/tests
189
+ ```
190
+
191
+ Set `SEARCH_TEST_DATABASE_URL` to a disposable PostgreSQL database with the migration applied to
192
+ enable integration tests. Unit tests mock OpenSearch. The service emits structured query latency,
193
+ result-count, index failure, retry, and provider-health logs. Optionally inject `SearchMetrics` to
194
+ export query latency/result counts, index failures, and provider health; inject `SearchJobMetrics`
195
+ into the worker to export job retries and failures.
@@ -0,0 +1,163 @@
1
+ # genorbex-dynamic-search
2
+
3
+ An async, multi-tenant search service package for Genorbex resources. PostgreSQL full-text
4
+ search is the default provider; OpenSearch is an optional adapter with the same response model.
5
+ The package is intentionally an installable service library, not a new web deployment or
6
+ database infrastructure definition.
7
+
8
+ ## Install and configure
9
+
10
+ Python 3.12 or newer is required.
11
+
12
+ ```sh
13
+ python -m pip install -e '.[migrations,test]'
14
+ ```
15
+
16
+ PostgreSQL is the default. Use the application database or a dedicated PostgreSQL database and
17
+ run the package migration before mounting the API:
18
+
19
+ ```env
20
+ SEARCH_PROVIDER=postgres
21
+ SEARCH_DATABASE_URL=postgresql://user:password@host:5432/genorbex
22
+ SEARCH_ENVIRONMENT=dev
23
+ SEARCH_QUERY_RATE_LIMIT=60
24
+ SEARCH_QUERY_RATE_WINDOW_SECONDS=60
25
+ ```
26
+
27
+ ```sh
28
+ alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
29
+ ```
30
+
31
+ The migration is package-owned Alembic SQL. This repository's main web application currently
32
+ uses Prisma migrations and has no Python FastAPI host. Keep migration ownership explicit: either
33
+ run this Alembic migration as part of the Python-service release or port the SQL into the host's
34
+ Prisma migration workflow before sharing deployment ownership of the table.
35
+
36
+ ## Use the service
37
+
38
+ Application code depends on `DynamicSearchService`, not provider classes:
39
+
40
+ ```python
41
+ from datetime import datetime, timezone
42
+
43
+ from genorbex_dynamic_search import ResourceType, SearchDocument, SearchRequest
44
+ from genorbex_dynamic_search.config import SearchSettings
45
+ from genorbex_dynamic_search.service import create_search_service
46
+
47
+ service = create_search_service(SearchSettings())
48
+ await service.initialize()
49
+ await service.index(SearchDocument(
50
+ id="workflow:flow-123",
51
+ tenant_id="org-123", # trusted organization ID; use user ID for personal tenants
52
+ resource_type=ResourceType.WORKFLOW,
53
+ title="Customer onboarding",
54
+ content="Verify the account and provision the customer workspace.",
55
+ source_id="flow-123",
56
+ created_at=datetime.now(timezone.utc),
57
+ updated_at=datetime.now(timezone.utc),
58
+ ))
59
+ response = await service.search(SearchRequest(tenant_id="org-123", query="onboarding"))
60
+ await service.aclose()
61
+ ```
62
+
63
+ `SearchRequest` supports resource type, project, tag, creator, date, sort, cursor, and page-size
64
+ filters. Results include ranked hits, PostgreSQL/OpenSearch highlights, type facets, total count,
65
+ latency, and an opaque `next_cursor`. Cursors are tied to their sort order; pass the same sort
66
+ on the next request. Treat highlight strings as untrusted display text and escape them in HTML.
67
+
68
+ ## FastAPI integration and authentication
69
+
70
+ ```python
71
+ from fastapi import FastAPI
72
+ from genorbex_dynamic_search.api.router import create_search_router
73
+ from genorbex_dynamic_search.service import create_search_service
74
+
75
+ app = FastAPI()
76
+ search_service = create_search_service()
77
+ app.include_router(create_search_router(search_service, identity_dependency=genorbex_identity))
78
+ ```
79
+
80
+ `genorbex_identity` must adapt the authenticated principal established by the host's middleware
81
+ to `SearchIdentity(user_id=..., tenant_id=..., role=..., permissions=...)`. The default adapter
82
+ reads `request.state.user` or `request.state.current_user`; it fails closed if no trusted identity
83
+ is present and uses `organizationId` as the tenant, falling back to the authenticated user ID for
84
+ personal workspaces. The package never reads a tenant from query parameters or request headers.
85
+ Manual index/delete endpoints require an admin/owner role or an explicit search-admin permission.
86
+ The GET endpoint applies a configurable per-tenant rate limit and max query length. Its built-in
87
+ limiter is process-local; inject a shared limiter implementation for multi-worker deployments.
88
+
89
+ The current Genorbex web application is Next.js/Prisma, not FastAPI, so its authentication
90
+ middleware cannot be imported directly into a Python process. Mount the router only in a host
91
+ that supplies that trusted identity adapter.
92
+
93
+ ## Resource indexing and queue lifecycle
94
+
95
+ Indexers are provided for workflows, executions, agents, generated apps, connectors, documents,
96
+ knowledge chunks, and audit events. Call `enqueue_upsert` after create/update and `enqueue_delete`
97
+ after delete, passing the host's durable `SearchJobQueue` adapter. That adapter's `enqueue` method
98
+ must persist the serialized job before returning. A worker deserializes `SearchIndexJob` and calls
99
+ `process_search_job(service, job)`; failures use bounded exponential retries and are logged with
100
+ structured context. Upserts and deletes are idempotent.
101
+
102
+ The repository has a Prisma-backed message queue in the TypeScript application, but no Python
103
+ consumer/adapter contract exists yet. The package therefore defines the queue protocol and worker
104
+ handler without creating another queue, Redis service, or cloud resource. Connect the protocol to
105
+ the existing queue consumer at the application boundary.
106
+
107
+ Indexers deliberately avoid workflow definitions, prompts, execution input/output blobs, connector
108
+ settings, and credential fields. SearchDocument additionally redacts common credential assignments
109
+ and JWT-shaped strings, and metadata is filtered through an explicit allowlist. Review extracted
110
+ document text before indexing if it may contain regulated or confidential data.
111
+
112
+ ## Backfill
113
+
114
+ The CLI requires exactly one scope. `--all-tenants` is an explicit opt-in and never the default:
115
+
116
+ ```sh
117
+ GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
118
+ python -m genorbex_dynamic_search.jobs.backfill --tenant-id org-123
119
+
120
+ GENORBEX_SEARCH_BACKFILL_SOURCE=my_app.search_backfill:create_source \
121
+ python -m genorbex_dynamic_search.jobs.backfill --all-tenants
122
+ ```
123
+
124
+ The source factory returns an adapter implementing `authorize_backfill(tenant_id, all_tenants)`,
125
+ `list_tenant_ids()`, and async `iter_documents(tenant_id)`. The authorization method must reject
126
+ operators who are not allowed to backfill the requested scope; it runs before any tenant records
127
+ are fetched. Every yielded document is checked against the tenant being backfilled. This adapter is
128
+ where host-specific Prisma/ORM reads and the package indexers belong.
129
+
130
+ ## OpenSearch later
131
+
132
+ OpenSearch is optional and is never started or provisioned by this package:
133
+
134
+ ```sh
135
+ python -m pip install -e '.[opensearch]'
136
+ ```
137
+
138
+ ```env
139
+ SEARCH_PROVIDER=opensearch
140
+ SEARCH_OPENSEARCH_URL=https://search.example.internal
141
+ SEARCH_OPENSEARCH_USERNAME=...
142
+ SEARCH_OPENSEARCH_PASSWORD=...
143
+ SEARCH_ENVIRONMENT=prod
144
+ # Defaults to genorbex-search-prod; override with SEARCH_OPENSEARCH_INDEX if needed.
145
+ ```
146
+
147
+ The provider creates an index with text, keyword, timestamp, tenant, and metadata mappings when
148
+ `await service.initialize()` runs. Every query includes a mandatory tenant filter. No embedding
149
+ service or vector index is required; vector/semantic search can be added later without changing
150
+ the `SearchResponse` contract.
151
+
152
+ ## Migrations, tests, and observability
153
+
154
+ ```sh
155
+ alembic -c packages/genorbex-dynamic-search/alembic.ini upgrade head
156
+ python -m unittest discover -s packages/genorbex-dynamic-search/tests
157
+ ```
158
+
159
+ Set `SEARCH_TEST_DATABASE_URL` to a disposable PostgreSQL database with the migration applied to
160
+ enable integration tests. Unit tests mock OpenSearch. The service emits structured query latency,
161
+ result-count, index failure, retry, and provider-health logs. Optionally inject `SearchMetrics` to
162
+ export query latency/result counts, index failures, and provider health; inject `SearchJobMetrics`
163
+ into the worker to export job retries and failures.
@@ -0,0 +1,61 @@
1
+ """Async Alembic environment for the package-owned search table."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import os
7
+ from logging.config import fileConfig
8
+
9
+ from alembic import context
10
+ from sqlalchemy import Connection, pool
11
+ from sqlalchemy.ext.asyncio import async_engine_from_config
12
+
13
+ config = context.config
14
+ if config.config_file_name is not None:
15
+ fileConfig(config.config_file_name)
16
+
17
+ database_url = os.getenv("SEARCH_DATABASE_URL") or os.getenv("DATABASE_URL")
18
+ if database_url:
19
+ if database_url.startswith("postgres://"):
20
+ database_url = "postgresql+asyncpg://" + database_url.removeprefix("postgres://")
21
+ elif database_url.startswith("postgresql://"):
22
+ database_url = "postgresql+asyncpg://" + database_url.removeprefix("postgresql://")
23
+ config.set_main_option("sqlalchemy.url", database_url.replace("%", "%%"))
24
+
25
+
26
+ def run_migrations_offline() -> None:
27
+ context.configure(
28
+ url=config.get_main_option("sqlalchemy.url"),
29
+ literal_binds=True,
30
+ dialect_opts={"paramstyle": "named"},
31
+ compare_type=True,
32
+ )
33
+ with context.begin_transaction():
34
+ context.run_migrations()
35
+
36
+
37
+ def do_run_migrations(connection: Connection) -> None:
38
+ context.configure(connection=connection, compare_type=True)
39
+ with context.begin_transaction():
40
+ context.run_migrations()
41
+
42
+
43
+ async def run_async_migrations() -> None:
44
+ connectable = async_engine_from_config(
45
+ config.get_section(config.config_ini_section, {}),
46
+ prefix="sqlalchemy.",
47
+ poolclass=pool.NullPool,
48
+ )
49
+ async with connectable.connect() as connection:
50
+ await connection.run_sync(do_run_migrations)
51
+ await connectable.dispose()
52
+
53
+
54
+ def run_migrations_online() -> None:
55
+ asyncio.run(run_async_migrations())
56
+
57
+
58
+ if context.is_offline_mode():
59
+ run_migrations_offline()
60
+ else:
61
+ run_migrations_online()
@@ -0,0 +1,81 @@
1
+ """Create tenant-scoped search documents and weighted full-text index."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from alembic import op
6
+ import sqlalchemy as sa
7
+ from sqlalchemy.dialects import postgresql
8
+
9
+ revision = "0001_search_documents"
10
+ down_revision = None
11
+ branch_labels = None
12
+ depends_on = None
13
+
14
+
15
+ def upgrade() -> None:
16
+ op.create_table(
17
+ "genorbex_search_documents",
18
+ sa.Column("id", sa.Text(), nullable=False),
19
+ sa.Column("tenant_id", sa.Text(), nullable=False),
20
+ sa.Column("project_id", sa.Text(), nullable=True),
21
+ sa.Column("resource_type", sa.Text(), nullable=False),
22
+ sa.Column("title", sa.Text(), nullable=False),
23
+ sa.Column("content", sa.Text(), nullable=False, server_default=""),
24
+ sa.Column("summary", sa.Text(), nullable=True),
25
+ sa.Column("tags", postgresql.ARRAY(sa.Text()), nullable=False, server_default="{}"),
26
+ sa.Column("metadata", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")),
27
+ sa.Column("source_id", sa.Text(), nullable=False),
28
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
29
+ sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
30
+ sa.Column("created_by", sa.Text(), nullable=True),
31
+ sa.Column(
32
+ "search_vector",
33
+ postgresql.TSVECTOR(),
34
+ sa.Computed(
35
+ "setweight(to_tsvector('english', coalesce(title, '')), 'A') || "
36
+ "setweight(to_tsvector('english', coalesce(summary, '')), 'B') || "
37
+ "setweight(to_tsvector('english', coalesce(content, '')), 'C') || "
38
+ "setweight(to_tsvector('english', coalesce(array_to_string(tags, ' '), '')), 'D')",
39
+ persisted=True,
40
+ ),
41
+ nullable=True,
42
+ ),
43
+ sa.PrimaryKeyConstraint("id", name="pk_genorbex_search_documents"),
44
+ sa.UniqueConstraint(
45
+ "tenant_id", "resource_type", "source_id", name="uq_genorbex_search_tenant_resource_source"
46
+ ),
47
+ sa.CheckConstraint(
48
+ "resource_type IN ('workflow','execution','agent','app','connector','document','knowledge_chunk','audit_event')",
49
+ name="ck_genorbex_search_resource_type",
50
+ ),
51
+ )
52
+ op.create_index(
53
+ "ix_genorbex_search_vector",
54
+ "genorbex_search_documents",
55
+ ["search_vector"],
56
+ postgresql_using="gin",
57
+ )
58
+ op.create_index(
59
+ "ix_genorbex_search_tenant_created",
60
+ "genorbex_search_documents",
61
+ ["tenant_id", "created_at", "id"],
62
+ )
63
+ op.create_index(
64
+ "ix_genorbex_search_tenant_project",
65
+ "genorbex_search_documents",
66
+ ["tenant_id", "project_id"],
67
+ )
68
+ op.create_index(
69
+ "ix_genorbex_search_tenant_type_created",
70
+ "genorbex_search_documents",
71
+ ["tenant_id", "resource_type", "created_at"],
72
+ )
73
+ op.create_index(
74
+ "ix_genorbex_search_tenant_creator",
75
+ "genorbex_search_documents",
76
+ ["tenant_id", "created_by"],
77
+ )
78
+
79
+
80
+ def downgrade() -> None:
81
+ op.drop_table("genorbex_search_documents")
@@ -0,0 +1,38 @@
1
+ [alembic]
2
+ script_location = %(here)s/alembic
3
+ prepend_sys_path = %(here)s/src
4
+ sqlalchemy.url = postgresql+asyncpg://unused:unused@localhost/unused
5
+
6
+ [loggers]
7
+ keys = root,sqlalchemy,alembic
8
+
9
+ [handlers]
10
+ keys = console
11
+
12
+ [formatters]
13
+ keys = generic
14
+
15
+ [logger_root]
16
+ level = WARN
17
+ handlers = console
18
+ qualname =
19
+
20
+ [logger_sqlalchemy]
21
+ level = WARN
22
+ handlers =
23
+ qualname = sqlalchemy.engine
24
+
25
+ [logger_alembic]
26
+ level = INFO
27
+ handlers =
28
+ qualname = alembic
29
+
30
+ [handler_console]
31
+ class = StreamHandler
32
+ args = (sys.stderr,)
33
+ level = NOTSET
34
+ formatter = generic
35
+
36
+ [formatter_generic]
37
+ format = %(levelname)-5.5s [%(name)s] %(message)s
38
+ datefmt = %H:%M:%S
@@ -0,0 +1,56 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.24"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "genorbex-dynamic-search"
7
+ version = "0.1.0"
8
+ description = "Tenant-isolated dynamic search service for Genorbex applications"
9
+ readme = "README.md"
10
+ requires-python = ">=3.12"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Genorbex" }]
13
+ keywords = ["genorbex", "search", "postgresql", "opensearch", "multi-tenant"]
14
+ dependencies = [
15
+ "asyncpg>=0.29",
16
+ "fastapi>=0.110",
17
+ "pydantic>=2.7",
18
+ "pydantic-settings>=2.2",
19
+ "sqlalchemy[asyncio]>=2.0",
20
+ ]
21
+
22
+ [project.optional-dependencies]
23
+ opensearch = ["opensearch-py[async]>=2.5"]
24
+ migrations = ["alembic>=1.13"]
25
+ test = ["httpx>=0.27", "pytest>=8", "pytest-asyncio>=0.23", "aiosqlite>=0.20"]
26
+ dev = [
27
+ "alembic>=1.13",
28
+ "httpx>=0.27",
29
+ "mypy>=1.10",
30
+ "opensearch-py[async]>=2.5",
31
+ "pytest>=8",
32
+ "pytest-asyncio>=0.23",
33
+ "ruff>=0.5",
34
+ ]
35
+
36
+ [project.scripts]
37
+ genorbex-search-backfill = "genorbex_dynamic_search.jobs.backfill:main"
38
+
39
+ [tool.hatch.build.targets.wheel]
40
+ packages = ["src/genorbex_dynamic_search"]
41
+ core-metadata-version = "2.4"
42
+
43
+ [tool.hatch.build.targets.sdist]
44
+ core-metadata-version = "2.4"
45
+
46
+ [tool.pytest.ini_options]
47
+ asyncio_mode = "auto"
48
+ testpaths = ["tests"]
49
+ addopts = "-ra"
50
+
51
+ [tool.ruff]
52
+ target-version = "py312"
53
+ line-length = 100
54
+
55
+ [tool.ruff.lint]
56
+ select = ["E", "F", "I", "UP", "B"]