echoss-db 2.2.0__tar.gz → 2.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {echoss_db-2.2.0/echoss_db.egg-info → echoss_db-2.2.2}/PKG-INFO +34 -5
  2. {echoss_db-2.2.0 → echoss_db-2.2.2}/README.md +32 -3
  3. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/elastic_search.py +3 -1
  4. echoss_db-2.2.2/echoss_db/env_config.py +53 -0
  5. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mysql_query.py +2 -1
  6. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/postgres_query.py +2 -1
  7. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/qdrant_vector.py +26 -3
  8. {echoss_db-2.2.0 → echoss_db-2.2.2/echoss_db.egg-info}/PKG-INFO +34 -5
  9. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db.egg-info/SOURCES.txt +4 -0
  10. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db.egg-info/requires.txt +1 -1
  11. {echoss_db-2.2.0 → echoss_db-2.2.2}/pyproject.toml +2 -2
  12. {echoss_db-2.2.0 → echoss_db-2.2.2}/requirements.txt +0 -2
  13. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/integration/conftest.py +2 -2
  14. echoss_db-2.2.2/tests/integration/test_env_config_integration.py +71 -0
  15. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/integration/test_qdrant_integration.py +63 -0
  16. echoss_db-2.2.2/tests/unit/test_env_config.py +65 -0
  17. echoss_db-2.2.2/tests/unit/test_qdrant_role.py +30 -0
  18. {echoss_db-2.2.0 → echoss_db-2.2.2}/LICENSE +0 -0
  19. {echoss_db-2.2.0 → echoss_db-2.2.2}/MANIFEST.in +0 -0
  20. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/__init__.py +0 -0
  21. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mapping/__init__.py +0 -0
  22. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mapping/dataclass_mapper.py +0 -0
  23. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mapping/errors.py +0 -0
  24. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mapping/param_mapper.py +0 -0
  25. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mapping/pydantic_mapper.py +0 -0
  26. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mapping/row_mapper.py +0 -0
  27. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mapping/types.py +0 -0
  28. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/mongo_query.py +0 -0
  29. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db/sql_transaction.py +0 -0
  30. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db.egg-info/dependency_links.txt +0 -0
  31. {echoss_db-2.2.0 → echoss_db-2.2.2}/echoss_db.egg-info/top_level.txt +0 -0
  32. {echoss_db-2.2.0 → echoss_db-2.2.2}/package_tests/test_package_imports.py +0 -0
  33. {echoss_db-2.2.0 → echoss_db-2.2.2}/setup.cfg +0 -0
  34. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/__init__.py +0 -0
  35. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/integration/__init__.py +0 -0
  36. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/integration/test_postgres_integration.py +0 -0
  37. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/__init__.py +0 -0
  38. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/_postgres_test_schema.py +0 -0
  39. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_elasticsearch.ipynb +0 -0
  40. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_elasticsearch.py +0 -0
  41. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_mapping_mysql.py +0 -0
  42. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_mongo.ipynb +0 -0
  43. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_mongo.py +0 -0
  44. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_mysql.ipynb +0 -0
  45. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_mysql.py +0 -0
  46. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_postgres.py +0 -0
  47. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_postgres_qdrant_ingest.py +0 -0
  48. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/example_qdrant.py +0 -0
  49. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/manual/setup_ai_rag_test_schema.sql +0 -0
  50. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/unit/__init__.py +0 -0
  51. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/unit/mapping/__init__.py +0 -0
  52. {echoss_db-2.2.0 → echoss_db-2.2.2}/tests/unit/mapping/test_param_mapper.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: echoss-db
3
- Version: 2.2.0
3
+ Version: 2.2.2
4
4
  Summary: echoss AI Bigdata Solution - Database Query Package
5
5
  Author-email: ckkim <ckkim@12cm.co.kr>
6
6
  License-Expression: Apache-2.0
@@ -18,9 +18,9 @@ Requires-Dist: sqlalchemy>=2.0.0
18
18
  Requires-Dist: PyMySQL>=1.0.2
19
19
  Requires-Dist: opensearch-py<3.0.0,>=2.8.0
20
20
  Requires-Dist: echoss-fileformat>=1.1.2
21
- Requires-Dist: Pillow<12.0,>=10.3.0
22
21
  Requires-Dist: psycopg[binary]<4.0.0,>=3.2
23
22
  Requires-Dist: qdrant-client[fastembed]<2.0.0,>=1.14.1
23
+ Requires-Dist: python-dotenv>=1.0.0
24
24
  Provides-Extra: mapping
25
25
  Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "mapping"
26
26
  Provides-Extra: mongo
@@ -43,10 +43,15 @@ MySQL, PostgreSQL, OpenSearch, Qdrant compatible query/vector access package
43
43
  `config/config.example.yaml`을 `config/config.yaml`로 복사해 실제 값을 채우세요 (실 config는 gitignore 대상).
44
44
  `config/config.yaml` 기준의 credential 제거 예시는 아래와 같습니다.
45
45
 
46
+ credential 필드(`passwd`, `api_key`, `read_only_api_key`)는 평문 대신 `${VAR_NAME}` 형태로
47
+ 적으면 `.env`(또는 OS 환경변수)에서 값을 읽어 채웁니다. `.env`는 git에 추적되지 않으므로
48
+ `.env.example`을 `.env`로 복사해 실제 값을 채우세요. host/port 등 credential이 아닌 필드는
49
+ 지금처럼 평문 값을 직접 씁니다.
50
+
46
51
  ```yaml
47
52
  mysql:
48
53
  user: <MYSQL_USER>
49
- passwd: <MYSQL_PASSWORD>
54
+ passwd: ${MYSQL_PASSWD}
50
55
  host: <MYSQL_HOST>
51
56
  port: <MYSQL_PORT>
52
57
  db: <MYSQL_DB>
@@ -59,7 +64,7 @@ mongo:
59
64
 
60
65
  elastic:
61
66
  user: <ELASTIC_USER>
62
- passwd: <ELASTIC_PASSWORD>
67
+ passwd: ${ELASTIC_PASSWD}
63
68
  host: <ELASTIC_HOST>
64
69
  port: <ELASTIC_PORT>
65
70
  scheme: <http|https>
@@ -67,7 +72,7 @@ elastic:
67
72
 
68
73
  postgres:
69
74
  user: <POSTGRES_USER>
70
- passwd: <POSTGRES_PASSWORD>
75
+ passwd: ${POSTGRES_PASSWD}
71
76
  host: <POSTGRES_HOST>
72
77
  port: <POSTGRES_PORT>
73
78
  db: <POSTGRES_DB>
@@ -82,8 +87,31 @@ qdrant:
82
87
  default_limit: <DEFAULT_LIMIT>
83
88
  # fastembed_model omitted -> default FastEmbed model automatically applied
84
89
  # fastembed_model: null -> external vector mode (no FastEmbed)
90
+ api_key: ${QDRANT_API_KEY} # optional, admin/전체 권한
91
+ read_only_api_key: ${QDRANT_READ_ONLY_API_KEY} # optional, 조회 전용
92
+ ```
93
+
94
+ `.env` 예시(`.env.example` 참고, 실값은 커밋하지 않음):
95
+
96
+ ```
97
+ MYSQL_PASSWD=
98
+ ELASTIC_PASSWD=
99
+ POSTGRES_PASSWD=
100
+ QDRANT_API_KEY=
101
+ QDRANT_READ_ONLY_API_KEY=
85
102
  ```
86
103
 
104
+ Qdrant는 role로 admin/조회 인스턴스를 나눠 생성할 수 있습니다:
105
+
106
+ ```python
107
+ from echoss_db.qdrant_vector import QdrantVector
108
+
109
+ admin = QdrantVector("config/config.yaml") # role 기본값 "admin" -> qdrant.api_key 사용
110
+ readonly = QdrantVector("config/config.yaml", role="readonly") # qdrant.read_only_api_key 사용
111
+ ```
112
+
113
+ `api_key`/`read_only_api_key`를 둘 다 설정하지 않으면 기존과 동일하게 무인증으로 연결됩니다.
114
+
87
115
  ## Installation
88
116
 
89
117
  ---
@@ -597,3 +625,4 @@ v1.2.3 add postgres DB and qdrant vector storage support
597
625
  v2.0.0 add optional `echoss_db.mapping` (dataclass default, pydantic optional)
598
626
  v2.1.0 raise Python baseline to 3.11, standardize on psycopg3, and keep Qdrant fastembed as default runtime
599
627
  v2.2.0 qdrant generalized dict filter (breaking behavior: previously ignored keys now applied) and get_collection_info, postgres upsert/parse_json options, upsert retry/backoff/batch_size, deprecate MongoQuery (pymongo moved to optional extra [mongo]), raise Python floor to 3.12
628
+ v2.2.2 qdrant admin/read-only API key support (`role` param, `api_key`/`read_only_api_key` config fields), config.yaml credential fields (mysql/postgres/elastic `passwd`, qdrant `api_key`/`read_only_api_key`) now accept `${VAR_NAME}` placeholders resolved from `.env`/OS env (new `python-dotenv` dependency)
@@ -11,10 +11,15 @@ MySQL, PostgreSQL, OpenSearch, Qdrant compatible query/vector access package
11
11
  `config/config.example.yaml`을 `config/config.yaml`로 복사해 실제 값을 채우세요 (실 config는 gitignore 대상).
12
12
  `config/config.yaml` 기준의 credential 제거 예시는 아래와 같습니다.
13
13
 
14
+ credential 필드(`passwd`, `api_key`, `read_only_api_key`)는 평문 대신 `${VAR_NAME}` 형태로
15
+ 적으면 `.env`(또는 OS 환경변수)에서 값을 읽어 채웁니다. `.env`는 git에 추적되지 않으므로
16
+ `.env.example`을 `.env`로 복사해 실제 값을 채우세요. host/port 등 credential이 아닌 필드는
17
+ 지금처럼 평문 값을 직접 씁니다.
18
+
14
19
  ```yaml
15
20
  mysql:
16
21
  user: <MYSQL_USER>
17
- passwd: <MYSQL_PASSWORD>
22
+ passwd: ${MYSQL_PASSWD}
18
23
  host: <MYSQL_HOST>
19
24
  port: <MYSQL_PORT>
20
25
  db: <MYSQL_DB>
@@ -27,7 +32,7 @@ mongo:
27
32
 
28
33
  elastic:
29
34
  user: <ELASTIC_USER>
30
- passwd: <ELASTIC_PASSWORD>
35
+ passwd: ${ELASTIC_PASSWD}
31
36
  host: <ELASTIC_HOST>
32
37
  port: <ELASTIC_PORT>
33
38
  scheme: <http|https>
@@ -35,7 +40,7 @@ elastic:
35
40
 
36
41
  postgres:
37
42
  user: <POSTGRES_USER>
38
- passwd: <POSTGRES_PASSWORD>
43
+ passwd: ${POSTGRES_PASSWD}
39
44
  host: <POSTGRES_HOST>
40
45
  port: <POSTGRES_PORT>
41
46
  db: <POSTGRES_DB>
@@ -50,8 +55,31 @@ qdrant:
50
55
  default_limit: <DEFAULT_LIMIT>
51
56
  # fastembed_model omitted -> default FastEmbed model automatically applied
52
57
  # fastembed_model: null -> external vector mode (no FastEmbed)
58
+ api_key: ${QDRANT_API_KEY} # optional, admin/전체 권한
59
+ read_only_api_key: ${QDRANT_READ_ONLY_API_KEY} # optional, 조회 전용
60
+ ```
61
+
62
+ `.env` 예시(`.env.example` 참고, 실값은 커밋하지 않음):
63
+
64
+ ```
65
+ MYSQL_PASSWD=
66
+ ELASTIC_PASSWD=
67
+ POSTGRES_PASSWD=
68
+ QDRANT_API_KEY=
69
+ QDRANT_READ_ONLY_API_KEY=
53
70
  ```
54
71
 
72
+ Qdrant는 role로 admin/조회 인스턴스를 나눠 생성할 수 있습니다:
73
+
74
+ ```python
75
+ from echoss_db.qdrant_vector import QdrantVector
76
+
77
+ admin = QdrantVector("config/config.yaml") # role 기본값 "admin" -> qdrant.api_key 사용
78
+ readonly = QdrantVector("config/config.yaml", role="readonly") # qdrant.read_only_api_key 사용
79
+ ```
80
+
81
+ `api_key`/`read_only_api_key`를 둘 다 설정하지 않으면 기존과 동일하게 무인증으로 연결됩니다.
82
+
55
83
  ## Installation
56
84
 
57
85
  ---
@@ -565,3 +593,4 @@ v1.2.3 add postgres DB and qdrant vector storage support
565
593
  v2.0.0 add optional `echoss_db.mapping` (dataclass default, pydantic optional)
566
594
  v2.1.0 raise Python baseline to 3.11, standardize on psycopg3, and keep Qdrant fastembed as default runtime
567
595
  v2.2.0 qdrant generalized dict filter (breaking behavior: previously ignored keys now applied) and get_collection_info, postgres upsert/parse_json options, upsert retry/backoff/batch_size, deprecate MongoQuery (pymongo moved to optional extra [mongo]), raise Python floor to 3.12
596
+ v2.2.2 qdrant admin/read-only API key support (`role` param, `api_key`/`read_only_api_key` config fields), config.yaml credential fields (mysql/postgres/elastic `passwd`, qdrant `api_key`/`read_only_api_key`) now accept `${VAR_NAME}` placeholders resolved from `.env`/OS env (new `python-dotenv` dependency)
@@ -6,6 +6,8 @@ from typing import Any, List, Tuple, Dict, Optional, Union
6
6
 
7
7
  from echoss_fileformat import FileUtil, get_logger, set_logger_level
8
8
 
9
+ from .env_config import resolve_credential
10
+
9
11
  logger = get_logger("echoss_query")
10
12
 
11
13
 
@@ -44,7 +46,7 @@ class ElasticSearch:
44
46
  raise TypeError("[Elastic] config info not exist")
45
47
 
46
48
  self.user = es_config.get('user')
47
- self.passwd = es_config.get('passwd')
49
+ self.passwd = resolve_credential(es_config.get('passwd'), section='elastic', field='passwd')
48
50
  self.auth = (self.user, self.passwd) if self.user and self.passwd else None
49
51
 
50
52
  self.host = es_config['host']
@@ -0,0 +1,53 @@
1
+ """PLACEHOLDER 치환 공용 헬퍼.
2
+
3
+ Design Ref: §4.1, §9 — mysql/postgres/elastic/qdrant 4개 DB 모듈이 공통으로 import하는
4
+ credential 치환 로직. config.yaml의 값이 '${VAR_NAME}' 전체 패턴과 일치하면 .env/OS
5
+ 환경변수에서 VAR_NAME을 찾아 치환하고, 아니면 원본 값을 그대로 반환한다(평문 하위 호환).
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ import re
11
+ from typing import Optional
12
+
13
+ from dotenv import find_dotenv, load_dotenv
14
+
15
+ PLACEHOLDER_PATTERN = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$")
16
+
17
+ _dotenv_loaded = False
18
+
19
+
20
+ def _ensure_dotenv_loaded() -> None:
21
+ # Design Ref: §4.1 — 프로세스당 1회만 로드, OS 환경변수가 .env보다 우선(override=False, FR-04).
22
+ global _dotenv_loaded
23
+ if _dotenv_loaded:
24
+ return
25
+ load_dotenv(find_dotenv(usecwd=True), override=False)
26
+ _dotenv_loaded = True
27
+
28
+
29
+ def resolve_credential(value: Optional[str], *, section: str, field: str) -> Optional[str]:
30
+ """value가 '${VAR_NAME}' 전체 일치 패턴이면 .env/OS 환경변수에서 VAR_NAME을 찾아 반환한다.
31
+
32
+ 패턴이 아니면 value를 그대로 반환한다(평문 config 하위 호환). value가 None이거나
33
+ 문자열이 아니면 그대로 반환한다.
34
+
35
+ Raises:
36
+ ValueError: PLACEHOLDER이지만 .env/OS 어디에도 VAR_NAME이 없을 때(fail-closed, FR-06).
37
+ """
38
+ if not isinstance(value, str):
39
+ return value
40
+
41
+ match = PLACEHOLDER_PATTERN.match(value)
42
+ if not match:
43
+ return value
44
+
45
+ _ensure_dotenv_loaded()
46
+ var_name = match.group(1)
47
+ resolved = os.environ.get(var_name)
48
+ if resolved is None:
49
+ raise ValueError(
50
+ f"[{section}.{field}] environment variable '{var_name}' not found "
51
+ f"(.env / OS env). Set it before creating this connection."
52
+ )
53
+ return resolved
@@ -10,6 +10,7 @@ from sqlalchemy.engine import Engine, Connection, Result
10
10
  from sqlalchemy.exc import SQLAlchemyError, DBAPIError
11
11
 
12
12
  from echoss_fileformat import FileUtil, get_logger
13
+ from .env_config import resolve_credential
13
14
  from .sql_transaction import SQLTransaction
14
15
 
15
16
  logger = get_logger("echoss_db")
@@ -143,7 +144,7 @@ class MysqlQuery:
143
144
  if (len(conn_info) > 0) and ('mysql' in conn_info) and all(key in conn_info['mysql'] for key in required_keys):
144
145
  m = conn_info["mysql"]
145
146
  self.user = m['user']
146
- self.passwd = m['passwd']
147
+ self.passwd = resolve_credential(m['passwd'], section='mysql', field='passwd')
147
148
  self.host = m['host']
148
149
  self.port = m.get('port', 3306)
149
150
  self.db = m['db']
@@ -11,6 +11,7 @@ from sqlalchemy.engine import Engine, Connection, Result
11
11
  from sqlalchemy.exc import SQLAlchemyError
12
12
 
13
13
  from echoss_fileformat import FileUtil, get_logger
14
+ from .env_config import resolve_credential
14
15
  from .sql_transaction import SQLTransaction
15
16
 
16
17
  logger = get_logger("echoss_db")
@@ -107,7 +108,7 @@ class PostgresQuery:
107
108
  if (len(conn_info) > 0) and ('postgres' in conn_info) and all(k in conn_info['postgres'] for k in required_keys):
108
109
  p = conn_info["postgres"]
109
110
  self.user = p['user']
110
- self.passwd = p['passwd']
111
+ self.passwd = resolve_credential(p['passwd'], section='postgres', field='passwd')
111
112
  self.host = p['host']
112
113
  self.port = p.get('port', 5432)
113
114
  self.db = p['db']
@@ -10,6 +10,8 @@ logger = get_logger("echoss_query")
10
10
  from qdrant_client import QdrantClient
11
11
  from qdrant_client.http import models as qm
12
12
 
13
+ from .env_config import resolve_credential
14
+
13
15
 
14
16
  class QdrantVector:
15
17
  conn = None
@@ -17,7 +19,7 @@ class QdrantVector:
17
19
  default_limit = 10
18
20
  default_fastembed_model = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
19
21
 
20
- def __init__(self, conn_info: Union[str, dict]):
22
+ def __init__(self, conn_info: Union[str, dict], role: str = "admin"):
21
23
  """
22
24
  Args:
23
25
  conn_info:
@@ -28,9 +30,13 @@ class QdrantVector:
28
30
  'scheme': 'http',
29
31
  'collection': 'ai_rag_chunks',
30
32
  'timeout': 5,
31
- 'default_limit': 10
33
+ 'default_limit': 10,
34
+ 'api_key': '${QDRANT_API_KEY}', # admin, optional
35
+ 'read_only_api_key': '${QDRANT_READ_ONLY_API_KEY}', # readonly, optional
32
36
  }
33
37
  }
38
+ role: "admin"(기본값) 또는 "readonly". config의 qdrant.api_key/read_only_api_key
39
+ 중 role에 대응하는 필드를 연결에 사용한다. Design Ref: §4.2.
34
40
  """
35
41
  if isinstance(conn_info, str):
36
42
  conn_info = FileUtil.dict_load(conn_info)
@@ -43,6 +49,23 @@ class QdrantVector:
43
49
  else:
44
50
  raise TypeError("[Qdrant] config info not exist")
45
51
 
52
+ if role not in ("admin", "readonly"):
53
+ raise ValueError(f"QdrantVector role must be 'admin' or 'readonly', got '{role}'")
54
+ self.role = role
55
+
56
+ # Design Ref: §4.2 — role에 대응하는 키 필드가 없고 다른 role의 키만 있으면 에러(FR-03).
57
+ # 둘 다 없으면 기존과 동일하게 무인증 연결(하위 호환).
58
+ key_field = "api_key" if role == "admin" else "read_only_api_key"
59
+ other_field = "read_only_api_key" if role == "admin" else "api_key"
60
+ if key_field in q_config:
61
+ self.api_key = resolve_credential(q_config[key_field], section="qdrant", field=key_field)
62
+ elif other_field in q_config:
63
+ raise ValueError(
64
+ f"role='{role}' requires qdrant.{key_field}, but only qdrant.{other_field} is configured"
65
+ )
66
+ else:
67
+ self.api_key = None
68
+
46
69
  self.host = q_config['host']
47
70
  self.port = q_config.get('port', 6333)
48
71
  self.scheme = q_config.get('scheme', 'http')
@@ -82,7 +105,7 @@ class QdrantVector:
82
105
 
83
106
  def connect(self):
84
107
  try:
85
- cli = QdrantClient(url=self.url, timeout=self.timeout)
108
+ cli = QdrantClient(url=self.url, api_key=self.api_key, timeout=self.timeout)
86
109
  # 간단 ping: collections 호출
87
110
  cli.get_collections()
88
111
  return cli
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: echoss-db
3
- Version: 2.2.0
3
+ Version: 2.2.2
4
4
  Summary: echoss AI Bigdata Solution - Database Query Package
5
5
  Author-email: ckkim <ckkim@12cm.co.kr>
6
6
  License-Expression: Apache-2.0
@@ -18,9 +18,9 @@ Requires-Dist: sqlalchemy>=2.0.0
18
18
  Requires-Dist: PyMySQL>=1.0.2
19
19
  Requires-Dist: opensearch-py<3.0.0,>=2.8.0
20
20
  Requires-Dist: echoss-fileformat>=1.1.2
21
- Requires-Dist: Pillow<12.0,>=10.3.0
22
21
  Requires-Dist: psycopg[binary]<4.0.0,>=3.2
23
22
  Requires-Dist: qdrant-client[fastembed]<2.0.0,>=1.14.1
23
+ Requires-Dist: python-dotenv>=1.0.0
24
24
  Provides-Extra: mapping
25
25
  Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "mapping"
26
26
  Provides-Extra: mongo
@@ -43,10 +43,15 @@ MySQL, PostgreSQL, OpenSearch, Qdrant compatible query/vector access package
43
43
  `config/config.example.yaml`을 `config/config.yaml`로 복사해 실제 값을 채우세요 (실 config는 gitignore 대상).
44
44
  `config/config.yaml` 기준의 credential 제거 예시는 아래와 같습니다.
45
45
 
46
+ credential 필드(`passwd`, `api_key`, `read_only_api_key`)는 평문 대신 `${VAR_NAME}` 형태로
47
+ 적으면 `.env`(또는 OS 환경변수)에서 값을 읽어 채웁니다. `.env`는 git에 추적되지 않으므로
48
+ `.env.example`을 `.env`로 복사해 실제 값을 채우세요. host/port 등 credential이 아닌 필드는
49
+ 지금처럼 평문 값을 직접 씁니다.
50
+
46
51
  ```yaml
47
52
  mysql:
48
53
  user: <MYSQL_USER>
49
- passwd: <MYSQL_PASSWORD>
54
+ passwd: ${MYSQL_PASSWD}
50
55
  host: <MYSQL_HOST>
51
56
  port: <MYSQL_PORT>
52
57
  db: <MYSQL_DB>
@@ -59,7 +64,7 @@ mongo:
59
64
 
60
65
  elastic:
61
66
  user: <ELASTIC_USER>
62
- passwd: <ELASTIC_PASSWORD>
67
+ passwd: ${ELASTIC_PASSWD}
63
68
  host: <ELASTIC_HOST>
64
69
  port: <ELASTIC_PORT>
65
70
  scheme: <http|https>
@@ -67,7 +72,7 @@ elastic:
67
72
 
68
73
  postgres:
69
74
  user: <POSTGRES_USER>
70
- passwd: <POSTGRES_PASSWORD>
75
+ passwd: ${POSTGRES_PASSWD}
71
76
  host: <POSTGRES_HOST>
72
77
  port: <POSTGRES_PORT>
73
78
  db: <POSTGRES_DB>
@@ -82,8 +87,31 @@ qdrant:
82
87
  default_limit: <DEFAULT_LIMIT>
83
88
  # fastembed_model omitted -> default FastEmbed model automatically applied
84
89
  # fastembed_model: null -> external vector mode (no FastEmbed)
90
+ api_key: ${QDRANT_API_KEY} # optional, admin/전체 권한
91
+ read_only_api_key: ${QDRANT_READ_ONLY_API_KEY} # optional, 조회 전용
92
+ ```
93
+
94
+ `.env` 예시(`.env.example` 참고, 실값은 커밋하지 않음):
95
+
96
+ ```
97
+ MYSQL_PASSWD=
98
+ ELASTIC_PASSWD=
99
+ POSTGRES_PASSWD=
100
+ QDRANT_API_KEY=
101
+ QDRANT_READ_ONLY_API_KEY=
85
102
  ```
86
103
 
104
+ Qdrant는 role로 admin/조회 인스턴스를 나눠 생성할 수 있습니다:
105
+
106
+ ```python
107
+ from echoss_db.qdrant_vector import QdrantVector
108
+
109
+ admin = QdrantVector("config/config.yaml") # role 기본값 "admin" -> qdrant.api_key 사용
110
+ readonly = QdrantVector("config/config.yaml", role="readonly") # qdrant.read_only_api_key 사용
111
+ ```
112
+
113
+ `api_key`/`read_only_api_key`를 둘 다 설정하지 않으면 기존과 동일하게 무인증으로 연결됩니다.
114
+
87
115
  ## Installation
88
116
 
89
117
  ---
@@ -597,3 +625,4 @@ v1.2.3 add postgres DB and qdrant vector storage support
597
625
  v2.0.0 add optional `echoss_db.mapping` (dataclass default, pydantic optional)
598
626
  v2.1.0 raise Python baseline to 3.11, standardize on psycopg3, and keep Qdrant fastembed as default runtime
599
627
  v2.2.0 qdrant generalized dict filter (breaking behavior: previously ignored keys now applied) and get_collection_info, postgres upsert/parse_json options, upsert retry/backoff/batch_size, deprecate MongoQuery (pymongo moved to optional extra [mongo]), raise Python floor to 3.12
628
+ v2.2.2 qdrant admin/read-only API key support (`role` param, `api_key`/`read_only_api_key` config fields), config.yaml credential fields (mysql/postgres/elastic `passwd`, qdrant `api_key`/`read_only_api_key`) now accept `${VAR_NAME}` placeholders resolved from `.env`/OS env (new `python-dotenv` dependency)
@@ -5,6 +5,7 @@ pyproject.toml
5
5
  requirements.txt
6
6
  echoss_db/__init__.py
7
7
  echoss_db/elastic_search.py
8
+ echoss_db/env_config.py
8
9
  echoss_db/mongo_query.py
9
10
  echoss_db/mysql_query.py
10
11
  echoss_db/postgres_query.py
@@ -26,6 +27,7 @@ package_tests/test_package_imports.py
26
27
  tests/__init__.py
27
28
  tests/integration/__init__.py
28
29
  tests/integration/conftest.py
30
+ tests/integration/test_env_config_integration.py
29
31
  tests/integration/test_postgres_integration.py
30
32
  tests/integration/test_qdrant_integration.py
31
33
  tests/manual/__init__.py
@@ -42,5 +44,7 @@ tests/manual/example_postgres_qdrant_ingest.py
42
44
  tests/manual/example_qdrant.py
43
45
  tests/manual/setup_ai_rag_test_schema.sql
44
46
  tests/unit/__init__.py
47
+ tests/unit/test_env_config.py
48
+ tests/unit/test_qdrant_role.py
45
49
  tests/unit/mapping/__init__.py
46
50
  tests/unit/mapping/test_param_mapper.py
@@ -3,9 +3,9 @@ sqlalchemy>=2.0.0
3
3
  PyMySQL>=1.0.2
4
4
  opensearch-py<3.0.0,>=2.8.0
5
5
  echoss-fileformat>=1.1.2
6
- Pillow<12.0,>=10.3.0
7
6
  psycopg[binary]<4.0.0,>=3.2
8
7
  qdrant-client[fastembed]<2.0.0,>=1.14.1
8
+ python-dotenv>=1.0.0
9
9
 
10
10
  [all]
11
11
  pydantic<3.0.0,>=2.0.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "echoss-db"
7
- version = "2.2.0"
7
+ version = "2.2.2"
8
8
  description = "echoss AI Bigdata Solution - Database Query Package"
9
9
  readme = { file = "README.md", content-type = "text/markdown" }
10
10
  license = "Apache-2.0"
@@ -25,9 +25,9 @@ dependencies = [
25
25
  "PyMySQL>=1.0.2",
26
26
  "opensearch-py>=2.8.0,<3.0.0",
27
27
  "echoss-fileformat>=1.1.2",
28
- "Pillow>=10.3.0,<12.0",
29
28
  "psycopg[binary]>=3.2,<4.0.0",
30
29
  "qdrant-client[fastembed]>=1.14.1,<2.0.0",
30
+ "python-dotenv>=1.0.0",
31
31
  ]
32
32
 
33
33
  [project.optional-dependencies]
@@ -1,9 +1,7 @@
1
1
  pandas>=1.5.3
2
- pymongo>=4.3.3
3
2
  PyMySQL>=1.0.2
4
3
  sqlalchemy>=2.0.0
5
4
  opensearch-py>=2.8.0,<3.0.0
6
5
  echoss-fileformat>=1.1.2
7
- Pillow>=10.3.0,<12.0
8
6
  qdrant-client[fastembed]>=1.14.1,<2.0.0
9
7
  psycopg[binary]>=3.2,<4.0.0
@@ -23,8 +23,8 @@ from echoss_db.qdrant_vector import QdrantVector
23
23
  PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
24
24
  CONFIG_PATH = os.path.join(PROJECT_ROOT, "config", "config_dev.yaml")
25
25
 
26
- POSTGRES_TEST_SCHEMA = "ai_rag_test"
27
- QDRANT_TEST_COLLECTION = "echoss_it_vectors"
26
+ POSTGRES_TEST_SCHEMA = "ai_push_rag_test"
27
+ QDRANT_TEST_COLLECTION = "ai_push_chunks_test"
28
28
 
29
29
 
30
30
  def load_config_section(section: str) -> dict:
@@ -0,0 +1,71 @@
1
+ """Real Postgres integration tests for env_config PLACEHOLDER credential resolution.
2
+
3
+ Verifies that a config.yaml credential field using '${VAR_NAME}' connects exactly like a
4
+ plaintext credential, using the real config_dev.yaml postgres section as the source of
5
+ truth for actual credentials (no fake DB, per [[no-fake-db-tests]]).
6
+ Run: pytest -m integration tests/integration/test_env_config_integration.py
7
+
8
+ Design Ref: §8.4. Both tests run an actual SELECT — PostgresQuery.connect_db() only
9
+ builds a lazy SQLAlchemy Engine and does not validate credentials at construction time,
10
+ so asserting on `client.engine` alone would not prove a real, correctly-authenticated
11
+ connection.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import pytest
16
+
17
+ from echoss_db.env_config import resolve_credential
18
+ from echoss_db.postgres_query import PostgresQuery
19
+
20
+ from .conftest import POSTGRES_TEST_SCHEMA, load_config_section
21
+
22
+ pytestmark = pytest.mark.integration
23
+
24
+
25
+ def test_postgres_connects_with_placeholder_passwd():
26
+ # config_dev.yaml의 postgres.passwd는 실제로 "${AI_PUSH_PG_PASSWD}" PLACEHOLDER다.
27
+ # .env/OS env에 실값이 없으면 이 시나리오 자체를 검증할 수 없으므로 skip한다.
28
+ pg_cfg = dict(load_config_section("postgres"))
29
+ pg_cfg["schema"] = POSTGRES_TEST_SCHEMA
30
+
31
+ try:
32
+ client = PostgresQuery({"postgres": pg_cfg})
33
+ except ValueError as e:
34
+ pytest.skip(f"PLACEHOLDER env var not set — cannot verify: {e}")
35
+
36
+ try:
37
+ df = client.select("SELECT 1 AS ok")
38
+ assert df.loc[0, "ok"] == 1
39
+ finally:
40
+ client.close()
41
+
42
+
43
+ def test_postgres_still_connects_with_plaintext_passwd():
44
+ # 기존 평문 config 하위 호환 회귀 확인 (FR-07) — 실 패스워드를 평문으로 config에 직접
45
+ # 넣어도 동작해야 한다. config_dev.yaml 자체는 이제 PLACEHOLDER를 쓰므로, 진짜 비밀값은
46
+ # resolve_credential로 한 번 해석해서(값을 출력/로그에 남기지 않고) 평문으로 재사용한다.
47
+ pg_cfg = dict(load_config_section("postgres"))
48
+ try:
49
+ real_passwd = resolve_credential(pg_cfg["passwd"], section="postgres", field="passwd")
50
+ except ValueError as e:
51
+ pytest.skip(f"cannot resolve real postgres passwd for plaintext regression test: {e}")
52
+
53
+ pg_cfg["passwd"] = real_passwd # 평문 — PLACEHOLDER 아님
54
+ pg_cfg["schema"] = POSTGRES_TEST_SCHEMA
55
+
56
+ client = PostgresQuery({"postgres": pg_cfg})
57
+ try:
58
+ df = client.select("SELECT 1 AS ok")
59
+ assert df.loc[0, "ok"] == 1
60
+ finally:
61
+ client.close()
62
+
63
+
64
+ def test_missing_placeholder_env_var_raises(monkeypatch):
65
+ pg_cfg = dict(load_config_section("postgres"))
66
+ monkeypatch.delenv("ECHOSS_IT_NONEXISTENT_VAR", raising=False)
67
+ pg_cfg["passwd"] = "${ECHOSS_IT_NONEXISTENT_VAR}"
68
+ pg_cfg["schema"] = POSTGRES_TEST_SCHEMA
69
+
70
+ with pytest.raises(ValueError, match="ECHOSS_IT_NONEXISTENT_VAR"):
71
+ PostgresQuery({"postgres": pg_cfg})
@@ -3,11 +3,18 @@
3
3
  Exercises actual create_collection -> upsert -> get_collection_info -> search(filter)
4
4
  -> delete against a live Qdrant in external (precomputed) vector mode.
5
5
  Run: pytest -m integration tests/integration/test_qdrant_integration.py
6
+
7
+ Design Ref: §8.3 — role-based API key scenarios are skipped when config_dev.yaml's
8
+ qdrant section has no api_key/read_only_api_key (this dev instance has no auth enabled).
6
9
  """
7
10
  from __future__ import annotations
8
11
 
9
12
  import pytest
10
13
 
14
+ from echoss_db.qdrant_vector import QdrantVector
15
+
16
+ from .conftest import QDRANT_TEST_COLLECTION, load_config_section
17
+
11
18
  pytestmark = pytest.mark.integration
12
19
 
13
20
  DIM = 8
@@ -64,3 +71,59 @@ def test_search_passthrough_raw_filter(qv):
64
71
  raw = qm.Filter(must_not=[qm.FieldCondition(key="doc_type", match=qm.MatchValue(value="b"))])
65
72
  hits = qv.search_vector(_vec(0.1), filter_=raw, limit=10)
66
73
  assert {h.id for h in hits} == {1}
74
+
75
+
76
+ def _role_test_config():
77
+ q_cfg = dict(load_config_section("qdrant"))
78
+ if "api_key" not in q_cfg and "read_only_api_key" not in q_cfg:
79
+ pytest.skip(
80
+ "qdrant.api_key/read_only_api_key not configured in config_dev.yaml — "
81
+ "this dev Qdrant instance has no API key auth enabled"
82
+ )
83
+ q_cfg["collection"] = QDRANT_TEST_COLLECTION
84
+ q_cfg["fastembed_model"] = None
85
+ q_cfg.setdefault("timeout", 30)
86
+ return q_cfg
87
+
88
+
89
+ def test_admin_role_connects_and_can_write():
90
+ q_cfg = _role_test_config()
91
+ client = QdrantVector({"qdrant": q_cfg}, role="admin")
92
+ try:
93
+ client.delete_collection()
94
+ client.create_collection(dim=DIM)
95
+ client.upsert_vectors([{"id": 1, "vector": _vec(0.1)}], wait=True)
96
+ assert client.get_collection_info()["points_count"] >= 1
97
+ finally:
98
+ try:
99
+ client.delete_collection()
100
+ except Exception:
101
+ pass
102
+ client.close()
103
+
104
+
105
+ def test_readonly_role_connects_and_can_search_but_not_write():
106
+ q_cfg = _role_test_config()
107
+ if "read_only_api_key" not in q_cfg:
108
+ pytest.skip("qdrant.read_only_api_key not configured in config_dev.yaml")
109
+
110
+ admin = QdrantVector(dict({"qdrant": q_cfg}), role="admin")
111
+ try:
112
+ admin.delete_collection()
113
+ admin.create_collection(dim=DIM)
114
+ admin.upsert_vectors([{"id": 1, "vector": _vec(0.1)}], wait=True)
115
+
116
+ readonly = QdrantVector({"qdrant": q_cfg}, role="readonly")
117
+ try:
118
+ hits = readonly.search_vector(_vec(0.1), limit=10)
119
+ assert {h.id for h in hits} == {1}
120
+ with pytest.raises(Exception):
121
+ readonly.create_collection(dim=DIM, collection="echoss_it_readonly_should_fail")
122
+ finally:
123
+ readonly.close()
124
+ finally:
125
+ try:
126
+ admin.delete_collection()
127
+ except Exception:
128
+ pass
129
+ admin.close()
@@ -0,0 +1,65 @@
1
+ """Pure function tests for echoss_db.env_config.resolve_credential.
2
+
3
+ No DB involved — this is a pure function (PLACEHOLDER regex + env var lookup), so unit
4
+ testing it here is not a fake-DB test (see [[no-fake-db-tests]]).
5
+ Design Ref: §8.2.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import pytest
10
+
11
+ from echoss_db import env_config
12
+ from echoss_db.env_config import resolve_credential
13
+
14
+
15
+ @pytest.fixture(autouse=True)
16
+ def reset_dotenv_cache(monkeypatch):
17
+ # _dotenv_loaded is a process-wide singleton flag; reset per test for isolation.
18
+ monkeypatch.setattr(env_config, "_dotenv_loaded", False)
19
+
20
+
21
+ def test_resolves_from_os_env(monkeypatch):
22
+ monkeypatch.setenv("MYSQL_PASSWD", "secret")
23
+ monkeypatch.setattr(env_config, "find_dotenv", lambda usecwd=True: "")
24
+ assert resolve_credential("${MYSQL_PASSWD}", section="mysql", field="passwd") == "secret"
25
+
26
+
27
+ def test_os_env_overrides_dotenv(monkeypatch, tmp_path):
28
+ dotenv_file = tmp_path / ".env"
29
+ dotenv_file.write_text("MYSQL_PASSWD=fromdotenv\n")
30
+ monkeypatch.setattr(env_config, "find_dotenv", lambda usecwd=True: str(dotenv_file))
31
+ monkeypatch.setenv("MYSQL_PASSWD", "fromos")
32
+
33
+ assert resolve_credential("${MYSQL_PASSWD}", section="mysql", field="passwd") == "fromos"
34
+
35
+
36
+ def test_loads_from_dotenv_when_os_env_absent(monkeypatch, tmp_path):
37
+ dotenv_file = tmp_path / ".env"
38
+ dotenv_file.write_text("MYSQL_PASSWD=fromdotenv\n")
39
+ monkeypatch.delenv("MYSQL_PASSWD", raising=False)
40
+ monkeypatch.setattr(env_config, "find_dotenv", lambda usecwd=True: str(dotenv_file))
41
+
42
+ assert resolve_credential("${MYSQL_PASSWD}", section="mysql", field="passwd") == "fromdotenv"
43
+
44
+
45
+ def test_missing_variable_raises(monkeypatch):
46
+ monkeypatch.delenv("MYSQL_PASSWD", raising=False)
47
+ monkeypatch.setattr(env_config, "find_dotenv", lambda usecwd=True: "")
48
+
49
+ with pytest.raises(ValueError, match="MYSQL_PASSWD"):
50
+ resolve_credential("${MYSQL_PASSWD}", section="mysql", field="passwd")
51
+
52
+
53
+ def test_plaintext_passthrough():
54
+ assert resolve_credential("plaintext-password", section="mysql", field="passwd") == "plaintext-password"
55
+
56
+
57
+ def test_none_passthrough():
58
+ assert resolve_credential(None, section="mysql", field="passwd") is None
59
+
60
+
61
+ def test_partial_placeholder_not_substituted(monkeypatch):
62
+ monkeypatch.setenv("VAR", "value")
63
+ value = "prefix-${VAR}-suffix"
64
+
65
+ assert resolve_credential(value, section="mysql", field="passwd") == value
@@ -0,0 +1,30 @@
1
+ """Unit tests for QdrantVector role/api_key validation logic.
2
+
3
+ These raise before any network connection is attempted (see qdrant_vector.py:__init__),
4
+ so they need no live Qdrant server — not a fake-DB test, just constructor logic.
5
+ Design Ref: §4.2, §8.3 (client-side portion).
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import pytest
10
+
11
+ from echoss_db.qdrant_vector import QdrantVector
12
+
13
+
14
+ def test_invalid_role_raises():
15
+ conn_info = {"qdrant": {"host": "localhost", "port": 6333}}
16
+ with pytest.raises(ValueError, match="role must be 'admin' or 'readonly'"):
17
+ QdrantVector(conn_info, role="superadmin")
18
+
19
+
20
+ def test_readonly_role_requires_read_only_api_key_field():
21
+ # api_key(admin)만 있고 read_only_api_key가 없는 상태에서 role="readonly" 요청 (FR-03).
22
+ conn_info = {"qdrant": {"host": "localhost", "port": 6333, "api_key": "admin-key"}}
23
+ with pytest.raises(ValueError, match="read_only_api_key"):
24
+ QdrantVector(conn_info, role="readonly")
25
+
26
+
27
+ def test_admin_role_requires_api_key_field_when_only_readonly_configured():
28
+ conn_info = {"qdrant": {"host": "localhost", "port": 6333, "read_only_api_key": "ro-key"}}
29
+ with pytest.raises(ValueError, match="qdrant.api_key"):
30
+ QdrantVector(conn_info, role="admin")
File without changes
File without changes
File without changes
File without changes