watchmen-collector-kernel 16.5.15__tar.gz → 16.5.17__tar.gz

{watchmen_collector_kernel-16.5.15 → watchmen_collector_kernel-16.5.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: watchmen-collector-kernel
-Version: 16.5.15
+Version: 16.5.17
 Summary: 
 License: MIT
 Author: botlikes
@@ -19,11 +19,11 @@ Provides-Extra: oss
 Provides-Extra: postgresql
 Provides-Extra: s3
 Requires-Dist: numpy (>=1.23.3,<2.0.0)
-Requires-Dist: watchmen-data-kernel (==16.5.15)
-Requires-Dist: watchmen-storage-mongodb (==16.5.15) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==16.5.15) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==16.5.15) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==16.5.15) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==16.5.15) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==16.5.15) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==16.5.15) ; extra == "s3"
+Requires-Dist: watchmen-data-kernel (==16.5.17)
+Requires-Dist: watchmen-storage-mongodb (==16.5.17) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==16.5.17) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==16.5.17) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==16.5.17) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==16.5.17) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==16.5.17) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==16.5.17) ; extra == "s3"

{watchmen_collector_kernel-16.5.15 → watchmen_collector_kernel-16.5.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "watchmen-collector-kernel"
-version = "16.5.15"
+version = "16.5.17"
 description = ""
 authors = ["botlikes <75356972+botlikes456@users.noreply.github.com>"]
 license = "MIT"
@@ -11,14 +11,14 @@ packages = [
 [tool.poetry.dependencies]
 python = "^3.9"
 numpy = "^1.23.3"
-watchmen-data-kernel = "16.5.15"
-watchmen-storage-mysql = { version = "16.5.15", optional = true }
-watchmen-storage-oracle = { version = "16.5.15", optional = true }
-watchmen-storage-mongodb = { version = "16.5.15", optional = true }
-watchmen-storage-mssql = { version = "16.5.15", optional = true }
-watchmen-storage-postgresql = { version = "16.5.15", optional = true }
-watchmen-storage-oss = { version = "16.5.15", optional = true }
-watchmen-storage-s3 = { version = "16.5.15", optional = true }
+watchmen-data-kernel = "16.5.17"
+watchmen-storage-mysql = { version = "16.5.17", optional = true }
+watchmen-storage-oracle = { version = "16.5.17", optional = true }
+watchmen-storage-mongodb = { version = "16.5.17", optional = true }
+watchmen-storage-mssql = { version = "16.5.17", optional = true }
+watchmen-storage-postgresql = { version = "16.5.17", optional = true }
+watchmen-storage-oss = { version = "16.5.17", optional = true }
+watchmen-storage-s3 = { version = "16.5.17", optional = true }
 
 [tool.poetry.dev-dependencies]
 

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/cache/__init__.py

@@ -0,0 +1 @@
+from .collector_cache_service import CollectorCacheService

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/cache/collector_cache_manger.py

@@ -0,0 +1,55 @@
+from cacheout import Cache
+from watchmen_data_kernel.cache.cache_manager import cache_set
+
+from watchmen_data_kernel.cache import configure_cache
+
+
+class ModelConfigByNameCache(Cache):
+	pass
+
+
+class TableConfigByNameCache(Cache):
+	pass
+
+
+class TableConfigByTableNameCache(Cache):
+	pass
+
+
+class TableConfigsByParentNameCache(Cache):
+	pass
+
+
+configure_cache('MODEL_CONFIG_BY_NAME',
+                {'cache_class': ModelConfigByNameCache, 'maxsize': 512})
+
+
+configure_cache('TABLE_CONFIG_BY_NAME',
+                {'cache_class': TableConfigByNameCache, 'maxsize': 512})
+
+
+configure_cache('TABLE_CONFIG_BY_TABLE_NAME',
+                {'cache_class': TableConfigByTableNameCache, 'maxsize': 512})
+
+
+configure_cache('TABLE_CONFIGS_BY_PARENT_NAME',
+                {'cache_class': TableConfigsByParentNameCache, 'maxsize': 512})
+
+
+def get_model_config_by_name_cache() -> ModelConfigByNameCache:
+	return cache_set['MODEL_CONFIG_BY_NAME']
+
+
+def get_table_config_by_name_cache() -> TableConfigByNameCache:
+	return cache_set['TABLE_CONFIG_BY_NAME']
+
+
+def get_table_config_by_table_name_cache() -> TableConfigByTableNameCache:
+	return cache_set['TABLE_CONFIG_BY_TABLE_NAME']
+
+
+def get_table_configs_by_parent_name_cache() -> TableConfigsByParentNameCache:
+	return cache_set['TABLE_CONFIGS_BY_PARENT_NAME']
+
+
+

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/cache/collector_cache_service.py

@@ -0,0 +1,18 @@
+from .model_config_cache import ModelConfigCache, model_config_cache
+from .table_config_cache import TableConfigCache, table_config_cache
+
+
+class CollectorCacheService:
+
+	@staticmethod
+	def model_config() -> ModelConfigCache:
+		return model_config_cache
+
+	@staticmethod
+	def table_config() -> TableConfigCache:
+		return table_config_cache
+
+	@staticmethod
+	def clear_all() -> None:
+		CollectorCacheService.model_config().clear()
+		CollectorCacheService.table_config().clear()

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/cache/model_config_cache.py

@@ -0,0 +1,45 @@
+from typing import List, Optional
+
+from watchmen_collector_kernel.model import CollectorModelConfig
+
+from watchmen_data_kernel.cache import InternalCache
+
+from .collector_cache_manger import get_model_config_by_name_cache
+from watchmen_collector_kernel.common import ask_collector_config_cache_enabled
+
+
+class ModelConfigCache:
+
+	def __init__(self):
+		self.ByNameCache = InternalCache(cache=get_model_config_by_name_cache)
+
+	def put(self, model_config: CollectorModelConfig) -> Optional[CollectorModelConfig]:
+		if ask_collector_config_cache_enabled():
+			self.ByNameCache.remove(model_config.modelName)
+			# refresh other caches
+			existing_config = self.ByNameCache.put(model_config.modelName, model_config)
+			return existing_config
+		else:
+			return None
+
+	def get(self, model_name: str) -> Optional[CollectorModelConfig]:
+		if ask_collector_config_cache_enabled():
+			return self.ByNameCache.get(model_name)
+		else:
+			return None
+
+	def remove(self, model_name: str) -> Optional[CollectorModelConfig]:
+		if ask_collector_config_cache_enabled():
+			existing: Optional[CollectorModelConfig] = self.ByNameCache.remove(model_name)
+			return existing
+		else:
+			return None
+
+	def all(self) -> List[CollectorModelConfig]:
+		return list(self.ByNameCache.values())
+
+	def clear(self) -> None:
+		self.ByNameCache.clear()
+
+
+model_config_cache = ModelConfigCache()

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/cache/table_config_cache.py

@@ -0,0 +1,89 @@
+from typing import List, Optional, Dict
+
+from watchmen_collector_kernel.model import CollectorTableConfig
+
+from watchmen_data_kernel.cache import InternalCache
+
+from .collector_cache_manger import get_table_config_by_name_cache, \
+	get_table_config_by_table_name_cache, get_table_configs_by_parent_name_cache
+from watchmen_collector_kernel.common import ask_collector_config_cache_enabled
+
+
+class TableConfigCache:
+
+	def __init__(self):
+		self.ByNameCache = InternalCache(cache=get_table_config_by_name_cache)
+		self.ByTableNameCache = InternalCache(cache=get_table_config_by_table_name_cache)
+		self.ByParentNameCache = InternalCache(cache=get_table_configs_by_parent_name_cache)
+
+	def put_config_by_name(self, table_config: CollectorTableConfig) -> Optional[CollectorTableConfig]:
+		if ask_collector_config_cache_enabled():
+			self.ByNameCache.remove(table_config.name)
+			existing_config = self.ByTableNameCache.put(table_config.name, table_config)
+			return existing_config
+		else:
+			return None
+
+	def get_config_by_name(self, name: str) -> Optional[CollectorTableConfig]:
+		if ask_collector_config_cache_enabled():
+			return self.ByNameCache.get(name)
+		else:
+			return None
+
+	def remove_config_by_name(self, name: str) -> Optional[CollectorTableConfig]:
+		if ask_collector_config_cache_enabled():
+			existing: Optional[CollectorTableConfig] = self.ByNameCache.remove(name)
+			return existing
+		else:
+			return None
+
+	def put_config_by_table_name(self, table_config: CollectorTableConfig) -> Optional[CollectorTableConfig]:
+		if ask_collector_config_cache_enabled():
+			self.ByTableNameCache.remove(table_config.tableName)
+			existing_config = self.ByTableNameCache.put(table_config.tableName, table_config)
+			return existing_config
+		else:
+			return None
+
+	def get_config_by_table_name(self, table_name: str) -> Optional[CollectorTableConfig]:
+		if ask_collector_config_cache_enabled():
+			return self.ByTableNameCache.get(table_name)
+		else:
+			return None
+
+	def remove_config_by_table_name(self, table_name: str) -> Optional[CollectorTableConfig]:
+		if ask_collector_config_cache_enabled():
+			existing: Optional[CollectorTableConfig] = self.ByTableNameCache.remove(table_name)
+			return existing
+		else:
+			return None
+
+	def put_configs_by_parent_name(self, parent_name: str,
+	                               configs: List[CollectorTableConfig]) -> Optional[List[CollectorTableConfig]]:
+		if ask_collector_config_cache_enabled():
+			self.ByParentNameCache.remove(parent_name)
+			existing_configs = self.ByParentNameCache.put(parent_name, configs)
+			return existing_configs
+		else:
+			return None
+
+	def get_configs_by_parent_name(self, parent_name: str) -> Optional[List[CollectorTableConfig]]:
+		if ask_collector_config_cache_enabled():
+			return self.ByParentNameCache.get(parent_name)
+		else:
+			return None
+
+	def remove_configs_by_parent_name(self, parent_name: str) -> Optional[List[CollectorTableConfig]]:
+		if ask_collector_config_cache_enabled():
+			existing_configs: Optional[List[CollectorTableConfig]] = self.ByParentNameCache.remove(parent_name)
+			return existing_configs
+		else:
+			return None
+
+	def clear(self) -> None:
+		self.ByNameCache.clear()
+		self.ByTableNameCache.clear()
+		self.ByParentNameCache.clear()
+
+
+table_config_cache = TableConfigCache()

{watchmen_collector_kernel-16.5.15 → watchmen_collector_kernel-16.5.17}/src/watchmen_collector_kernel/common/__init__.py

@@ -1,3 +1,5 @@
 from .constants import CHANGE_RECORD_ID, TENANT_ID, IS_MERGED, LEFT_BRACE, RIGHT_BRACE, COMMA, \
 	IS_POSTED, CHANGE_JSON_ID, WAVE, IS_FINISHED, IS_EXTRACTED, MODEL_TRIGGER_ID
-from .settings import ask_lock_clean_interval, ask_lock_clean_timeout, ask_partial_size
+from .settings import ask_lock_clean_interval, ask_lock_clean_timeout, ask_partial_size, \
+	ask_collector_config_cache_enabled
+from .exception import CollectorKernelException

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/common/exception.py

@@ -0,0 +1,2 @@
+class CollectorKernelException(Exception):
+	pass

{watchmen_collector_kernel-16.5.15 → watchmen_collector_kernel-16.5.17}/src/watchmen_collector_kernel/common/settings.py

@@ -9,6 +9,8 @@ class CollectorSettings(BaseSettings):
 	LOCK_CLEAN_TIMEOUT: int = 3600
 	PARTIAL_SIZE: int = 100
 
+	COLLECTOR_CONFIG_CACHE_ENABLED: bool = True  # enable collector config cache heart beat
+
 	class Config:
 		# secrets_dir = '/var/run'
 		env_file = '.env'
@@ -30,3 +32,7 @@ def ask_lock_clean_timeout() -> int:
 
 def ask_partial_size() -> int:
 	return collector_settings.PARTIAL_SIZE
+
+
+def ask_collector_config_cache_enabled() -> bool:
+	return collector_settings.COLLECTOR_CONFIG_CACHE_ENABLED

{watchmen_collector_kernel-16.5.15 → watchmen_collector_kernel-16.5.17}/src/watchmen_collector_kernel/service/__init__.py

@@ -6,3 +6,4 @@ from .extract_source import SourceTableExtractor
 from .task_service import get_task_service
 from .criteria_builder import CriteriaBuilder
 from .extract_utils import build_audit_column_criteria, cal_array2d_diff, build_data_id, get_data_id
+from .table_config_service import get_table_config_service

{watchmen_collector_kernel-16.5.15 → watchmen_collector_kernel-16.5.17}/src/watchmen_collector_kernel/service/data_capture.py

@@ -3,6 +3,7 @@ from typing import Optional, Dict, Any, Tuple
 
 from watchmen_auth import PrincipalService
 from watchmen_collector_kernel.model import CollectorTableConfig
+from .table_config_service import get_table_config_service
 from .extract_source import SourceTableExtractor
 from watchmen_collector_kernel.storage import get_collector_table_config_service
 from watchmen_storage import TransactionalStorageSPI, SnowflakeGenerator
@@ -23,6 +24,7 @@ class DataCaptureService:
 		self.collector_table_config_service = get_collector_table_config_service(self.meta_storage,
 		                                                                         self.snowflake_generator,
 		                                                                         self.principal_service)
+		self.table_config_service = get_table_config_service(self.principal_service)
 
 	def find_data_by_data_id(self, config: CollectorTableConfig, data_id: Dict) -> Optional[Dict[str, Any]]:
 		return SourceTableExtractor(config, self.principal_service).find_by_id(data_id)
@@ -30,7 +32,7 @@ class DataCaptureService:
 	def find_parent_node(self, config: CollectorTableConfig,
 	                     data_: Dict) -> Tuple[CollectorTableConfig, Optional[Dict[str, Any]]]:
 		if config.parentName:
-			parent_config = self.collector_table_config_service.find_by_name(config.parentName)
+			parent_config = self.table_config_service.find_by_name(config.parentName)
 			parent_data = SourceTableExtractor(parent_config, self.principal_service).find(
 				ArrayHelper(config.joinKeys).map(lambda join_key: build_criteria_by_join_key(join_key, data_)).to_list()
 			)
@@ -44,7 +46,7 @@ class DataCaptureService:
 	def build_json(self,
 	               config: CollectorTableConfig,
 	               data: Dict):
-		child_configs = self.collector_table_config_service.find_by_parent_name(config.name)
+		child_configs = self.table_config_service.find_by_parent_name(config.name)
 		if child_configs:
 			ArrayHelper(child_configs).map(lambda child_config: self.get_child_data(child_config, data))
 

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/service/model_config_service.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from watchmen_meta.common import ask_meta_storage, ask_snowflake_generator
+
+from watchmen_collector_kernel.storage import get_collector_model_config_service
+from watchmen_collector_kernel.cache import CollectorCacheService
+from watchmen_collector_kernel.common import CollectorKernelException
+from watchmen_collector_kernel.model import CollectorModelConfig
+
+from watchmen_auth import PrincipalService
+
+
+class ModelConfigService:
+
+	def __init__(self, principal_service: PrincipalService):
+		self.principalService = principal_service
+
+	def find_by_name(self, model_name: str) -> Optional[CollectorModelConfig]:
+		config = CollectorCacheService.model_config().get(model_name)
+		if config is not None:
+			return config
+
+		storage_service = get_collector_model_config_service(
+			ask_meta_storage(), ask_snowflake_generator(), self.principalService
+		)
+		model_config: CollectorModelConfig = storage_service.find_by_name(model_name)
+		if model_config is None:
+			return None
+
+		CollectorCacheService.model_config().put(model_config)
+		return model_config
+
+
+def get_model_config_service(principal_service: PrincipalService) -> ModelConfigService:
+	return ModelConfigService(principal_service)

watchmen_collector_kernel-16.5.17/src/watchmen_collector_kernel/service/table_config_service.py

@@ -0,0 +1,70 @@
+from typing import Optional, List
+
+from watchmen_utilities import is_blank
+
+from watchmen_meta.common import ask_meta_storage, ask_snowflake_generator
+
+from watchmen_collector_kernel.storage import get_collector_table_config_service
+from watchmen_collector_kernel.cache import CollectorCacheService
+from watchmen_collector_kernel.common import CollectorKernelException
+from watchmen_collector_kernel.model import CollectorTableConfig
+
+from watchmen_auth import PrincipalService
+
+
+class TableConfigService:
+
+	def __init__(self, principal_service: PrincipalService):
+		self.principalService = principal_service
+
+	def find_by_table_name(self, table_name: str) -> Optional[CollectorTableConfig]:
+		config = CollectorCacheService.table_config().get_config_by_table_name(table_name)
+		if config is not None:
+			return config
+
+		storage_service = get_collector_table_config_service(
+			ask_meta_storage(), ask_snowflake_generator(), self.principalService
+		)
+		table_config: CollectorTableConfig = storage_service.find_by_table_name(table_name)
+		if table_config is None:
+			return None
+
+		CollectorCacheService.table_config().put_config_by_table_name(table_config)
+		return table_config
+
+	def find_by_name(self, name: str) -> Optional[CollectorTableConfig]:
+		config = CollectorCacheService.table_config().get_config_by_name(name)
+		if config is not None:
+			return config
+
+		storage_service = get_collector_table_config_service(
+			ask_meta_storage(), ask_snowflake_generator(), self.principalService
+		)
+		table_config: CollectorTableConfig = storage_service.find_by_name(name)
+		if table_config is None:
+			return None
+
+		CollectorCacheService.table_config().put_config_by_name(table_config)
+		return table_config
+
+	def find_by_parent_name(self, parent_name: Optional[str]) -> Optional[List[CollectorTableConfig]]:
+		if is_blank(parent_name):
+			return None
+
+		configs = CollectorCacheService.table_config().get_configs_by_parent_name(parent_name)
+		if configs is not None:
+			return configs
+
+		storage_service = get_collector_table_config_service(
+			ask_meta_storage(), ask_snowflake_generator(), self.principalService
+		)
+		table_configs: List[CollectorTableConfig] = storage_service.find_by_parent_name(parent_name)
+		if table_configs is None:
+			return None
+
+		CollectorCacheService.table_config().put_configs_by_parent_name(parent_name, table_configs)
+		return table_configs
+
+
+def get_table_config_service(principal_service: PrincipalService) -> TableConfigService:
+	return TableConfigService(principal_service)