watchmen-collector-kernel 16.4.7__py3-none-any.whl → 16.4.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- watchmen_collector_kernel/common/__init__.py +2 -2
- watchmen_collector_kernel/common/constants.py +16 -0
- watchmen_collector_kernel/model/__init__.py +15 -1
- watchmen_collector_kernel/model/change_data_json.py +28 -0
- watchmen_collector_kernel/model/change_data_record.py +19 -0
- watchmen_collector_kernel/model/collector_model_config.py +12 -0
- watchmen_collector_kernel/model/collector_table_config.py +78 -0
- watchmen_collector_kernel/model/competitive_lock.py +11 -0
- watchmen_collector_kernel/model/condition.py +73 -0
- watchmen_collector_kernel/model/scheduled_task.py +30 -0
- watchmen_collector_kernel/model/trigger_event.py +11 -0
- watchmen_collector_kernel/model/trigger_model.py +8 -0
- watchmen_collector_kernel/model/trigger_table.py +11 -0
- watchmen_collector_kernel/service/__init__.py +8 -0
- watchmen_collector_kernel/service/criteria_builder.py +53 -0
- watchmen_collector_kernel/service/data_capture.py +60 -0
- watchmen_collector_kernel/service/extract_source.py +71 -0
- watchmen_collector_kernel/service/extract_utils.py +70 -0
- watchmen_collector_kernel/service/lock_clean.py +39 -0
- watchmen_collector_kernel/service/lock_helper.py +28 -0
- watchmen_collector_kernel/service/task_housekeeping.py +61 -0
- watchmen_collector_kernel/service/task_service.py +72 -0
- watchmen_collector_kernel/service/trigger_collector.py +147 -0
- watchmen_collector_kernel/storage/__init__.py +17 -0
- watchmen_collector_kernel/storage/change_data_json_service.py +173 -0
- watchmen_collector_kernel/storage/change_data_record_service.py +185 -0
- watchmen_collector_kernel/storage/collector_model_config_service.py +107 -0
- watchmen_collector_kernel/storage/collector_table_config_service.py +168 -0
- watchmen_collector_kernel/storage/competitive_lock_service.py +83 -0
- watchmen_collector_kernel/storage/scheduled_task_service.py +155 -0
- watchmen_collector_kernel/storage/trigger_event_service.py +86 -0
- watchmen_collector_kernel/storage/trigger_model_service.py +96 -0
- watchmen_collector_kernel/storage/trigger_table_service.py +113 -0
- {watchmen_collector_kernel-16.4.7.dist-info → watchmen_collector_kernel-16.4.9.dist-info}/METADATA +9 -9
- watchmen_collector_kernel-16.4.9.dist-info/RECORD +38 -0
- watchmen_collector_kernel/common/settings.py +0 -14
- watchmen_collector_kernel/connector/__init__.py +0 -1
- watchmen_collector_kernel/connector/handler.py +0 -42
- watchmen_collector_kernel/connector/housekeeping.py +0 -58
- watchmen_collector_kernel/connector/s3_connector.py +0 -210
- watchmen_collector_kernel/lock/__init__.py +0 -3
- watchmen_collector_kernel/lock/distributed_lock.py +0 -23
- watchmen_collector_kernel/lock/oss_collector_lock_service.py +0 -127
- watchmen_collector_kernel/lock/unique_key_distributed_lock.py +0 -39
- watchmen_collector_kernel/model/oss_collector_competitive_lock.py +0 -15
- watchmen_collector_kernel-16.4.7.dist-info/RECORD +0 -17
- {watchmen_collector_kernel-16.4.7.dist-info → watchmen_collector_kernel-16.4.9.dist-info}/LICENSE +0 -0
- {watchmen_collector_kernel-16.4.7.dist-info → watchmen_collector_kernel-16.4.9.dist-info}/WHEEL +0 -0
@@ -1,2 +1,2 @@
|
|
1
|
-
from .
|
2
|
-
|
1
|
+
from .constants import CHANGE_RECORD_ID, TENANT_ID, IS_MERGED, LEFT_BRACE, RIGHT_BRACE, COMMA, \
|
2
|
+
IS_POSTED, CHANGE_JSON_ID, WAVE, IS_FINISHED, IS_EXTRACTED, MODEL_TRIGGER_ID
|
@@ -0,0 +1,16 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
CHANGE_RECORD_ID = 'change_record_id'
|
4
|
+
CHANGE_JSON_ID = 'change_json_id'
|
5
|
+
TENANT_ID = 'tenant_id'
|
6
|
+
MODEL_TRIGGER_ID = 'model_trigger_id'
|
7
|
+
|
8
|
+
IS_FINISHED = 'is_finished'
|
9
|
+
IS_EXTRACTED = 'is_extracted'
|
10
|
+
IS_MERGED = 'is_merged'
|
11
|
+
IS_POSTED = 'is_posted'
|
12
|
+
|
13
|
+
LEFT_BRACE = '{'
|
14
|
+
RIGHT_BRACE = '}'
|
15
|
+
COMMA = ','
|
16
|
+
WAVE = '~'
|
@@ -1 +1,15 @@
|
|
1
|
-
from .
|
1
|
+
from .competitive_lock import CompetitiveLock
|
2
|
+
from .scheduled_task import ScheduledTask
|
3
|
+
|
4
|
+
from .collector_model_config import CollectorModelConfig
|
5
|
+
from .collector_table_config import CollectorTableConfig
|
6
|
+
|
7
|
+
from .trigger_event import TriggerEvent
|
8
|
+
from .trigger_model import TriggerModel
|
9
|
+
from .trigger_table import TriggerTable
|
10
|
+
|
11
|
+
from .change_data_record import ChangeDataRecord
|
12
|
+
from .change_data_json import ChangeDataJson
|
13
|
+
|
14
|
+
from .condition import construct_conditions, Condition, ConditionJoint, ConditionExpression, \
|
15
|
+
ConditionJointConjunction
|
@@ -0,0 +1,28 @@
|
|
1
|
+
from typing import Dict, List
|
2
|
+
|
3
|
+
from pydantic import BaseModel
|
4
|
+
|
5
|
+
from watchmen_model.common import TenantBasedTuple, Storable
|
6
|
+
|
7
|
+
|
8
|
+
class Dependence(Storable, BaseModel):
|
9
|
+
modelName: str
|
10
|
+
objectId: str
|
11
|
+
|
12
|
+
|
13
|
+
class ChangeDataJson(TenantBasedTuple, BaseModel):
|
14
|
+
changeJsonId: int
|
15
|
+
resourceId: str
|
16
|
+
modelName: str
|
17
|
+
objectId: str
|
18
|
+
sequence: int
|
19
|
+
content: Dict
|
20
|
+
dataId: Dict
|
21
|
+
dependOn: List[Dependence]
|
22
|
+
isPosted: bool
|
23
|
+
result: Dict
|
24
|
+
taskId: int
|
25
|
+
tableTriggerId: int
|
26
|
+
modelTriggerId: int
|
27
|
+
eventTriggerId: int
|
28
|
+
|
@@ -0,0 +1,19 @@
|
|
1
|
+
from typing import Dict, Optional
|
2
|
+
|
3
|
+
from watchmen_model.common import TenantBasedTuple
|
4
|
+
|
5
|
+
|
6
|
+
class ChangeDataRecord(TenantBasedTuple):
|
7
|
+
changeRecordId: int
|
8
|
+
modelName: str
|
9
|
+
tableName: str
|
10
|
+
dataId: Dict
|
11
|
+
rootTableName: str
|
12
|
+
rootDataId: Dict
|
13
|
+
isMerged: bool
|
14
|
+
result: Dict
|
15
|
+
tableTriggerId: int
|
16
|
+
modelTriggerId: int
|
17
|
+
eventTriggerId: int
|
18
|
+
|
19
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
from typing import List
|
2
|
+
from watchmen_model.common import TenantBasedTuple, OptimisticLock
|
3
|
+
from pydantic import BaseModel
|
4
|
+
|
5
|
+
|
6
|
+
class CollectorModelConfig(TenantBasedTuple, OptimisticLock, BaseModel):
|
7
|
+
modelId: str
|
8
|
+
modelName: str
|
9
|
+
dependOn: List[str]
|
10
|
+
rawTopicCode: str
|
11
|
+
isParalleled: bool
|
12
|
+
|
@@ -0,0 +1,78 @@
|
|
1
|
+
from typing import Dict, List, Optional, Union
|
2
|
+
|
3
|
+
from pydantic import BaseModel
|
4
|
+
|
5
|
+
from .condition import Condition, construct_conditions
|
6
|
+
from watchmen_model.common import TenantBasedTuple, Storable, OptimisticLock
|
7
|
+
from watchmen_utilities import ArrayHelper
|
8
|
+
|
9
|
+
|
10
|
+
class JoinKey(Storable, BaseModel):
|
11
|
+
parentKey: str = None
|
12
|
+
childKey: str = None
|
13
|
+
|
14
|
+
|
15
|
+
class Dependence(Storable, BaseModel):
|
16
|
+
modelName: str
|
17
|
+
objectKey: str # the dependent column
|
18
|
+
|
19
|
+
|
20
|
+
def construct_join_key(join_key: Union[JoinKey, Dict]) -> Optional[JoinKey]:
|
21
|
+
if join_key is None:
|
22
|
+
return None
|
23
|
+
elif isinstance(join_key, JoinKey):
|
24
|
+
return join_key
|
25
|
+
else:
|
26
|
+
return JoinKey(**join_key)
|
27
|
+
|
28
|
+
|
29
|
+
def construct_join_keys(join_keys: Optional[List[Union[JoinKey, Dict]]]) -> Optional[List[JoinKey]]:
|
30
|
+
if join_keys is None:
|
31
|
+
return None
|
32
|
+
else:
|
33
|
+
return ArrayHelper(join_keys).map(lambda x: construct_join_key(x)).to_list()
|
34
|
+
|
35
|
+
|
36
|
+
def construct_dependence(dependence: Union[Dependence, Dict]) -> Optional[Dependence]:
|
37
|
+
if dependence is None:
|
38
|
+
return None
|
39
|
+
elif isinstance(dependence, Dependence):
|
40
|
+
return dependence
|
41
|
+
else:
|
42
|
+
return Dependence(**dependence)
|
43
|
+
|
44
|
+
|
45
|
+
def construct_depend_on(depend_on: Optional[List[Union[Dependence, Dict]]]) -> Optional[List[Dependence]]:
|
46
|
+
if depend_on is None:
|
47
|
+
return None
|
48
|
+
else:
|
49
|
+
return ArrayHelper(depend_on).map(lambda x: construct_dependence(x)).to_list()
|
50
|
+
|
51
|
+
|
52
|
+
class CollectorTableConfig(TenantBasedTuple, OptimisticLock, BaseModel):
|
53
|
+
configId: str = None
|
54
|
+
name: str = None
|
55
|
+
tableName: str = None
|
56
|
+
primaryKey: List[str] = None
|
57
|
+
objectKey: str = None
|
58
|
+
sequenceKey: str = None
|
59
|
+
modelName: str = None
|
60
|
+
parentName: str = None
|
61
|
+
label: str = None
|
62
|
+
joinKeys: List[JoinKey] = []
|
63
|
+
dependOn: List[Dependence] = []
|
64
|
+
auditColumn: str = None
|
65
|
+
conditions: List[Condition] = []
|
66
|
+
dataSourceId: str = None
|
67
|
+
isList: bool = False
|
68
|
+
triggered: bool = False
|
69
|
+
|
70
|
+
def __setattr__(self, name, value):
|
71
|
+
if name == 'joinKeys':
|
72
|
+
super().__setattr__(name, construct_join_keys(value))
|
73
|
+
elif name == 'dependOn':
|
74
|
+
super().__setattr__(name, construct_depend_on(value))
|
75
|
+
elif name == 'conditions':
|
76
|
+
super().__setattr__(name, construct_conditions(value))
|
77
|
+
else:
|
78
|
+
super().__setattr__(name, value)
|
@@ -0,0 +1,11 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from watchmen_model.common import CompetitiveLockId, Storable, TenantId
|
5
|
+
|
6
|
+
|
7
|
+
class CompetitiveLock(Storable):
|
8
|
+
lockId: CompetitiveLockId
|
9
|
+
resourceId: str
|
10
|
+
registeredAt: Optional[datetime]
|
11
|
+
tenantId: TenantId
|
@@ -0,0 +1,73 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from datetime import date, datetime
|
4
|
+
from enum import Enum
|
5
|
+
from typing import Optional, List, Union, Dict
|
6
|
+
|
7
|
+
from pydantic import BaseModel
|
8
|
+
|
9
|
+
from watchmen_model.common import DataModel
|
10
|
+
from watchmen_utilities import ArrayHelper
|
11
|
+
|
12
|
+
|
13
|
+
class ConditionOperator(str, Enum):
|
14
|
+
IS_EMPTY = 'is-empty',
|
15
|
+
IS_NOT_EMPTY = 'is-not-empty',
|
16
|
+
IS_BLANK = 'is-blank',
|
17
|
+
IS_NOT_BLANK = 'is-not-blank',
|
18
|
+
EQUALS = 'equals',
|
19
|
+
NOT_EQUALS = 'not-equals',
|
20
|
+
LESS_THAN = 'less-than',
|
21
|
+
LESS_THAN_OR_EQUALS = 'less-than-or-equals',
|
22
|
+
GREATER_THAN = 'greater-than',
|
23
|
+
GREATER_THAN_OR_EQUALS = 'greater-than-or-equals',
|
24
|
+
IN = 'in',
|
25
|
+
NOT_IN = 'not-in'
|
26
|
+
|
27
|
+
|
28
|
+
class Condition(DataModel, BaseModel):
|
29
|
+
pass
|
30
|
+
|
31
|
+
|
32
|
+
class ConditionExpression(Condition):
|
33
|
+
columnName: str
|
34
|
+
operator: ConditionOperator = ConditionOperator.EQUALS
|
35
|
+
columnValue: Optional[Union[List[int], List[str], int, str, date, datetime]] = None
|
36
|
+
|
37
|
+
|
38
|
+
class ConditionJointConjunction(str, Enum):
|
39
|
+
AND = 'and',
|
40
|
+
OR = 'or'
|
41
|
+
|
42
|
+
|
43
|
+
class ConditionJoint(Condition):
|
44
|
+
conjunction: ConditionJointConjunction = ConditionJointConjunction.AND
|
45
|
+
children: List[Condition]
|
46
|
+
|
47
|
+
def __setattr__(self, name, value):
|
48
|
+
if name == 'children':
|
49
|
+
super().__setattr__(name, construct_conditions(value))
|
50
|
+
else:
|
51
|
+
super().__setattr__(name, value)
|
52
|
+
|
53
|
+
|
54
|
+
def construct_condition(condition: Optional[Union[Condition, Dict]]) -> Optional[Condition]:
|
55
|
+
if condition is None:
|
56
|
+
return None
|
57
|
+
elif isinstance(condition, Condition):
|
58
|
+
return condition
|
59
|
+
elif isinstance(condition, ConditionExpression):
|
60
|
+
return condition
|
61
|
+
elif isinstance(condition, ConditionJoint):
|
62
|
+
return condition
|
63
|
+
elif condition.get('conjunction'):
|
64
|
+
return ConditionJoint(**condition)
|
65
|
+
else:
|
66
|
+
return ConditionExpression(**condition)
|
67
|
+
|
68
|
+
|
69
|
+
def construct_conditions(conditions: Optional[List[Union[Condition, Dict]]]) -> Optional[List[Condition]]:
|
70
|
+
if conditions is None:
|
71
|
+
return None
|
72
|
+
else:
|
73
|
+
return ArrayHelper(conditions).map(lambda x: construct_condition(x)).to_list()
|
@@ -0,0 +1,30 @@
|
|
1
|
+
from typing import Dict, List
|
2
|
+
|
3
|
+
from watchmen_model.common import TenantBasedTuple, ScheduledTaskId, Storable
|
4
|
+
from pydantic import BaseModel
|
5
|
+
from enum import Enum
|
6
|
+
|
7
|
+
|
8
|
+
class Result(str, Enum):
|
9
|
+
DEPENDENCY_FAILED = "DEPENDENCY_FAILED"
|
10
|
+
PROCESS_TASK_SUCCESS = "PROCESS_TASK_SUCCESS"
|
11
|
+
PROCESS_TASK_FAILED = "PROCESS_TASK_FAILED"
|
12
|
+
|
13
|
+
|
14
|
+
class Dependence(Storable, BaseModel):
|
15
|
+
modelName: str
|
16
|
+
objectId: str
|
17
|
+
|
18
|
+
|
19
|
+
class ScheduledTask(TenantBasedTuple, BaseModel):
|
20
|
+
taskId: ScheduledTaskId
|
21
|
+
resourceId: str # global unique, monotonous increase
|
22
|
+
topicCode: str
|
23
|
+
content: Dict
|
24
|
+
modelName: str
|
25
|
+
objectId: str
|
26
|
+
dependOn: List[Dependence]
|
27
|
+
parentTaskId: List[int]
|
28
|
+
isFinished: bool
|
29
|
+
result: Dict
|
30
|
+
|
@@ -0,0 +1,11 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
|
3
|
+
from watchmen_model.common import TenantBasedTuple
|
4
|
+
from pydantic import BaseModel
|
5
|
+
|
6
|
+
|
7
|
+
class TriggerEvent(TenantBasedTuple, BaseModel):
|
8
|
+
eventTriggerId: int
|
9
|
+
startTime: datetime
|
10
|
+
endTime: datetime
|
11
|
+
isFinished: bool = False
|
@@ -0,0 +1,8 @@
|
|
1
|
+
from .lock_helper import try_lock_nowait, unlock
|
2
|
+
from .lock_clean import init_lock_clean
|
3
|
+
from .trigger_collector import get_trigger_collector
|
4
|
+
from .data_capture import DataCaptureService
|
5
|
+
from .extract_source import SourceTableExtractor
|
6
|
+
from .task_service import get_task_service
|
7
|
+
from .criteria_builder import CriteriaBuilder
|
8
|
+
from .extract_utils import build_audit_column_criteria, cal_array2d_diff, build_data_id, get_data_id
|
@@ -0,0 +1,53 @@
|
|
1
|
+
from typing import List, Dict, Any
|
2
|
+
from watchmen_collector_kernel.common import LEFT_BRACE, RIGHT_BRACE
|
3
|
+
from watchmen_collector_kernel.model import Condition, ConditionJoint, ConditionExpression
|
4
|
+
from watchmen_data_kernel.common import ask_all_date_formats
|
5
|
+
from watchmen_storage import EntityCriteria, EntityCriteriaExpression, ColumnNameLiteral, EntityCriteriaJoint, \
|
6
|
+
EntityCriteriaStatement
|
7
|
+
from watchmen_utilities import ArrayHelper, is_date
|
8
|
+
|
9
|
+
|
10
|
+
class CriteriaBuilder:
|
11
|
+
|
12
|
+
def __init__(self, variables: Dict):
|
13
|
+
self.variables = variables
|
14
|
+
|
15
|
+
def add_variable(self, variable_name: str, variable_value: Any):
|
16
|
+
self.variables[variable_name] = variable_value
|
17
|
+
|
18
|
+
def build_criteria(self, conditions: List[Condition]) -> EntityCriteria:
|
19
|
+
return ArrayHelper(conditions).map(self.build_statement).to_list()
|
20
|
+
|
21
|
+
def build_statement(self, condition: Condition) -> EntityCriteriaStatement:
|
22
|
+
if isinstance(condition, ConditionJoint):
|
23
|
+
return self.build_criteria_joint(condition)
|
24
|
+
if isinstance(condition, ConditionExpression):
|
25
|
+
return self.build_criteria_expression(condition)
|
26
|
+
else:
|
27
|
+
raise ValueError(f'Unsupported condition[{condition}].')
|
28
|
+
|
29
|
+
def build_criteria_expression(self, condition: ConditionExpression) -> EntityCriteriaExpression:
|
30
|
+
return EntityCriteriaExpression(left=ColumnNameLiteral(columnName=condition.columnName),
|
31
|
+
operator=condition.operator,
|
32
|
+
right=self.parse_condition_value(condition.columnValue))
|
33
|
+
|
34
|
+
def build_criteria_joint(self, condition: ConditionJoint) -> EntityCriteriaJoint:
|
35
|
+
return EntityCriteriaJoint(conjunction=condition.conjunction,
|
36
|
+
children=ArrayHelper(condition.children)
|
37
|
+
.map(lambda child: self.build_statement(child))
|
38
|
+
.to_list())
|
39
|
+
|
40
|
+
def parse_condition_value(self, condition_value: Any) -> Any:
|
41
|
+
if isinstance(condition_value, str):
|
42
|
+
if condition_value.startswith(LEFT_BRACE) and condition_value.startswith(RIGHT_BRACE):
|
43
|
+
variable_name = condition_value.removeprefix(LEFT_BRACE).removesuffix(RIGHT_BRACE)
|
44
|
+
variable_value = self.variables.get(variable_name)
|
45
|
+
parsed, value = is_date(variable_value, ask_all_date_formats())
|
46
|
+
if parsed:
|
47
|
+
return value
|
48
|
+
else:
|
49
|
+
return variable_value
|
50
|
+
else:
|
51
|
+
return condition_value
|
52
|
+
else:
|
53
|
+
return condition_value
|
@@ -0,0 +1,60 @@
|
|
1
|
+
from logging import getLogger
|
2
|
+
from typing import Optional, Dict, Any, Tuple
|
3
|
+
|
4
|
+
from watchmen_auth import PrincipalService
|
5
|
+
from watchmen_collector_kernel.model import CollectorTableConfig
|
6
|
+
from .extract_source import SourceTableExtractor
|
7
|
+
from watchmen_collector_kernel.storage import get_collector_table_config_service
|
8
|
+
from watchmen_storage import TransactionalStorageSPI, SnowflakeGenerator
|
9
|
+
from watchmen_utilities import ArrayHelper
|
10
|
+
from .extract_utils import build_criteria_by_join_key
|
11
|
+
|
12
|
+
logger = getLogger(__name__)
|
13
|
+
|
14
|
+
|
15
|
+
class DataCaptureService:
|
16
|
+
|
17
|
+
def __init__(self, storage: TransactionalStorageSPI,
|
18
|
+
snowflake_generator: SnowflakeGenerator,
|
19
|
+
principal_service: PrincipalService):
|
20
|
+
self.meta_storage = storage
|
21
|
+
self.snowflake_generator = snowflake_generator
|
22
|
+
self.principal_service = principal_service
|
23
|
+
self.collector_table_config_service = get_collector_table_config_service(self.meta_storage,
|
24
|
+
self.snowflake_generator,
|
25
|
+
self.principal_service)
|
26
|
+
|
27
|
+
def find_data_by_data_id(self, config: CollectorTableConfig, data_id: Dict) -> Optional[Dict[str, Any]]:
|
28
|
+
return SourceTableExtractor(config, self.principal_service).find_by_id(data_id)
|
29
|
+
|
30
|
+
def find_parent_node(self, config: CollectorTableConfig,
|
31
|
+
data_: Dict) -> Tuple[CollectorTableConfig, Optional[Dict[str, Any]]]:
|
32
|
+
if config.parentName:
|
33
|
+
parent_config = self.collector_table_config_service.find_by_name(config.parentName)
|
34
|
+
parent_data = SourceTableExtractor(parent_config, self.principal_service).find(
|
35
|
+
ArrayHelper(config.joinKeys).map(lambda join_key: build_criteria_by_join_key(join_key, data_)).to_list()
|
36
|
+
)
|
37
|
+
if len(parent_data) != 1:
|
38
|
+
raise RuntimeError(f'The data : {data_}, config_name: {config.name}, '
|
39
|
+
f'parent_config_name: {parent_config.name}, size: {len(parent_data)}')
|
40
|
+
return self.find_parent_node(parent_config, parent_data[0])
|
41
|
+
else:
|
42
|
+
return config, data_
|
43
|
+
|
44
|
+
def build_json(self,
|
45
|
+
config: CollectorTableConfig,
|
46
|
+
data: Dict):
|
47
|
+
child_configs = self.collector_table_config_service.find_by_parent_name(config.name)
|
48
|
+
if child_configs:
|
49
|
+
ArrayHelper(child_configs).map(lambda child_config: self.get_child_data(child_config, data))
|
50
|
+
|
51
|
+
def get_child_data(self, child_config: CollectorTableConfig, data_: Dict):
|
52
|
+
child_data = SourceTableExtractor(child_config, self.principal_service).find(
|
53
|
+
ArrayHelper(child_config.joinKeys).map(lambda join_key: build_criteria_by_join_key(join_key, data_, True)).to_list()
|
54
|
+
)
|
55
|
+
if child_data:
|
56
|
+
if child_config.isList:
|
57
|
+
data_[child_config.label] = child_data
|
58
|
+
else:
|
59
|
+
data_[child_config.label] = child_data[0]
|
60
|
+
ArrayHelper(child_data).each(lambda child: self.build_json(child_config, child))
|
@@ -0,0 +1,71 @@
|
|
1
|
+
from typing import Optional, List, Dict, Any
|
2
|
+
|
3
|
+
from watchmen_auth import PrincipalService
|
4
|
+
from .extract_utils import build_criteria_by_primary_key
|
5
|
+
from watchmen_collector_kernel.model import CollectorTableConfig
|
6
|
+
from watchmen_data_kernel.service import ask_topic_storage, ask_topic_data_service
|
7
|
+
from watchmen_data_kernel.topic_schema import TopicSchema
|
8
|
+
from watchmen_model.admin import Topic, TopicKind
|
9
|
+
from watchmen_storage import EntityCriteria, EntityStraightColumn
|
10
|
+
from watchmen_utilities import get_current_time_in_seconds, ArrayHelper
|
11
|
+
|
12
|
+
|
13
|
+
class SourceTableExtractor:
|
14
|
+
|
15
|
+
def __init__(self, config: CollectorTableConfig, principal_service: PrincipalService):
|
16
|
+
self.config = config
|
17
|
+
self.principal_service = principal_service
|
18
|
+
self.topic = self.fake_extracted_table_to_topic(self.config)
|
19
|
+
self.storage = ask_topic_storage(self.topic, self.principal_service)
|
20
|
+
self.storage.register_topic(self.topic)
|
21
|
+
self.service = ask_topic_data_service(TopicSchema(self.topic), self.storage, principal_service)
|
22
|
+
|
23
|
+
# noinspection PyMethodMayBeStatic
|
24
|
+
def fake_topic_id(self, config_id: str) -> str:
|
25
|
+
return f'f-{config_id}'
|
26
|
+
|
27
|
+
def fake_extracted_table_to_topic(self, config: CollectorTableConfig) -> Topic:
|
28
|
+
# Fake synonym topic to visit source table
|
29
|
+
topic = Topic(topicId=self.fake_topic_id(config.configId),
|
30
|
+
name=config.tableName,
|
31
|
+
dataSourceId=config.dataSourceId,
|
32
|
+
kind=TopicKind.SYNONYM,
|
33
|
+
tenantId=self.principal_service.tenantId
|
34
|
+
)
|
35
|
+
topic_storage = ask_topic_storage(topic, self.principal_service)
|
36
|
+
factors = topic_storage.ask_synonym_factors(config.tableName)
|
37
|
+
topic.factors = factors
|
38
|
+
now = get_current_time_in_seconds()
|
39
|
+
topic.createdAt = now
|
40
|
+
topic.createdBy = self.principal_service.get_user_id()
|
41
|
+
topic.lastModifiedAt = now
|
42
|
+
topic.lastModifiedBy = self.principal_service.get_user_id()
|
43
|
+
return topic
|
44
|
+
|
45
|
+
def find_change_data(self, criteria: EntityCriteria) -> Optional[List[Dict[str, Any]]]:
|
46
|
+
return self.lower_key(self.service.find_straight_values(
|
47
|
+
criteria=criteria,
|
48
|
+
columns=ArrayHelper(self.config.primaryKey).map(
|
49
|
+
lambda column_name: EntityStraightColumn(columnName=column_name)
|
50
|
+
).to_list()
|
51
|
+
))
|
52
|
+
|
53
|
+
def find_by_id(self, data_id: Dict) -> Optional[Dict[str, Any]]:
|
54
|
+
results = self.service.find(build_criteria_by_primary_key(data_id))
|
55
|
+
if len(results) == 1:
|
56
|
+
return self.lower_key(results)[0]
|
57
|
+
elif len(results) == 0:
|
58
|
+
return None
|
59
|
+
else:
|
60
|
+
raise RuntimeError(f'too many results with {data_id} find')
|
61
|
+
|
62
|
+
def find(self, criteria: EntityCriteria) -> Optional[List[Dict[str, Any]]]:
|
63
|
+
return self.lower_key(self.service.find(criteria))
|
64
|
+
|
65
|
+
def exists(self, criteria: EntityCriteria) -> bool:
|
66
|
+
return self.service.exists(criteria)
|
67
|
+
|
68
|
+
# noinspection PyMethodMayBeStatic
|
69
|
+
def lower_key(self, data_: List) -> Optional[List[Dict]]:
|
70
|
+
return ArrayHelper(data_).map(
|
71
|
+
lambda row: {k.lower(): v for k, v in row.items()}).to_list()
|
@@ -0,0 +1,70 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from logging import getLogger
|
3
|
+
from typing import Dict, Any, List
|
4
|
+
|
5
|
+
import numpy as np
|
6
|
+
|
7
|
+
from watchmen_collector_kernel.model.collector_table_config import JoinKey
|
8
|
+
from watchmen_storage import EntityCriteriaExpression, ColumnNameLiteral, EntityCriteriaStatement, EntityCriteria, \
|
9
|
+
EntityCriteriaOperator
|
10
|
+
from watchmen_utilities import ArrayHelper
|
11
|
+
|
12
|
+
logger = getLogger(__name__)
|
13
|
+
|
14
|
+
|
15
|
+
def get_data_id(primary_key: List[str], data_: Dict[str, Any]) -> Dict:
|
16
|
+
data_id = {}
|
17
|
+
|
18
|
+
def set_data_id(column_name: str, data_dict: Dict):
|
19
|
+
data_id[column_name] = data_dict.get(column_name)
|
20
|
+
|
21
|
+
ArrayHelper(primary_key).each(lambda column_name: set_data_id(column_name, data_))
|
22
|
+
return data_id
|
23
|
+
|
24
|
+
|
25
|
+
def build_data_id(primary_key: List[str], values: List) -> Dict:
|
26
|
+
data_id = {}
|
27
|
+
|
28
|
+
def set_data_id(column_name: str, index: int, data_list: List):
|
29
|
+
data_id[column_name] = data_list[index]
|
30
|
+
|
31
|
+
ArrayHelper(primary_key).each_with_index(lambda column_name, index: set_data_id(column_name, index, values))
|
32
|
+
return data_id
|
33
|
+
|
34
|
+
|
35
|
+
def build_audit_column_criteria(audit_column_name: str, start_time: datetime, end_time: datetime) -> EntityCriteria:
|
36
|
+
return [
|
37
|
+
EntityCriteriaExpression(
|
38
|
+
left=ColumnNameLiteral(columnName=audit_column_name),
|
39
|
+
operator=EntityCriteriaOperator.GREATER_THAN_OR_EQUALS,
|
40
|
+
right=start_time),
|
41
|
+
EntityCriteriaExpression(
|
42
|
+
left=ColumnNameLiteral(columnName=audit_column_name),
|
43
|
+
operator=EntityCriteriaOperator.LESS_THAN_OR_EQUALS,
|
44
|
+
right=end_time),
|
45
|
+
]
|
46
|
+
|
47
|
+
|
48
|
+
def build_criteria_by_primary_key(data_id: Dict) -> List[EntityCriteriaExpression]:
|
49
|
+
criteria = []
|
50
|
+
for key, value in data_id.items():
|
51
|
+
criteria.append(
|
52
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName=key), right=value)
|
53
|
+
)
|
54
|
+
return criteria
|
55
|
+
|
56
|
+
|
57
|
+
def build_criteria_by_join_key(join_key: JoinKey, data: Dict, is_child: bool = False) -> EntityCriteriaStatement:
|
58
|
+
if is_child:
|
59
|
+
column_name = join_key.childKey
|
60
|
+
column_value = data.get(join_key.parentKey)
|
61
|
+
else:
|
62
|
+
column_name = join_key.parentKey
|
63
|
+
column_value = data.get(join_key.childKey)
|
64
|
+
return EntityCriteriaExpression(left=ColumnNameLiteral(columnName=column_name), right=column_value)
|
65
|
+
|
66
|
+
|
67
|
+
def cal_array2d_diff(array_0: np.ndarray, array_1: np.ndarray) -> np.ndarray:
|
68
|
+
array_0_rows = array_0.view([('', array_0.dtype)] * array_0.shape[1])
|
69
|
+
array_1_rows = array_1.view([('', array_1.dtype)] * array_1.shape[1])
|
70
|
+
return np.setdiff1d(array_0_rows, array_1_rows).view(array_0.dtype).reshape(-1, array_0.shape[1])
|
@@ -0,0 +1,39 @@
|
|
1
|
+
from datetime import datetime, timedelta
|
2
|
+
from logging import getLogger
|
3
|
+
from threading import Thread
|
4
|
+
|
5
|
+
from time import sleep
|
6
|
+
|
7
|
+
from watchmen_collector_kernel.storage import get_competitive_lock_service
|
8
|
+
from watchmen_meta.common import ask_meta_storage
|
9
|
+
|
10
|
+
|
11
|
+
class LockClean:
|
12
|
+
|
13
|
+
def __init__(self):
|
14
|
+
self.lock_service = get_competitive_lock_service(ask_meta_storage())
|
15
|
+
self.cleanInterval = 60
|
16
|
+
|
17
|
+
def run(self):
|
18
|
+
try:
|
19
|
+
while True:
|
20
|
+
self.clean_lock()
|
21
|
+
sleep(self.cleanInterval)
|
22
|
+
except Exception as e:
|
23
|
+
getLogger(__name__).error(e, exc_info=True, stack_info=True)
|
24
|
+
sleep(60)
|
25
|
+
self.restart()
|
26
|
+
|
27
|
+
def clean_lock(self):
|
28
|
+
query_time = datetime.now() - timedelta(minutes=120)
|
29
|
+
locks = self.lock_service.find_overtime_lock(query_time)
|
30
|
+
for lock in locks:
|
31
|
+
self.lock_service.delete_by_id(lock.lockId)
|
32
|
+
|
33
|
+
def restart(self):
|
34
|
+
Thread(target=LockClean.run, args=(self,), daemon=True).start()
|
35
|
+
|
36
|
+
|
37
|
+
def init_lock_clean():
|
38
|
+
lock_clean = LockClean()
|
39
|
+
lock_clean.restart()
|
@@ -0,0 +1,28 @@
|
|
1
|
+
from typing import Union
|
2
|
+
|
3
|
+
from sqlalchemy.exc import IntegrityError
|
4
|
+
|
5
|
+
from watchmen_collector_kernel.model import CompetitiveLock
|
6
|
+
from watchmen_collector_kernel.storage.competitive_lock_service import CompetitiveLockService
|
7
|
+
from watchmen_utilities import get_current_time_in_seconds
|
8
|
+
|
9
|
+
|
10
|
+
def try_lock_nowait(lock_service: CompetitiveLockService, lock: CompetitiveLock) -> bool:
|
11
|
+
try:
|
12
|
+
lock_service.insert_one(lock)
|
13
|
+
return True
|
14
|
+
except IntegrityError:
|
15
|
+
return False
|
16
|
+
|
17
|
+
|
18
|
+
def unlock(lock_service: CompetitiveLockService, lock: CompetitiveLock) -> bool:
|
19
|
+
lock_service.delete_by_id(lock.lockId)
|
20
|
+
return True
|
21
|
+
|
22
|
+
|
23
|
+
def get_resource_lock(lock_id: int, resource_id: Union[str, int], tenant_id: str) -> CompetitiveLock:
|
24
|
+
return CompetitiveLock(
|
25
|
+
lockId=lock_id,
|
26
|
+
resourceId=resource_id,
|
27
|
+
registeredAt=get_current_time_in_seconds(),
|
28
|
+
tenantId=tenant_id)
|