dapu 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dapu/__init__.py +0 -0
- dapu/agents/__init__.py +0 -0
- dapu/agents/agent.py +152 -0
- dapu/agents/agent_dig.py +80 -0
- dapu/agents/agent_drop.py +65 -0
- dapu/agents/agent_gen.py +167 -0
- dapu/agents/agent_import.py +200 -0
- dapu/agents/agent_map.py +34 -0
- dapu/agents/agent_runsql.py +21 -0
- dapu/agents/agent_stream.py +160 -0
- dapu/context.py +169 -0
- dapu/downloader.py +86 -0
- dapu/enabler.py +185 -0
- dapu/job.py +260 -0
- dapu/jobstatus.py +19 -0
- dapu/manager.py +282 -0
- dapu/observer.py +135 -0
- dapu/perks.py +520 -0
- dapu/process.py +263 -0
- dapu/registrar.py +553 -0
- dapu/util_yaml.py +30 -0
- dapu/ver/__init__.py +0 -0
- dapu/ver/changes.yaml +10 -0
- dapu/ver/v000_a_dba.sql +32 -0
- dapu/ver/v000_b_ext.sql +27 -0
- dapu/ver/v001_schema.sql +42 -0
- dapu/ver/v002_registry.sql +182 -0
- dapu/ver/v003_stopper.sql +14 -0
- dapu/ver/v004_route.sql +4 -0
- dapu/ver/v005_to_date.sql +31 -0
- dapu/ver/v006_date.sql +9 -0
- dapu/ver/v007_overflow.sql +78 -0
- dapu/worker.py +229 -0
- dapu-0.0.2.dist-info/METADATA +178 -0
- dapu-0.0.2.dist-info/RECORD +38 -0
- dapu-0.0.2.dist-info/WHEEL +4 -0
- dapu-0.0.2.dist-info/licenses/LICENSE copy.txt +10 -0
- dapu-0.0.2.dist-info/licenses/LICENSE.txt +1 -0
dapu/__init__.py
ADDED
|
File without changes
|
dapu/agents/__init__.py
ADDED
|
File without changes
|
dapu/agents/agent.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
from dapu.process import logging
|
|
4
|
+
from dapu.context import DapuContext
|
|
5
|
+
from dapu.perks import make_replacements, read_sql_from_file
|
|
6
|
+
|
|
7
|
+
class AgentGeneric:
|
|
8
|
+
|
|
9
|
+
def __init__(self, task_id: str, action: dict, context: DapuContext, route_alias: str):
|
|
10
|
+
self.task_id: str = task_id
|
|
11
|
+
self.action: dict = action
|
|
12
|
+
self.context: DapuContext = context
|
|
13
|
+
self.target_alias: str = self.context.TARGET_ALIAS
|
|
14
|
+
self.route_alias: str = route_alias
|
|
15
|
+
self.temp_dir: str = self.find_temp_dir_path()
|
|
16
|
+
self.task_dir: str = self.find_task_dir_path(must_exists=True) # {work_dir}/{route}/{schema}/{table}, where resides haulwork.yaml
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def do_action(self) -> bool:
|
|
20
|
+
"""
|
|
21
|
+
Main action. Must be defined in subclass.
|
|
22
|
+
"""
|
|
23
|
+
logging.error("You are using too general class or subclass don't have do_action() implemented")
|
|
24
|
+
return False
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def collect_files(self, file_element: str='file') -> list[str]:
|
|
28
|
+
"""
|
|
29
|
+
Collects files mentioned in self.action from some tag/element (which values are dual-type: string or list of strings)
|
|
30
|
+
File_element name from YAML definition, usually "file". If agents supports files for different purposes then more clear name
|
|
31
|
+
eg. "create", "imput", "pre_shadow", "post_process"
|
|
32
|
+
Order of files (full path) is as in definition file
|
|
33
|
+
"""
|
|
34
|
+
existing_files = [] # return list, may remine empty
|
|
35
|
+
if file_element not in self.action:
|
|
36
|
+
logging.debug(f"No element {file_element} in action definition") # don't need to be error (consumer decides)
|
|
37
|
+
return []
|
|
38
|
+
def_files = []
|
|
39
|
+
def_files_part = self.action.get(file_element, None) # short names
|
|
40
|
+
if def_files_part is not None:
|
|
41
|
+
if isinstance(def_files_part, str):
|
|
42
|
+
def_files.append(def_files_part) # teeme stringist listi
|
|
43
|
+
else:
|
|
44
|
+
if isinstance(def_files_part, list):
|
|
45
|
+
def_files = def_files_part
|
|
46
|
+
|
|
47
|
+
for file_name in def_files:
|
|
48
|
+
file_local = os.path.join(self.task_dir, file_name)
|
|
49
|
+
if os.path.exists(file_local):
|
|
50
|
+
existing_files.append(file_local) # full file name
|
|
51
|
+
|
|
52
|
+
return existing_files # samas järjekoras, mis def failis, aga ainult need, mis eksisteerivad ka
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def apply_files_to_target(self, existing_files: list[str], replacements: list[tuple[str, str]] | None) -> bool:
|
|
56
|
+
"""
|
|
57
|
+
Name makes very clear that we never apply changes to source-databases, only to one single target-database
|
|
58
|
+
Files are applied in order they are in list.
|
|
59
|
+
List contains full paths of file names.
|
|
60
|
+
Empty and problematic files return False as failure. First error breaks (returns False)
|
|
61
|
+
Replacements is list of tuple (find -> replace)
|
|
62
|
+
'"""
|
|
63
|
+
for file_full_name in existing_files:
|
|
64
|
+
# Read file and make replacements
|
|
65
|
+
sql: str = make_replacements(read_sql_from_file(file_full_name), replacements) # does logging
|
|
66
|
+
if sql is None:
|
|
67
|
+
return False # break on first error
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
logging.debug(f"Will apply SQL from file {file_full_name}")
|
|
71
|
+
self.context.target(sql, False)
|
|
72
|
+
except Exception as e1:
|
|
73
|
+
logging.error(f"Applying SQL file failed ({file_full_name})")
|
|
74
|
+
#logging.error(f"{sql}") # deep logging or local logging only (too unsafe for remote logging)
|
|
75
|
+
logging.error(e1)
|
|
76
|
+
return False # break on first error
|
|
77
|
+
logging.debug(f"SQL file {file_full_name} applied")
|
|
78
|
+
return True
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def find_temp_dir_path(self) -> str:
|
|
82
|
+
"""
|
|
83
|
+
Encapsulated logic for calculating project temp-dir. It may reside inside target-dir (work_dir),
|
|
84
|
+
# os.path.realpath(os.path.join(self.context.work_dir, 'tmp'))
|
|
85
|
+
which is good for maintenance (quickly finding files) and cleaning process (delete all temp files for target),
|
|
86
|
+
but so this may spoil files for local (dev, which is git-based) and git-based deploy, so conflicts may emerge.
|
|
87
|
+
May-be we stick op.system temp directory and create under it app-named (dapu) and target-named dir
|
|
88
|
+
assuming, that actual deploys are with short living hard-disks (like pods). So we don't need to worry much about
|
|
89
|
+
files remaining there due failures.
|
|
90
|
+
Just remember that our temp-files must be accessible inside one process
|
|
91
|
+
(we dont need to keep them after failure or any other possible ends)
|
|
92
|
+
but one process may execute many independent Agents, so they all must know path same way.
|
|
93
|
+
Files inside temp dir are named as {task_id}.dat -> one temp dir for target, all dat-files inside
|
|
94
|
+
"""
|
|
95
|
+
system_temp_dir = tempfile.gettempdir() # https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir
|
|
96
|
+
target_name = os.path.split(self.context.work_dir)[1]
|
|
97
|
+
# temp_path_for_app = os.path.join(system_temp_dir, self.context.app_name)
|
|
98
|
+
# should we need to create intermediate dir? no, we are using os.makedirs later
|
|
99
|
+
temp_path_for_target = os.path.join(system_temp_dir, self.context.app_name, target_name)
|
|
100
|
+
logging.debug(f"Temporary directory path for target is {temp_path_for_target}, if needed")
|
|
101
|
+
return temp_path_for_target
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def find_task_dir_path(self, must_exists: bool=False) -> str:
|
|
105
|
+
"""
|
|
106
|
+
Full path from current (internal) task_id (gives 3 directories) and self.context root path (work_dir)
|
|
107
|
+
"""
|
|
108
|
+
if not self.task_id:
|
|
109
|
+
logging.error(f"Empty task_id {self.task_id}")
|
|
110
|
+
return None
|
|
111
|
+
if self.context.work_dir is None:
|
|
112
|
+
logging.error(f"Context work_dir is missing for task {self.task_id}")
|
|
113
|
+
return None
|
|
114
|
+
path_way: list = self.task_id.split('.')
|
|
115
|
+
if len(path_way) < 3:
|
|
116
|
+
logging.error(f"Too short task_id {self.task_id}")
|
|
117
|
+
return None
|
|
118
|
+
path: str = self.context.full_name_from_pull(path_way)
|
|
119
|
+
if must_exists and not os.path.exists(path):
|
|
120
|
+
logging.error(f"Path {path} for task '{self.task_id}' not exists in local file system")
|
|
121
|
+
return None
|
|
122
|
+
logging.debug(f"Dir for task {self.task_id} is {path}")
|
|
123
|
+
return path
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def make_temp_dir(self) -> bool:
|
|
127
|
+
"""
|
|
128
|
+
Lets make temp dir on first demand using system-wide logic for path
|
|
129
|
+
Returns False only if creation failed silently
|
|
130
|
+
"""
|
|
131
|
+
if os.path.exists(self.temp_dir):
|
|
132
|
+
return True
|
|
133
|
+
# Not exists, lets create
|
|
134
|
+
nix_mode_owner_group: int = 0o770 # Lets the new dir have full rights for process and its group
|
|
135
|
+
#os.mkdir(self.temp_dir, mode=nix_mode_owner_group) -- this one dont make missing dirs
|
|
136
|
+
os.makedirs(self.temp_dir, mode=nix_mode_owner_group, exist_ok=True)
|
|
137
|
+
return os.path.exists(self.temp_dir) # Does it exists after creation
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def find_task_temp_file_name(self, extension:str ="dat") -> str | None:
|
|
141
|
+
"""
|
|
142
|
+
Temporary file for external data outside of git-dir for task_id that current object already knows
|
|
143
|
+
"""
|
|
144
|
+
if self.make_temp_dir(): # checks if temp-dir exists, creates
|
|
145
|
+
if extension is None:
|
|
146
|
+
extension = ''
|
|
147
|
+
file_name = self.task_id + '.' + extension
|
|
148
|
+
return os.path.join(self.temp_dir, file_name)
|
|
149
|
+
else:
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
dapu/agents/agent_dig.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
|
|
2
|
+
from dapu.agents.agent import AgentGeneric
|
|
3
|
+
from dapu.process import logging
|
|
4
|
+
from dapu.perks import split_task_id
|
|
5
|
+
|
|
6
|
+
class Agent(AgentGeneric):
|
|
7
|
+
"""
|
|
8
|
+
Dig - dig source database to find possible tables to import and prepare definitions and files (those must put to target database (in meta) as blob with zip
|
|
9
|
+
Needs: task_id => schema.table, eg. dig_asjur5.the_table, dig_asjur5.the_column
|
|
10
|
+
=> route (from_asjur5) => route_alias
|
|
11
|
+
first "file" (tabels.sql) -- select for ASA metaquery (columns must correspond to the_table structure)
|
|
12
|
+
|
|
13
|
+
NB! See dig, mis võtab ja panem metadata kirjetena 2-3 tabelisse on tegelikult tehtav tavalise pump abil
|
|
14
|
+
St tuleb teha iga src baasi jaoks 2-3 tabelit ja 2-3 laadimisülesannet
|
|
15
|
+
saame tuvastada muudatused (__hist tabelisse nt, või changes tabelisse)
|
|
16
|
+
-- see võib olla hea enda juurde muudtuste tegemiseks
|
|
17
|
+
Aga dig tuleks teha ümber selleks, et kes võtab õiged metatabelid ja loob failide kupatuse... mille paneme... kuhu? meta.file_like
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
yaml:
|
|
21
|
+
once: true
|
|
22
|
+
until_first_error: true
|
|
23
|
+
schemas_cmd: schemas.sql
|
|
24
|
+
tables_in_schema_cmd: tables.sql
|
|
25
|
+
columns_in_table_cmd: columns.sql
|
|
26
|
+
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def do_action(self) -> bool:
|
|
31
|
+
|
|
32
|
+
_, target_schema, target_table = split_task_id(self.task_id)
|
|
33
|
+
file_element = 'file'
|
|
34
|
+
existing_files = self.collect_files(file_element)
|
|
35
|
+
if not existing_files:
|
|
36
|
+
logging.error(f"No files for tag {file_element} in {self.task_dir}")
|
|
37
|
+
return False
|
|
38
|
+
|
|
39
|
+
source_select = self.read_sql_from_file(existing_files[0], []) # siin on asenduseks nt in schema names -- millistest vaja, või kõik?
|
|
40
|
+
|
|
41
|
+
result_set = self.hub.run(self.route_alias, source_select)
|
|
42
|
+
|
|
43
|
+
if not result_set:
|
|
44
|
+
logging.warning(f"No data from {self.route_alias}")
|
|
45
|
+
return True
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
prepare_temp = f"""DROP TABLE IF EXISTS current_import"""
|
|
49
|
+
logging.debug(prepare_temp)
|
|
50
|
+
self.hub.run(self.target_alias, prepare_temp, False)
|
|
51
|
+
|
|
52
|
+
create_temp = f"""CREATE TEMP TABLE current_import
|
|
53
|
+
(schema_id integer, schema_name varchar(100), table_id integer, table_name varchar(100), table_comment text)"""
|
|
54
|
+
logging.debug(create_temp)
|
|
55
|
+
self.hub.run(self.target_alias, create_temp, False)
|
|
56
|
+
|
|
57
|
+
for row in result_set:
|
|
58
|
+
sql_ins = f"""INSERT INTO current_import(schema_id, schema_name, table_id, table_name, table_comment)
|
|
59
|
+
VALUES ({row[0]}, '{row[1]}', {row[2]}, '{row[3]}', '{row[4]}') """
|
|
60
|
+
self.hub.run(self.target_alias, sql_ins, False)
|
|
61
|
+
|
|
62
|
+
# compare -- simple variant (delete and insert)
|
|
63
|
+
sql_truncate = f"""DELETE FROM {target_schema}.{target_table} WHERE true"""
|
|
64
|
+
logging.debug(sql_truncate)
|
|
65
|
+
self.hub.run(self.target_alias, sql_truncate, False)
|
|
66
|
+
|
|
67
|
+
sql_refresh = f"""INSERT INTO {target_schema}.{target_table} (schema_id, schema_name, table_id, table_name, table_comment)
|
|
68
|
+
SELECT schema_id, schema_name, table_id, table_name, table_comment FROM current_import"""
|
|
69
|
+
logging.debug(sql_refresh)
|
|
70
|
+
self.hub.run(self.target_alias, sql_refresh, False)
|
|
71
|
+
|
|
72
|
+
logging.debug(prepare_temp)
|
|
73
|
+
self.hub.run(self.target_alias, prepare_temp, False) # polite!
|
|
74
|
+
|
|
75
|
+
logging.info('done')
|
|
76
|
+
return True
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from dapu.agents.agent import AgentGeneric
|
|
2
|
+
from dapu.process import logging
|
|
3
|
+
from dapu.perks import split_task_id
|
|
4
|
+
|
|
5
|
+
class Agent(AgentGeneric):
|
|
6
|
+
"""
|
|
7
|
+
drop - töötab tulemtabeliga ja teeb Postgre kiiksuga
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
def do_action(self) -> bool:
|
|
11
|
+
_, target_schema, target_table = split_task_id(self.task_id)
|
|
12
|
+
sql = self.make_drop_command(self.target_alias, target_schema, target_table, with_cascade=True)
|
|
13
|
+
if sql > '':
|
|
14
|
+
try:
|
|
15
|
+
logging.info(f"DROP current structure (in target base)")
|
|
16
|
+
self.context.target(sql, False)
|
|
17
|
+
except Exception as e1:
|
|
18
|
+
msg = f"Failed DROP command: {sql}"
|
|
19
|
+
self.last_error = msg
|
|
20
|
+
logging.error(msg)
|
|
21
|
+
return False
|
|
22
|
+
|
|
23
|
+
# tühja SQL-i ei rakenda ja loeme selle õnnestumiseks
|
|
24
|
+
return True
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def make_drop_command(self, target_alias, target_schema: str, target_table: str, with_cascade: bool=False) -> str:
|
|
28
|
+
"""
|
|
29
|
+
If object exists in postgre database then return sql command for drop corresponding to object type (table, view, etc)
|
|
30
|
+
Needs database alias
|
|
31
|
+
"""
|
|
32
|
+
# Tulembaasis (alati PostrgeSQL) vaja minev tegevus (õiget tüüpi drop käsk)
|
|
33
|
+
# Postgre metainfo päring, kas objekt on olemas ja mis tüüpi
|
|
34
|
+
|
|
35
|
+
command = '' # jääb tühjaks, kui pole vaja droppida
|
|
36
|
+
|
|
37
|
+
sql = f"""
|
|
38
|
+
SELECT
|
|
39
|
+
CASE
|
|
40
|
+
WHEN c.relkind = 'r' THEN 'TABLE'
|
|
41
|
+
WHEN c.relkind = 'v' THEN 'VIEW'
|
|
42
|
+
WHEN c.relkind = 'm' THEN 'MATERIALIZED VIEW'
|
|
43
|
+
ELSE 'ZZ' -- our uncleariness-marker
|
|
44
|
+
END,
|
|
45
|
+
c.relkind -- for debuging unclear state in far future
|
|
46
|
+
FROM pg_class c
|
|
47
|
+
JOIN pg_namespace tns ON tns.oid = c.relnamespace
|
|
48
|
+
WHERE tns.nspname = '{target_schema}' AND c.relname = '{target_table}'
|
|
49
|
+
UNION SELECT 'XX' -- our non-existence marker
|
|
50
|
+
ORDER BY 1
|
|
51
|
+
"""
|
|
52
|
+
result_set = self.context.target(sql)
|
|
53
|
+
|
|
54
|
+
if result_set:
|
|
55
|
+
unit = (result_set[0][0]).upper() # unit on "TABLE", "VIEW", "MATERIALIZED VIEW"
|
|
56
|
+
if unit not in ('XX', 'ZZ'):
|
|
57
|
+
command = f'DROP {unit} IF EXISTS {target_schema}.{target_table}'
|
|
58
|
+
if with_cascade:
|
|
59
|
+
command = command + ' CASCADE'
|
|
60
|
+
if unit == 'ZZ':
|
|
61
|
+
logging.warning("Unclear tabular type: {result_set[0][1]}")
|
|
62
|
+
|
|
63
|
+
return command
|
|
64
|
+
|
|
65
|
+
|
dapu/agents/agent_gen.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This is a Agent in Dapu framework, so dapu.perks is awailable
|
|
3
|
+
"""
|
|
4
|
+
import os
|
|
5
|
+
from dapu.agents.agent import AgentGeneric
|
|
6
|
+
from dapu.process import logging
|
|
7
|
+
from dapu.perks import split_task_id
|
|
8
|
+
from dapu.perks import generate, calculate_dir_hash, temporary_dir_context, content_to_file
|
|
9
|
+
from dapu.perks import make_column_creation_expression, type_mapper_code_2_function, compress_temp_dir
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Agent(AgentGeneric):
|
|
13
|
+
"""
|
|
14
|
+
Generator (for hauling definitions)
|
|
15
|
+
Assumes that metadata is already grabbed into PG, selects those from own database (not original source database)
|
|
16
|
+
Uses two SQL: first for all tables, second for columns inside one table (by table name, markup)
|
|
17
|
+
both SQLs have schema name fixed (TODO later to variable/multi-schema) - for ASA database usually 'dba' as schema
|
|
18
|
+
|
|
19
|
+
Puts result into target table as new line if there are no similar line (route_code, schema_name, table_name, package_hash)
|
|
20
|
+
|
|
21
|
+
Result will be generated using 3 template files -- can we make it more dynamic?
|
|
22
|
+
haulwork.yaml
|
|
23
|
+
input.sql
|
|
24
|
+
ver/001.sql
|
|
25
|
+
|
|
26
|
+
And ZIP-ed into correct structure
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def get_file_content_by_action_reference(self, key_name_in_action: str) -> str | None:
|
|
30
|
+
"""
|
|
31
|
+
Reads text from file pointed from definition, if not str but list then first file
|
|
32
|
+
"""
|
|
33
|
+
existing_files = self.collect_files(key_name_in_action)
|
|
34
|
+
if not existing_files:
|
|
35
|
+
logging.error(f"No files for tag {key_name_in_action} in {self.task_dir}")
|
|
36
|
+
return None
|
|
37
|
+
return self.read_sql_from_file(existing_files[0], [])
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def prechecks(self) -> bool:
|
|
41
|
+
target_info : dict = self.action.get('target')
|
|
42
|
+
if target_info is None:
|
|
43
|
+
logging.error(f"No target info in definition")
|
|
44
|
+
return False
|
|
45
|
+
|
|
46
|
+
route_code = target_info.get('route') # route for generated task (not for task which uses agent_gen)
|
|
47
|
+
if route_code is None:
|
|
48
|
+
logging.error(f"No route info in definition")
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
schema_name = target_info.get('schema') # schema for generated task (schema in target database)
|
|
52
|
+
if schema_name is None:
|
|
53
|
+
logging.error(f"No schema info in definition")
|
|
54
|
+
return False
|
|
55
|
+
|
|
56
|
+
if 'templates' in self.action:
|
|
57
|
+
for template in ['def', 'sql', 'ver']:
|
|
58
|
+
if template not in self.action['templates']:
|
|
59
|
+
logging.error(f"Missing {template}-template in definition")
|
|
60
|
+
return False
|
|
61
|
+
else:
|
|
62
|
+
logging.error(f"No templates info in definition")
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
return True # everything looks fine
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def do_action(self) -> bool:
|
|
69
|
+
# result files names
|
|
70
|
+
result_name_sql = 'input.sql'
|
|
71
|
+
result_name_def = 'haulwork.yaml'
|
|
72
|
+
result_name_ver = '001.sql'
|
|
73
|
+
ver_sub_folder = 'ver'
|
|
74
|
+
_, target_schema, target_table = split_task_id(self.task_id)
|
|
75
|
+
|
|
76
|
+
if not self.prechecks(): # checks existence next keys but dont read them, so read it after (no check needed later)
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
if (tables_select := self.get_file_content_by_action_reference('table_file')) is None: # walrus operator py 3.8
|
|
80
|
+
return False
|
|
81
|
+
# LETS DEMAND: SELECT table_name, table_comment, table_id
|
|
82
|
+
|
|
83
|
+
if (columns_select := self.get_file_content_by_action_reference('columns_file')) is None:
|
|
84
|
+
return False
|
|
85
|
+
# LETS DEMAND: SELECT col_pos, col_name, col_type, col_width, col_scale, col_primary, col_comment
|
|
86
|
+
|
|
87
|
+
fully_excluded_file_list = [] # täielikult välistatud tabelid
|
|
88
|
+
if 'exclude' in self.action:
|
|
89
|
+
if 'full' in self.action['exclude']:
|
|
90
|
+
fully_excluded_file_list = self.action['exclude']['full']
|
|
91
|
+
|
|
92
|
+
route_code = self.action['target']['route'] # route to use inside generated target
|
|
93
|
+
schema_name = self.action['target']['schema']
|
|
94
|
+
# file names
|
|
95
|
+
template_name_def = self.action['templates']['def']
|
|
96
|
+
template_name_ver = self.action['templates']['ver']
|
|
97
|
+
template_name_sql = self.action['templates']['sql']
|
|
98
|
+
fn_mapper = type_mapper_code_2_function(self.action.get('type_mapper'))
|
|
99
|
+
|
|
100
|
+
result_set = self.hub.run(self.route_alias, tables_select) # result_tuples
|
|
101
|
+
|
|
102
|
+
if not result_set:
|
|
103
|
+
logging.warning(f"No data from {self.route_alias}")
|
|
104
|
+
return True
|
|
105
|
+
|
|
106
|
+
added: int = 0
|
|
107
|
+
for row in result_set: # for each table
|
|
108
|
+
table_name = row[0]
|
|
109
|
+
|
|
110
|
+
if table_name in fully_excluded_file_list:
|
|
111
|
+
continue
|
|
112
|
+
logging.debug(f" Working with table {table_name}")
|
|
113
|
+
|
|
114
|
+
sql = columns_select # always take fresh one where tags are not replaced yet
|
|
115
|
+
for replacement in [('{{table_name}}', table_name)]:
|
|
116
|
+
sql = sql.replace(replacement[0], replacement[1])
|
|
117
|
+
col_tuples = self.hub.run(self.route_alias, sql)
|
|
118
|
+
select_columns_str = ", ".join([col_tuple[1] for col_tuple in col_tuples])
|
|
119
|
+
create_columns_str = "\n, ".join([make_column_creation_expression(col_tuple, fn_mapper) for col_tuple in col_tuples])
|
|
120
|
+
|
|
121
|
+
my_data : dict = {
|
|
122
|
+
"route_code" : route_code
|
|
123
|
+
, "schema_name" : schema_name
|
|
124
|
+
, "table_name" : table_name
|
|
125
|
+
, "columns" : col_tuples
|
|
126
|
+
, "select_columns" : select_columns_str
|
|
127
|
+
, "create_columns" : create_columns_str
|
|
128
|
+
, "select_file_name" : result_name_sql
|
|
129
|
+
}
|
|
130
|
+
content_def = generate(my_data, self.task_dir, template_name_def)
|
|
131
|
+
content_ver = generate(my_data, self.task_dir, template_name_ver)
|
|
132
|
+
content_select = generate(my_data, self.task_dir, template_name_sql)
|
|
133
|
+
|
|
134
|
+
# lets make file-hierarhy
|
|
135
|
+
# assuming we are in haul_dir
|
|
136
|
+
# haulwork.yaml, input.sql, ver/001.sql
|
|
137
|
+
package_hash = None # lõppsaaduseks oleva zip-faili räsi (mida PG insert-conflict arvestab)
|
|
138
|
+
with temporary_dir_context() as content_temp_dir:
|
|
139
|
+
logging.debug(f"Source files to zip later will be put into {content_temp_dir}")
|
|
140
|
+
# save prepared text-content to files in new temporary folder
|
|
141
|
+
content_to_file(content_def, content_temp_dir, result_name_def)
|
|
142
|
+
content_to_file(content_select, content_temp_dir, result_name_sql)
|
|
143
|
+
content_to_file(content_ver, os.path.join(content_temp_dir, ver_sub_folder), result_name_ver)
|
|
144
|
+
|
|
145
|
+
package_hash = calculate_dir_hash(content_temp_dir) # always calculate hash with own method to get same hash for same content
|
|
146
|
+
logging.debug(f"Hash of temp-dir content is {package_hash}")
|
|
147
|
+
package_file_hex = compress_temp_dir(content_temp_dir)
|
|
148
|
+
# end with -- temporary files in temporary dir
|
|
149
|
+
|
|
150
|
+
if not package_hash:
|
|
151
|
+
continue
|
|
152
|
+
|
|
153
|
+
sql_ins = f"""INSERT INTO {target_schema}.{target_table}
|
|
154
|
+
(route_code, schema_name, table_name, package_hash, package_file)
|
|
155
|
+
VALUES ('{route_code}', '{schema_name}', '{table_name}', '{package_hash}', decode('{package_file_hex}', 'hex')::bytea)
|
|
156
|
+
ON CONFLICT ON CONSTRAINT ak_{target_table}_4_uniq DO NOTHING
|
|
157
|
+
RETURNING id """
|
|
158
|
+
new_row_with_id = self.hub.run(self.target_alias, sql_ins)
|
|
159
|
+
if new_row_with_id:
|
|
160
|
+
added += 1
|
|
161
|
+
logging.info(f"Generation of {table_name} is done {new_row_with_id[0][0]}")
|
|
162
|
+
|
|
163
|
+
logging.info(f"Done generation of {added} items")
|
|
164
|
+
return True
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
|
|
2
|
+
from dapu.agents.agent import AgentGeneric
|
|
3
|
+
from dapu.process import logging
|
|
4
|
+
from dapu.perks import split_task_id
|
|
5
|
+
|
|
6
|
+
class Agent(AgentGeneric):
|
|
7
|
+
"""
|
|
8
|
+
Take all data from source and ...
|
|
9
|
+
a) put into temp file
|
|
10
|
+
b) stream into target table -> for that is agent_stream
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def get_last_ts(self) -> str:
|
|
17
|
+
task_id = self.task_id
|
|
18
|
+
registry = 'meta.registry'
|
|
19
|
+
sql_last_ts = f"""SELECT to_char(coalesce(synced_until_ts, '1990-01-01 00:00:00'), 'YYYY-MM-DD HH24:MI:SS')
|
|
20
|
+
FROM {registry} WHERE task_id = '{task_id}' """
|
|
21
|
+
result_set = self.hub.run(self.target_alias, sql_last_ts)
|
|
22
|
+
return result_set[0][0] # return ISO DATE/TIME as string
|
|
23
|
+
|
|
24
|
+
def do_action(self, task_id, action_def, task_dir): # FIXME -- 2 do_Action fn!!! which one is correct??
|
|
25
|
+
_, target_schema, target_table = split_task_id(self.task_id)
|
|
26
|
+
file_element = 'file'
|
|
27
|
+
existing_files = self.collect_files(file_element)
|
|
28
|
+
if not existing_files:
|
|
29
|
+
logging.error(f"No files for tag {file_element} in {self.task_dir}")
|
|
30
|
+
return False
|
|
31
|
+
|
|
32
|
+
replacements = []
|
|
33
|
+
# this select query can be modified knowing source stuff (pagination? start TS?)
|
|
34
|
+
# for ts we need last ts saved to us
|
|
35
|
+
# for maxid we need ...
|
|
36
|
+
# for hash source should be pg? why? sql must include hash expression of source dbms
|
|
37
|
+
method = self.action.get('changedetection', None)
|
|
38
|
+
# method determines what data we need from our side before to start asking from source
|
|
39
|
+
match method: # 3.10+
|
|
40
|
+
case 'ts': # we need last sync ts and put it sql in place of ts_time
|
|
41
|
+
ts_time = self.get_last_ts()
|
|
42
|
+
replacements.append(('{ts_time}', ts_time))
|
|
43
|
+
case _:
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
source_select = self.read_sql_from_file(existing_files[0], replacements)
|
|
49
|
+
result_set = self.hub.run(self.route_alias, source_select) # SIIA run asemel savetofile + filename
|
|
50
|
+
# ja sellel pole väljundit (või siis on true false või kirjete arv)
|
|
51
|
+
if not result_set:
|
|
52
|
+
logging.warning(f"No data from {self.route_alias}")
|
|
53
|
+
return True
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
pure_sql = sql
|
|
59
|
+
|
|
60
|
+
destination = 'file'
|
|
61
|
+
if 'to' in action_def:
|
|
62
|
+
destination = action_def['to']
|
|
63
|
+
|
|
64
|
+
# ajutine fail, kuhu panna tõmmatud andmed
|
|
65
|
+
if destination == 'file':
|
|
66
|
+
task_temp_file_name = self.find_task_temp_file_name()
|
|
67
|
+
self.log('DEBUG', f'ajutine fail saab olema {task_temp_file_name}')
|
|
68
|
+
|
|
69
|
+
tulem = True
|
|
70
|
+
while True:
|
|
71
|
+
try:
|
|
72
|
+
sql = pure_sql
|
|
73
|
+
replacements = [(self.context.PLACEHOLDER_TARGET_SCHEMA, f'{schema_name}'), (self.context.PLACEHOLDER_TARGET_TABLE, f'{table_name}')]
|
|
74
|
+
|
|
75
|
+
rs = bm.run(idx, sql, True, True) # commit pole vist oluline
|
|
76
|
+
self.log('INFO', f'Andmed mälus ({len(rs)} kirjet)')
|
|
77
|
+
except Exception as e1:
|
|
78
|
+
self.log('ERROR', f'Andmete tõmbamine nurjus ({schema_name}.{table_name}): {bm.last_error}')
|
|
79
|
+
self.log('ERROR', f'{e1}')
|
|
80
|
+
return False # esimese vigase faili peale katkestame
|
|
81
|
+
|
|
82
|
+
if len(rs) == 0:
|
|
83
|
+
self.log('INFO', f'Polnud kirjeid ({schema_name}.{table_name})')
|
|
84
|
+
# teeme tühja faili kui on kartus, et keegi järgmisena tahab seda faili kasvõi näha
|
|
85
|
+
if destination == 'file':
|
|
86
|
+
with open(task_temp_file_name, 'w') as sf:
|
|
87
|
+
sf.write("")
|
|
88
|
+
break # kui ei saadud kirjeid, siis näib kõik kena ja läheme tsüklist välja
|
|
89
|
+
|
|
90
|
+
if destination == 'file':
|
|
91
|
+
# rs on nüüd data-ga, vaja salvestada PG jaoks sobivale kujule
|
|
92
|
+
self.log('INFO', f'salvestame andmed ({len(rs)}) ajutisse faili')
|
|
93
|
+
tulem = self.save_data_for_postgre(task_temp_file_name, rs) # kas see on piisav info, vb on vaja null jms?
|
|
94
|
+
break # failikirjutamisel ei ole korduseid
|
|
95
|
+
|
|
96
|
+
if destination == 'direct':
|
|
97
|
+
self.log('INFO', f'käime saadud kirjed ({len(rs)}) insert+conflict abil üle')
|
|
98
|
+
all_cols_array = action_def['cols']
|
|
99
|
+
all_cols = ','.join(all_cols_array)
|
|
100
|
+
#all_cols_array = all_cols.split(',') # tühiku trimime/stripime hiljem võrdluse ajal
|
|
101
|
+
# mitmes veerg on ts_col kõigi hulgast (tegelikult mitte kõigi hulgast, vaid input.sql sees oleva selecti veergude hulgast, aga püüame ühtsena hoida
|
|
102
|
+
ts_col_pos = -1
|
|
103
|
+
if method == 'ts':
|
|
104
|
+
all_col_pos = 0
|
|
105
|
+
for col in all_cols_array:
|
|
106
|
+
if col.strip() == ts_col:
|
|
107
|
+
ts_col_pos = all_col_pos # progr-indeks (st algab 0-st, increment toimub lõpus)
|
|
108
|
+
all_col_pos = all_col_pos + 1
|
|
109
|
+
if ts_col > '' and ts_col_pos == -1:
|
|
110
|
+
ts_col_pos = all_col_pos # viimane veerg, peale sisulisi data veerge, mis kantakse meile
|
|
111
|
+
esimene = True
|
|
112
|
+
for jarjekorranumber, this_row in enumerate(rs):
|
|
113
|
+
if method == 'ts':
|
|
114
|
+
# mitmes veerg on TS veerg? eh?
|
|
115
|
+
last_ts = this_row[ts_col_pos]
|
|
116
|
+
if last_ts is None:
|
|
117
|
+
# FATAL!?!
|
|
118
|
+
self.log('ERROR', f'Viimase muutuse veeru väärtus on NULL, mida ei saa mitte heaks kiita')
|
|
119
|
+
continue # või break, või return???
|
|
120
|
+
value_part_str = self.make_insert_value_part(this_row, len(all_cols_array), esimene)
|
|
121
|
+
conflict_set_part = self.make_insert_conflict_set_part(all_cols_array) # panna välistamise massiiv ka? pkColsArray
|
|
122
|
+
sql_ins_upd = f'''
|
|
123
|
+
INSERT INTO {schema_name}.{table_name} ({all_cols}) VALUES ({value_part_str})
|
|
124
|
+
ON CONFLICT ON CONSTRAINT ak_{table_name}_4_uniq DO UPDATE
|
|
125
|
+
SET {conflict_set_part}
|
|
126
|
+
'''
|
|
127
|
+
bm.run(idx_trg, sql_ins_upd, False, True)
|
|
128
|
+
if method == 'ts':
|
|
129
|
+
sql_upd_ts = f'''UPDATE bis.bis_registry SET synced_until = '{last_ts}' WHERE task_id = '{task_id}' '''
|
|
130
|
+
bm.run(idx_trg, sql_upd_ts, False, True) # teeme iga kirje järel, see aeglustab, aga võimaldab katkiminemisi paremini hallata
|
|
131
|
+
esimene = False
|
|
132
|
+
|
|
133
|
+
if method == 'ts':
|
|
134
|
+
sql_need = f'''SELECT case when '{last_ts}'::timestamp + '2 minutes'::interval > current_timestamp THEN 0 ELSE 1 END'''
|
|
135
|
+
need_for_more_rs = bm.run(idx_trg, sql_need, True, True)
|
|
136
|
+
if need_for_more_rs[0][0] == 0:
|
|
137
|
+
break
|
|
138
|
+
else:
|
|
139
|
+
break # kui ei olnud ajaveergu olemas (ei soovitud tuvastada aja abil muutusi), siis väljume tsüklist peale esimest läbimist
|
|
140
|
+
|
|
141
|
+
if len(rs) < 3000: # esialgne ümbertee MS SQL ajamuredele
|
|
142
|
+
break
|
|
143
|
+
|
|
144
|
+
self.log('INFO', f'ekspordisamm on tehtud')
|
|
145
|
+
return tulem
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def do_action(self) -> bool:
|
|
149
|
+
|
|
150
|
+
self.last_error = None
|
|
151
|
+
|
|
152
|
+
_, target_schema, target_table = split_task_id(self.task_id)
|
|
153
|
+
file_element = 'file'
|
|
154
|
+
existing_files = self.collect_files(file_element)
|
|
155
|
+
if not existing_files:
|
|
156
|
+
logging.error(f"No files for tag {file_element} in {self.task_dir}")
|
|
157
|
+
return False
|
|
158
|
+
|
|
159
|
+
source_select = self.read_sql_from_file(existing_files[0], []) # siin on asenduseks nt in schema names -- millistest vaja, või kõik?
|
|
160
|
+
|
|
161
|
+
result_set = self.hub.run(self.route_alias, source_select)
|
|
162
|
+
|
|
163
|
+
if not result_set:
|
|
164
|
+
logging.warning(f"No data from {self.route_alias}")
|
|
165
|
+
return True
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
prepare_temp = f"""DROP TABLE IF EXISTS current_import"""
|
|
169
|
+
logging.debug(prepare_temp)
|
|
170
|
+
self.hub.run(self.target_alias, prepare_temp, False)
|
|
171
|
+
|
|
172
|
+
create_temp = f"""CREATE TEMP TABLE current_import
|
|
173
|
+
(schema_id integer, schema_name varchar(100), table_id integer, table_name varchar(100), table_comment text)"""
|
|
174
|
+
logging.debug(create_temp)
|
|
175
|
+
self.hub.run(self.target_alias, create_temp, False)
|
|
176
|
+
|
|
177
|
+
for row in result_set:
|
|
178
|
+
sql_ins = f"""INSERT INTO current_import(schema_id, schema_name, table_id, table_name, table_comment)
|
|
179
|
+
VALUES ({row[0]}, '{row[1]}', {row[2]}, '{row[3]}', '{row[4]}') """
|
|
180
|
+
self.hub.run(self.target_alias, sql_ins, False)
|
|
181
|
+
|
|
182
|
+
# compare -- simple variant (delete and insert)
|
|
183
|
+
sql_truncate = f"""DELETE FROM {target_schema}.{target_table} WHERE true"""
|
|
184
|
+
logging.debug(sql_truncate)
|
|
185
|
+
self.hub.run(self.target_alias, sql_truncate, False)
|
|
186
|
+
|
|
187
|
+
sql_refresh = f"""INSERT INTO {target_schema}.{target_table} (schema_id, schema_name, table_id, table_name, table_comment)
|
|
188
|
+
SELECT schema_id, schema_name, table_id, table_name, table_comment FROM current_import"""
|
|
189
|
+
logging.debug(sql_refresh)
|
|
190
|
+
self.hub.run(self.target_alias, sql_refresh, False)
|
|
191
|
+
|
|
192
|
+
logging.debug(prepare_temp)
|
|
193
|
+
self.hub.run(self.target_alias, prepare_temp, False) # polite!
|
|
194
|
+
|
|
195
|
+
logging.info('done')
|
|
196
|
+
return True
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
|