dapu 0.7.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dapu/__init__.py +0 -0
- dapu/agents/__init__.py +0 -0
- dapu/agents/agent.py +144 -0
- dapu/agents/agent_dig.py +81 -0
- dapu/agents/agent_drop.py +69 -0
- dapu/agents/agent_exceltartu.py +233 -0
- dapu/agents/agent_export.py +163 -0
- dapu/agents/agent_gen.py +167 -0
- dapu/agents/agent_history.py +91 -0
- dapu/agents/agent_import.py +191 -0
- dapu/agents/agent_map.py +34 -0
- dapu/agents/agent_runsql.py +42 -0
- dapu/agents/agent_shadow.py +244 -0
- dapu/agents/agent_stream.py +211 -0
- dapu/agents.yaml +18 -0
- dapu/context.py +457 -0
- dapu/downloader.py +83 -0
- dapu/drivers.yaml +9 -0
- dapu/enabler.py +167 -0
- dapu/fileops.py +65 -0
- dapu/job.py +272 -0
- dapu/jobstatus.py +17 -0
- dapu/maintenance.py +111 -0
- dapu/manager.py +302 -0
- dapu/observer.py +133 -0
- dapu/perks.py +910 -0
- dapu/placeholder.py +16 -0
- dapu/process.py +246 -0
- dapu/registrar.py +579 -0
- dapu/server.py +132 -0
- dapu/static/__init__.py +0 -0
- dapu/static/app.png +0 -0
- dapu/static/ikoon.txt +8 -0
- dapu/templates/__init__.py +0 -0
- dapu/templates/running.html +44 -0
- dapu/textops.py +68 -0
- dapu/ver/__init__.py +0 -0
- dapu/ver/changes.yaml +22 -0
- dapu/ver/v000_a_dba.sql +43 -0
- dapu/ver/v000_b_ext.sql +27 -0
- dapu/ver/v001_schema.sql +42 -0
- dapu/ver/v002_registry.sql +182 -0
- dapu/ver/v003_stopper.sql +14 -0
- dapu/ver/v004_paused.sql +9 -0
- dapu/ver/v005_route_weight.sql +3 -0
- dapu/ver/v006_reg_disabled.sql +6 -0
- dapu/ver/xv004_route.sql +4 -0
- dapu/ver/xv005_to_date.sql +31 -0
- dapu/ver/xv006_date.sql +9 -0
- dapu/ver/xv007_overflow.sql +78 -0
- dapu/versops.py +58 -0
- dapu/worker.py +252 -0
- dapu-0.7.11.dist-info/METADATA +198 -0
- dapu-0.7.11.dist-info/RECORD +57 -0
- dapu-0.7.11.dist-info/WHEEL +4 -0
- dapu-0.7.11.dist-info/licenses/LICENSE copy.txt +10 -0
- dapu-0.7.11.dist-info/licenses/LICENSE.txt +1 -0
dapu/__init__.py
ADDED
|
File without changes
|
dapu/agents/__init__.py
ADDED
|
File without changes
|
dapu/agents/agent.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
from loguru import logger
|
|
2
|
+
import os
|
|
3
|
+
import tempfile
|
|
4
|
+
from dapu.context import DapuContext
|
|
5
|
+
from dapu.perks import make_replacements, read_sql_from_file
|
|
6
|
+
from dapu.placeholder import Placeholder
|
|
7
|
+
|
|
8
|
+
class AgentGeneric:
|
|
9
|
+
|
|
10
|
+
def __init__(self, task_id: str, action: dict, context: DapuContext, route_alias: str):
|
|
11
|
+
self.task_id: str = task_id
|
|
12
|
+
self.action: dict = action
|
|
13
|
+
self.context: DapuContext = context
|
|
14
|
+
self.target_alias: str = self.context.TARGET_ALIAS
|
|
15
|
+
self.route_alias: str = route_alias
|
|
16
|
+
self.temp_dir: str = self.find_temp_dir_path()
|
|
17
|
+
self.task_dir: str|None = self.find_task_dir_path(must_exists=True) # {work_dir}/routes/{route}/{schema}/{table}, where resides haulwork.yaml
|
|
18
|
+
|
|
19
|
+
def do_action(self) -> bool:
|
|
20
|
+
"""
|
|
21
|
+
Main action. Must be defined in subclass.
|
|
22
|
+
"""
|
|
23
|
+
logger.error("You are using too general class or subclass don't have do_action() implemented")
|
|
24
|
+
return False
|
|
25
|
+
|
|
26
|
+
def collect_files(self, file_element: str='file') -> list[str]:
|
|
27
|
+
"""
|
|
28
|
+
Collects files mentioned in self.action from some tag/element (which values are dual-type: string or list of strings)
|
|
29
|
+
File_element name from YAML definition, usually "file". If agents supports files for different purposes then more clear name
|
|
30
|
+
eg. "create", "imput", "pre_shadow", "post_process"
|
|
31
|
+
Order of files (full path) is as in definition file
|
|
32
|
+
"""
|
|
33
|
+
existing_files = [] # return list, may remine empty
|
|
34
|
+
if file_element not in self.action:
|
|
35
|
+
logger.debug(f"No element {file_element} in action definition") # don't need to be error (consumer decides)
|
|
36
|
+
return []
|
|
37
|
+
def_files = []
|
|
38
|
+
def_files_part = self.action.get(file_element, None) # short names
|
|
39
|
+
if def_files_part is not None:
|
|
40
|
+
if isinstance(def_files_part, str):
|
|
41
|
+
def_files.append(def_files_part) # teeme stringist listi
|
|
42
|
+
else:
|
|
43
|
+
if isinstance(def_files_part, list):
|
|
44
|
+
def_files = def_files_part
|
|
45
|
+
|
|
46
|
+
for file_name in def_files:
|
|
47
|
+
file_local = os.path.join(self.task_dir or ".", file_name)
|
|
48
|
+
if os.path.exists(file_local):
|
|
49
|
+
existing_files.append(file_local) # full file name
|
|
50
|
+
|
|
51
|
+
return existing_files # samas järjekoras, mis def failis, aga ainult need, mis eksisteerivad ka
|
|
52
|
+
|
|
53
|
+
def apply_files_to_target(self, existing_files: list[str], replacements: list[tuple[str | Placeholder, str]] | None) -> bool:
|
|
54
|
+
"""
|
|
55
|
+
Name makes very clear that we never apply changes to source-databases, only to one single target-database
|
|
56
|
+
Files are applied in order they are in list.
|
|
57
|
+
List contains full paths of file names.
|
|
58
|
+
Empty and problematic files return False as failure. First error breaks (returns False)
|
|
59
|
+
Replacements is list of tuples (find -> replace)
|
|
60
|
+
'"""
|
|
61
|
+
for file_full_name in existing_files:
|
|
62
|
+
# Read file and make replacements
|
|
63
|
+
sql: str | None = make_replacements(read_sql_from_file(file_full_name), replacements)# does logging
|
|
64
|
+
if sql is None:
|
|
65
|
+
return False # break on first error
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
logger.debug(f"Will apply SQL from file {file_full_name}")
|
|
69
|
+
self.context.target(sql, False)
|
|
70
|
+
except Exception as e1:
|
|
71
|
+
logger.error(f"Applying SQL file failed ({file_full_name})")
|
|
72
|
+
#logger.error(f"{sql}") # deep logging or local logging only (too unsafe for remote logging)
|
|
73
|
+
logger.error(e1)
|
|
74
|
+
return False # break on first error
|
|
75
|
+
logger.debug(f"SQL file {file_full_name} applied")
|
|
76
|
+
return True
|
|
77
|
+
|
|
78
|
+
def find_temp_dir_path(self) -> str:
|
|
79
|
+
"""
|
|
80
|
+
Encapsulated logic for calculating project temp-dir. It may reside inside target-dir (work_dir),
|
|
81
|
+
# os.path.realpath(os.path.join(self.context.work_dir, 'tmp'))
|
|
82
|
+
which is good for maintenance (quickly finding files) and cleaning process (delete all temp files for target),
|
|
83
|
+
but so this may spoil files for local (dev, which is git-based) and git-based deploy, so conflicts may emerge.
|
|
84
|
+
May-be we stick op.system temp directory and create under it app-named (dapu) and target-named dir
|
|
85
|
+
assuming, that actual deploys are with short living hard-disks (like pods). So we don't need to worry much about
|
|
86
|
+
files remaining there due failures.
|
|
87
|
+
Just remember that our temp-files must be accessible inside one process
|
|
88
|
+
(we dont need to keep them after failure or any other possible ends)
|
|
89
|
+
but one process may execute many independent Agents, so they all must know path same way.
|
|
90
|
+
Files inside temp dir are named as {task_id}.dat -> one temp dir for target, all dat-files inside
|
|
91
|
+
"""
|
|
92
|
+
system_temp_dir = tempfile.gettempdir() # https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir
|
|
93
|
+
target_name = os.path.split(self.context.work_dir)[1] # 2nd element is "tail" - everything after last (back)slash
|
|
94
|
+
# temp_path_for_app = os.path.join(system_temp_dir, self.context.APP_NAME)
|
|
95
|
+
# should we need to create intermediate dir? no, we are using os.makedirs later
|
|
96
|
+
temp_path_for_target = os.path.join(system_temp_dir, self.context.APP_NAME, target_name)
|
|
97
|
+
logger.debug(f"Temporary directory path for target is {temp_path_for_target}, if needed")
|
|
98
|
+
return temp_path_for_target
|
|
99
|
+
|
|
100
|
+
def find_task_dir_path(self, must_exists: bool=False) -> str|None:
|
|
101
|
+
"""
|
|
102
|
+
Full path from current (internal) task_id (gives 3 directories) and self.context root path (work_dir)
|
|
103
|
+
"""
|
|
104
|
+
if not self.task_id:
|
|
105
|
+
logger.error(f"Empty task_id {self.task_id}")
|
|
106
|
+
return None
|
|
107
|
+
if self.context.work_dir is None:
|
|
108
|
+
logger.error(f"Context work_dir is missing for task {self.task_id}")
|
|
109
|
+
return None
|
|
110
|
+
path_way: list = self.task_id.split('.')
|
|
111
|
+
if len(path_way) < 3:
|
|
112
|
+
logger.error(f"Too short task_id {self.task_id}")
|
|
113
|
+
return None
|
|
114
|
+
path: str = self.context.full_name_from_pull(path_way)
|
|
115
|
+
if must_exists and not os.path.exists(path):
|
|
116
|
+
logger.error(f"Path {path} for task '{self.task_id}' not exists in local file system")
|
|
117
|
+
return None
|
|
118
|
+
logger.debug(f"Dir for task {self.task_id} is {path}")
|
|
119
|
+
return path
|
|
120
|
+
|
|
121
|
+
def make_temp_dir(self) -> bool:
|
|
122
|
+
"""
|
|
123
|
+
Lets make temp dir on first demand using system-wide logic for path
|
|
124
|
+
Returns False only if creation failed silently
|
|
125
|
+
"""
|
|
126
|
+
if os.path.exists(self.temp_dir):
|
|
127
|
+
return True
|
|
128
|
+
# Not exists, lets create
|
|
129
|
+
nix_mode_owner_group: int = 0o770 # Lets the new dir have full rights for process and its group
|
|
130
|
+
#os.mkdir(self.temp_dir, mode=nix_mode_owner_group) -- this one dont make missing dirs
|
|
131
|
+
os.makedirs(self.temp_dir, mode=nix_mode_owner_group, exist_ok=True)
|
|
132
|
+
return os.path.exists(self.temp_dir) # Does it exists after creation
|
|
133
|
+
|
|
134
|
+
def find_task_temp_file_name(self, extension:str ="dat") -> str | None:
|
|
135
|
+
"""
|
|
136
|
+
Temporary file for external data outside of git-dir for task_id that current object already knows
|
|
137
|
+
"""
|
|
138
|
+
if self.make_temp_dir(): # checks if temp-dir exists, creates
|
|
139
|
+
if extension is None:
|
|
140
|
+
extension = ''
|
|
141
|
+
file_name = self.task_id + '.' + extension
|
|
142
|
+
return os.path.join(self.temp_dir, file_name)
|
|
143
|
+
else:
|
|
144
|
+
return None
|
dapu/agents/agent_dig.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
from loguru import logger
|
|
2
|
+
|
|
3
|
+
from dapu.agents.agent import AgentGeneric
|
|
4
|
+
from dapu.process import logging
|
|
5
|
+
from dapu.perks import split_task_id
|
|
6
|
+
|
|
7
|
+
class Agent(AgentGeneric):
|
|
8
|
+
"""
|
|
9
|
+
Dig - dig source database to find possible tables to import and prepare definitions and files (those must put to target database (in meta) as blob with zip
|
|
10
|
+
Needs: task_id => schema.table, eg. dig_asjur5.the_table, dig_asjur5.the_column
|
|
11
|
+
=> route (from_asjur5) => route_alias
|
|
12
|
+
first "file" (tabels.sql) -- select for ASA metaquery (columns must correspond to the_table structure)
|
|
13
|
+
|
|
14
|
+
NB! See dig, mis võtab ja panem metadata kirjetena 2-3 tabelisse on tegelikult tehtav tavalise pump abil
|
|
15
|
+
St tuleb teha iga src baasi jaoks 2-3 tabelit ja 2-3 laadimisülesannet
|
|
16
|
+
saame tuvastada muudatused (__hist tabelisse nt, või changes tabelisse)
|
|
17
|
+
-- see võib olla hea enda juurde muudtuste tegemiseks
|
|
18
|
+
Aga dig tuleks teha ümber selleks, et kes võtab õiged metatabelid ja loob failide kupatuse... mille paneme... kuhu? meta.file_like
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
yaml:
|
|
22
|
+
once: true
|
|
23
|
+
until_first_error: true
|
|
24
|
+
schemas_cmd: schemas.sql
|
|
25
|
+
tables_in_schema_cmd: tables.sql
|
|
26
|
+
columns_in_table_cmd: columns.sql
|
|
27
|
+
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def do_action(self) -> bool:
|
|
32
|
+
|
|
33
|
+
_, target_schema, target_table = split_task_id(self.task_id)
|
|
34
|
+
file_element = 'file'
|
|
35
|
+
existing_files = self.collect_files(file_element)
|
|
36
|
+
if not existing_files:
|
|
37
|
+
logger.error(f"No files for tag {file_element} in {self.task_dir}")
|
|
38
|
+
return False
|
|
39
|
+
|
|
40
|
+
source_select = self.read_sql_from_file(existing_files[0], []) # siin on asenduseks nt in schema names -- millistest vaja, või kõik?
|
|
41
|
+
|
|
42
|
+
result_set = self.hub.run(self.route_alias, source_select)
|
|
43
|
+
|
|
44
|
+
if not result_set:
|
|
45
|
+
logger.warning(f"No data from {self.route_alias}")
|
|
46
|
+
return True
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
prepare_temp = f"""DROP TABLE IF EXISTS current_import"""
|
|
50
|
+
logger.debug(prepare_temp)
|
|
51
|
+
self.hub.run(self.target_alias, prepare_temp, False)
|
|
52
|
+
|
|
53
|
+
create_temp = f"""CREATE TEMP TABLE current_import
|
|
54
|
+
(schema_id integer, schema_name varchar(100), table_id integer, table_name varchar(100), table_comment text)"""
|
|
55
|
+
logger.debug(create_temp)
|
|
56
|
+
self.hub.run(self.target_alias, create_temp, False)
|
|
57
|
+
|
|
58
|
+
for row in result_set:
|
|
59
|
+
sql_ins = f"""INSERT INTO current_import(schema_id, schema_name, table_id, table_name, table_comment)
|
|
60
|
+
VALUES ({row[0]}, '{row[1]}', {row[2]}, '{row[3]}', '{row[4]}') """
|
|
61
|
+
self.hub.run(self.target_alias, sql_ins, False)
|
|
62
|
+
|
|
63
|
+
# compare -- simple variant (delete and insert)
|
|
64
|
+
sql_truncate = f"""DELETE FROM {target_schema}.{target_table} WHERE true"""
|
|
65
|
+
logger.debug(sql_truncate)
|
|
66
|
+
self.hub.run(self.target_alias, sql_truncate, False)
|
|
67
|
+
|
|
68
|
+
sql_refresh = f"""INSERT INTO {target_schema}.{target_table} (schema_id, schema_name, table_id, table_name, table_comment)
|
|
69
|
+
SELECT schema_id, schema_name, table_id, table_name, table_comment FROM current_import"""
|
|
70
|
+
logger.debug(sql_refresh)
|
|
71
|
+
self.hub.run(self.target_alias, sql_refresh, False)
|
|
72
|
+
|
|
73
|
+
logger.debug(prepare_temp)
|
|
74
|
+
self.hub.run(self.target_alias, prepare_temp, False) # polite!
|
|
75
|
+
|
|
76
|
+
logger.info('done')
|
|
77
|
+
return True
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from loguru import logger
|
|
2
|
+
from dapu.agents.agent import AgentGeneric
|
|
3
|
+
from dapu.perks import split_task_id
|
|
4
|
+
|
|
5
|
+
class Agent(AgentGeneric):
|
|
6
|
+
"""
|
|
7
|
+
drop - töötab tulemtabeliga ja teeb Postgre kiiksuga
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
def do_action(self) -> bool:
|
|
11
|
+
_, target_schema, target_table = split_task_id(self.task_id)
|
|
12
|
+
if not target_schema or not target_table:
|
|
13
|
+
return False
|
|
14
|
+
sql = self.make_drop_command(self.target_alias, target_schema, target_table, with_cascade=True)
|
|
15
|
+
if sql > '':
|
|
16
|
+
try:
|
|
17
|
+
logger.info(f"DROP current structure (in target base)")
|
|
18
|
+
self.context.target(sql, False)
|
|
19
|
+
except Exception as e1:
|
|
20
|
+
msg = f"Failed DROP command: {sql}"
|
|
21
|
+
self.last_error = msg
|
|
22
|
+
logger.error(msg)
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
# tühja SQL-i ei rakenda ja loeme selle õnnestumiseks
|
|
26
|
+
return True
|
|
27
|
+
|
|
28
|
+
# FIXME: repeats in shadow-agent => find reuse solution
|
|
29
|
+
def make_drop_command(self, target_alias, target_schema: str, target_table: str, with_cascade: bool=False) -> str:
|
|
30
|
+
"""
|
|
31
|
+
If object exists in postgre database then return sql command for drop corresponding to object type (table, view, etc)
|
|
32
|
+
Needs database alias
|
|
33
|
+
"""
|
|
34
|
+
# Tulembaasis (alati PostrgeSQL) vaja minev tegevus (õiget tüüpi drop käsk)
|
|
35
|
+
# Postgre metainfo päring, kas objekt on olemas ja mis tüüpi
|
|
36
|
+
|
|
37
|
+
command = '' # jääb tühjaks, kui pole vaja droppida
|
|
38
|
+
|
|
39
|
+
sql = f"""
|
|
40
|
+
SELECT
|
|
41
|
+
CASE
|
|
42
|
+
WHEN c.relkind = 'r' THEN 'TABLE'
|
|
43
|
+
WHEN c.relkind = 'v' THEN 'VIEW'
|
|
44
|
+
WHEN c.relkind = 'm' THEN 'MATERIALIZED VIEW'
|
|
45
|
+
ELSE 'ZZ' -- our uncleariness-marker
|
|
46
|
+
END
|
|
47
|
+
, c.relkind -- for debuging unclear state in far future
|
|
48
|
+
FROM pg_class c
|
|
49
|
+
JOIN pg_namespace tns ON tns.oid = c.relnamespace
|
|
50
|
+
WHERE tns.nspname = '{target_schema}' AND c.relname = '{target_table}'
|
|
51
|
+
UNION
|
|
52
|
+
SELECT 'XX' -- our non-existence marker
|
|
53
|
+
, NULL
|
|
54
|
+
ORDER BY 1
|
|
55
|
+
"""
|
|
56
|
+
result_set = self.context.target(sql)
|
|
57
|
+
|
|
58
|
+
if result_set:
|
|
59
|
+
unit = (result_set[0][0]).upper() # unit on "TABLE", "VIEW", "MATERIALIZED VIEW"
|
|
60
|
+
if unit not in ('XX', 'ZZ'):
|
|
61
|
+
command = f'DROP {unit} IF EXISTS {target_schema}.{target_table}'
|
|
62
|
+
if with_cascade:
|
|
63
|
+
command = command + ' CASCADE'
|
|
64
|
+
if unit == 'ZZ':
|
|
65
|
+
logger.warning("Unclear tabular type: {result_set[0][1]}")
|
|
66
|
+
|
|
67
|
+
return command
|
|
68
|
+
|
|
69
|
+
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
from loguru import logger
|
|
2
|
+
import openpyxl # kuradi kehv dokumentatsioon on asjal!!
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
from dapu.agents.agent import AgentGeneric
|
|
6
|
+
from dapu.perks import split_task_id, calculate_file_content_hash, python_value_as_sql_string
|
|
7
|
+
from dapu.placeholder import Placeholder
|
|
8
|
+
from dapu.perks import excel_label_to_number, excel_number_to_label, sanityze, convert_to_iso_date, end_of_month, start_of_month, end_of_year, start_of_year
|
|
9
|
+
from dapu.textops import yaml_string_to_dict
|
|
10
|
+
|
|
11
|
+
class Agent(AgentGeneric):
|
|
12
|
+
"""
|
|
13
|
+
-
|
|
14
|
+
do: exceltartu # custom agent.yaml may use word "excel"
|
|
15
|
+
source:
|
|
16
|
+
name: "sample.xlsx" # Excel new format (openpyxl don't work with old format)
|
|
17
|
+
sheet: 1 # positsion of sheet in workbook (1 is first, humanly way)
|
|
18
|
+
start_row: 2 # to eliminate header rows
|
|
19
|
+
management: hash # delete
|
|
20
|
+
delete: month # all, year, month, none, keys_file - do we need delete all data before loading, or periodic by range of excel existing dates
|
|
21
|
+
date_column: arve_algus_dt
|
|
22
|
+
insert: simple # simple, key (key is not implemented)
|
|
23
|
+
key: #### not implemented yet, just idea... to allow more optimized key-based comparision instead of simple insert
|
|
24
|
+
constraint: ak_harrastussport_4_uniq
|
|
25
|
+
update_ts: dwh_updated_ts # võimalus träkkimiseks, veerg peab olema loodud
|
|
26
|
+
deleted_keys: #### not implemented yet, just idea...
|
|
27
|
+
format: excel # csv või text
|
|
28
|
+
file: "katse3deleted.xlsx"
|
|
29
|
+
start_row: 2
|
|
30
|
+
delimiter: "," # või ";" või "; " või " " -- csv jaoks
|
|
31
|
+
cols: # list, kuna võti võib olla mitme-veeruline. vajame allikapoolset (st veerunumbrid excelis)
|
|
32
|
+
- from: 3 # nt isikukood
|
|
33
|
+
- from: 2 # nt arve number
|
|
34
|
+
cols: # tulemtabeli veergude nimed ja kõik muu
|
|
35
|
+
- name: isikukood # just text value
|
|
36
|
+
from: a
|
|
37
|
+
- name: isikukood_hashed
|
|
38
|
+
from: a
|
|
39
|
+
action: hash # some action before saving to database
|
|
40
|
+
- name: arve_algus # just text
|
|
41
|
+
from: b
|
|
42
|
+
- name: arve_algus_dt
|
|
43
|
+
from: b
|
|
44
|
+
action: to_date # transformed to ISO date string
|
|
45
|
+
- name: summa_eur_dec # numeric(11,2)
|
|
46
|
+
from: f
|
|
47
|
+
action: to_dec # transformaed to numeric literal
|
|
48
|
+
- name: dwh_imported_ts
|
|
49
|
+
fixed: current_timestamp # insert ...(current_timestamp) -- as-is
|
|
50
|
+
# value: tere --> insert ..('tere') -- surround with apostrophes
|
|
51
|
+
"""
|
|
52
|
+
myname: str = 'exceltartu'
|
|
53
|
+
|
|
54
|
+
def do_action(self) -> bool:
|
|
55
|
+
logger.info(f"start {self.myname} for {self.task_id}")
|
|
56
|
+
route_code, schema_name, table_name = split_task_id(self.task_id)
|
|
57
|
+
if 'source' not in self.action:
|
|
58
|
+
logger.error(f'Missing source (excel file to import data from)');
|
|
59
|
+
return False
|
|
60
|
+
if 'cols' not in self.action:
|
|
61
|
+
logger.error(f'Missing cols (listof columns of target table with instructions how them fullfil from excel)');
|
|
62
|
+
return False
|
|
63
|
+
profile: dict = self.context.get_profile(self.route_alias) or {}
|
|
64
|
+
if not profile:
|
|
65
|
+
logger.error(f"Missing profile {self.route_alias} (for task_id '{self.task_id}')")
|
|
66
|
+
return False
|
|
67
|
+
excel_dir = profile.get('path', None)
|
|
68
|
+
if excel_dir is None:
|
|
69
|
+
logger.error(f"Missing path in profile {self.route_alias} (for task_id '{self.task_id}')")
|
|
70
|
+
return False
|
|
71
|
+
if not os.path.exists(excel_dir):
|
|
72
|
+
logger.error(f"Missing folder '{excel_dir}' (for task_id '{self.task_id}')")
|
|
73
|
+
return False
|
|
74
|
+
if 'name' not in self.action['source']:
|
|
75
|
+
logger.error(f"Missing name of file (action>source>name) in loading definition (for task_id '{self.task_id}')")
|
|
76
|
+
return False
|
|
77
|
+
|
|
78
|
+
excel_file = os.path.join(excel_dir, self.action['source']['name']) # full name
|
|
79
|
+
if not os.path.exists(excel_file):
|
|
80
|
+
logger.info(f"No file '{excel_file}' (for task_id '{self.task_id}'), may be already consumed, end of pulling")
|
|
81
|
+
return True # TRUE since no data since last loading, absolutelly fine
|
|
82
|
+
|
|
83
|
+
file_hash = ""
|
|
84
|
+
file_management = self.action.get('management', 'hash') # hash (compare using last hash, do if different) vs delete (if exists then do)
|
|
85
|
+
if file_management == 'hash': # take last hash from registry
|
|
86
|
+
old_hash = self.context.get_task_hash_from_registry(self.task_id)
|
|
87
|
+
file_hash = calculate_file_content_hash(excel_file)
|
|
88
|
+
if old_hash == file_hash:
|
|
89
|
+
return True # No need to load, ok
|
|
90
|
+
|
|
91
|
+
delete_before = self.action.get('delete', 'month')
|
|
92
|
+
delete_date_column = self.action.get('delete_date_column', None)
|
|
93
|
+
sql_del = None
|
|
94
|
+
periodic = False
|
|
95
|
+
if delete_before == 'all':
|
|
96
|
+
sql_del = f"""DELETE FROM {schema_name}.{table_name} WHERE TRUE"""
|
|
97
|
+
if delete_before in ('day', 'month', 'year') and delete_date_column:
|
|
98
|
+
periodic = True
|
|
99
|
+
|
|
100
|
+
insert_mode = self.action.get('insert', 'simple')
|
|
101
|
+
if insert_mode != 'simple':
|
|
102
|
+
logger.error('Realiseerimata insert meetod, jääb katki')
|
|
103
|
+
return False
|
|
104
|
+
|
|
105
|
+
sheet = self.get_sheet(excel_file, self.action['source'].get('sheet', 1)) # file and human understandable position of sheet (starts with 1)
|
|
106
|
+
min_row, max_row = self.row_limits(sheet)
|
|
107
|
+
if periodic: # if time-based deletion then we must read sheet once to find dates
|
|
108
|
+
(period_start_dt, period_end_dt) = self.calculate_deletion_dates(sheet, min_row, max_row, delete_date_column)
|
|
109
|
+
if period_start_dt is not None and period_end_dt is not None:
|
|
110
|
+
sql_del = f"""DELETE FROM {schema_name}.{table_name} WHERE {delete_date_column} BETWEEN '{period_start_dt}' AND '{period_end_dt}' """
|
|
111
|
+
|
|
112
|
+
if sql_del: # delete if is reason for that
|
|
113
|
+
self.context.target(sql_del, False) # FIXME: change logic -> load into temp table similar to target table, if succeeded than delete and insert
|
|
114
|
+
self.load_sheet(sheet, min_row, max_row, schema_name, table_name, insert_mode)
|
|
115
|
+
|
|
116
|
+
logger.debug(f'Fail imporditud {excel_file}')
|
|
117
|
+
if file_management == 'hash':
|
|
118
|
+
self.context.save_task_hash_to_registry(self.task_id, file_hash)
|
|
119
|
+
else: # 'delete'
|
|
120
|
+
# delete file
|
|
121
|
+
os.remove(excel_file)
|
|
122
|
+
logger.info(f"done {self.myname} for {self.task_id}")
|
|
123
|
+
return True
|
|
124
|
+
|
|
125
|
+
def load_sheet(self, sheet, min_row, max_row, schema_name, table_name, insert_mode):
|
|
126
|
+
logger.debug(f'Laeme sisse read {min_row} .. {max_row}')
|
|
127
|
+
rida = 0
|
|
128
|
+
for row in sheet.iter_rows(min_row=min_row, max_row=max_row, values_only=True):
|
|
129
|
+
# row on tuple tüüpi
|
|
130
|
+
col_names = []
|
|
131
|
+
col_values = []
|
|
132
|
+
rida = rida + 1
|
|
133
|
+
logger.debug(f"uus rida {rida}, analüüsime veerge def faili abil")
|
|
134
|
+
for col in self.action['cols']:
|
|
135
|
+
col_name = col['name'] # baasi tabeli veerunimi -- peab alati olema
|
|
136
|
+
logger.debug(f"veerg: {col_name}")
|
|
137
|
+
col_label_excel = col.get('from', None) # võib ka puududa? nt constant
|
|
138
|
+
if col_label_excel:
|
|
139
|
+
logger.debug(f"veeru silt on: {col_label_excel}")
|
|
140
|
+
if not isinstance(col_label_excel, str):
|
|
141
|
+
logger.debug(type(col_label_excel))
|
|
142
|
+
col_label_excel = str(col_label_excel)
|
|
143
|
+
pos_excel = excel_label_to_number(col_label_excel) # 0-index
|
|
144
|
+
logger.debug(f"veeru indeks on: {pos_excel}")
|
|
145
|
+
logger.debug(f"väärtus on {row[pos_excel]}")
|
|
146
|
+
logger.debug(f"väärtuse tüüp on: {type(row[pos_excel])}")
|
|
147
|
+
col_value = sanityze(row[pos_excel], col.get('transform', None))
|
|
148
|
+
logger.debug(f"väärtus alguses: {col_value}")
|
|
149
|
+
# rega, et on olemas paar
|
|
150
|
+
# (None -> NULL, väärtus -> 'väärtus') + ülakoma varjestus
|
|
151
|
+
if isinstance(col_value, str):
|
|
152
|
+
col_value = col_value.replace("'", "''")
|
|
153
|
+
#else:
|
|
154
|
+
#logger.debug('pole string')
|
|
155
|
+
col_values.append(python_value_as_sql_string(col_value))
|
|
156
|
+
#col_values.append('NULL' if col_value is None else f"'{col_value}'") # võiks veidi tüünida numbrite käitumist, aga PG saab ka nii hakkama
|
|
157
|
+
col_names.append(col_name)
|
|
158
|
+
logger.debug('read läbi')
|
|
159
|
+
if insert_mode == 'simple':
|
|
160
|
+
#sql_ins = f"""INSERT INTO {schema_name}.{table_name} ({', '.join(col_names)}) VALUES ({', '.join(map(str, col_values))})"""
|
|
161
|
+
sql_ins = f"""INSERT INTO {schema_name}.{table_name} ({', '.join(col_names)}) VALUES ({', '.join(col_values)})"""
|
|
162
|
+
else:
|
|
163
|
+
sql_ins = None
|
|
164
|
+
if sql_ins:
|
|
165
|
+
logger.debug(f" rea {rida} lisamine")
|
|
166
|
+
self.context.target(sql_ins, False)
|
|
167
|
+
logger.debug(f" rida {rida} lisatud")
|
|
168
|
+
|
|
169
|
+
def calculate_deletion_dates(self, sheet, min_row, max_row, delete_date_column) -> tuple[str|None, str|None]:
|
|
170
|
+
excel_col = None
|
|
171
|
+
excel_col_action = None
|
|
172
|
+
for col in self.action['cols']:
|
|
173
|
+
if col.get('name', '') == delete_date_column:
|
|
174
|
+
excel_col = col.get('from', None)
|
|
175
|
+
excel_col_action = col.get('transform', None)
|
|
176
|
+
break
|
|
177
|
+
if excel_col is None:
|
|
178
|
+
return (None, None)
|
|
179
|
+
|
|
180
|
+
# on teada täht (tähed), leiame veeru jrk numbri (0-algusega)
|
|
181
|
+
excel_pos = excel_label_to_number(excel_col) # failis utils.py
|
|
182
|
+
|
|
183
|
+
cell_values = [] # list of ISO date strings (2024-01-23, 2024-11-06', ..)
|
|
184
|
+
# läbime selle ÜHE veeru kõigi ridadega (min_col ja max_col on see jrk/pos)
|
|
185
|
+
for jrk, row in enumerate(sheet.iter_rows(min_row=min_row, max_row=max_row, min_col=excel_pos, max_col=excel_pos, values_only=True), min_row):
|
|
186
|
+
cell_value = sanityze(row[0], excel_col_action)
|
|
187
|
+
if cell_value is not None:
|
|
188
|
+
cell_values.append(cell_value) # kuupäev ISO kujul, et tekst oleks mõistlikult sorteeritav
|
|
189
|
+
else:
|
|
190
|
+
# kui kasvõi üks kuupäevaveerus olev väärtus pole kuupäev (või on tühi), siis me keeldume töötamast
|
|
191
|
+
# jamasid saab liiga palju tulla sellest
|
|
192
|
+
logger.debug(f'Kuupäevaveerus leiti mittesobiv väärtus. Rida {jrk} veerg {excel_col}')
|
|
193
|
+
return (None, None)
|
|
194
|
+
|
|
195
|
+
cell_values.sort() # sorteerime kasvavalt
|
|
196
|
+
cell_min_value = cell_values[0] # ja võtame esimese ehk väikseima
|
|
197
|
+
cell_values.sort(reverse=True) # sorteerime kahanevalt
|
|
198
|
+
cell_max_value = cell_values[0] # ja võtame esimese ehk suurima
|
|
199
|
+
|
|
200
|
+
delete_before = self.action.get('delete', 'month')
|
|
201
|
+
# leida min jaoks year/month 1.päev ja max jaoks viimane (iso formaadis)
|
|
202
|
+
period_start_dt: str | None = None
|
|
203
|
+
period_end_dt: str | None = None
|
|
204
|
+
if delete_before == 'year':
|
|
205
|
+
period_start_dt = start_of_year(cell_min_value)
|
|
206
|
+
period_end_dt = end_of_year(cell_max_value)
|
|
207
|
+
if delete_before == 'month':
|
|
208
|
+
period_start_dt = start_of_month(cell_min_value)
|
|
209
|
+
period_end_dt = end_of_month(cell_max_value)
|
|
210
|
+
if delete_before == 'day':
|
|
211
|
+
period_start_dt = cell_min_value
|
|
212
|
+
period_end_dt = cell_max_value
|
|
213
|
+
return (period_start_dt, period_end_dt)
|
|
214
|
+
|
|
215
|
+
def get_sheet(self, excel_file, human_pos):
|
|
216
|
+
"""
|
|
217
|
+
Return sheet object
|
|
218
|
+
"""
|
|
219
|
+
# lets find out needed sheet
|
|
220
|
+
workbook = openpyxl.load_workbook(excel_file, data_only=True) # data_only
|
|
221
|
+
|
|
222
|
+
# workbookil on olemas list lehtede nimedega (.sheetnames) ja workbook ise on dict, kus võtmeks on lehe nimi
|
|
223
|
+
# list on positsionaalne (loodetavasti ikka sama järjekord, mis inimestel näha)
|
|
224
|
+
# def-failis arendaja poolt määratud sheet 1 peaks ideeliselt olema sheetnames listi element 0
|
|
225
|
+
sheet_names_list = workbook.sheetnames
|
|
226
|
+
needed_name = sheet_names_list[human_pos - 1]
|
|
227
|
+
sheet = workbook[needed_name]
|
|
228
|
+
return sheet
|
|
229
|
+
|
|
230
|
+
def row_limits(self, sheet) -> tuple[int, int]:
|
|
231
|
+
min_row = self.action['source'].get('start_row', 1) # min row = 1 (iter_rows kasutab inimmumbreid)
|
|
232
|
+
max_row = sheet.max_row
|
|
233
|
+
return (min_row, max_row)
|