dapu 0.7.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. dapu/__init__.py +0 -0
  2. dapu/agents/__init__.py +0 -0
  3. dapu/agents/agent.py +144 -0
  4. dapu/agents/agent_dig.py +81 -0
  5. dapu/agents/agent_drop.py +69 -0
  6. dapu/agents/agent_exceltartu.py +233 -0
  7. dapu/agents/agent_export.py +163 -0
  8. dapu/agents/agent_gen.py +167 -0
  9. dapu/agents/agent_history.py +91 -0
  10. dapu/agents/agent_import.py +191 -0
  11. dapu/agents/agent_map.py +34 -0
  12. dapu/agents/agent_runsql.py +42 -0
  13. dapu/agents/agent_shadow.py +244 -0
  14. dapu/agents/agent_stream.py +211 -0
  15. dapu/agents.yaml +18 -0
  16. dapu/context.py +457 -0
  17. dapu/downloader.py +83 -0
  18. dapu/drivers.yaml +9 -0
  19. dapu/enabler.py +167 -0
  20. dapu/fileops.py +65 -0
  21. dapu/job.py +272 -0
  22. dapu/jobstatus.py +17 -0
  23. dapu/maintenance.py +111 -0
  24. dapu/manager.py +302 -0
  25. dapu/observer.py +133 -0
  26. dapu/perks.py +910 -0
  27. dapu/placeholder.py +16 -0
  28. dapu/process.py +246 -0
  29. dapu/registrar.py +579 -0
  30. dapu/server.py +132 -0
  31. dapu/static/__init__.py +0 -0
  32. dapu/static/app.png +0 -0
  33. dapu/static/ikoon.txt +8 -0
  34. dapu/templates/__init__.py +0 -0
  35. dapu/templates/running.html +44 -0
  36. dapu/textops.py +68 -0
  37. dapu/ver/__init__.py +0 -0
  38. dapu/ver/changes.yaml +22 -0
  39. dapu/ver/v000_a_dba.sql +43 -0
  40. dapu/ver/v000_b_ext.sql +27 -0
  41. dapu/ver/v001_schema.sql +42 -0
  42. dapu/ver/v002_registry.sql +182 -0
  43. dapu/ver/v003_stopper.sql +14 -0
  44. dapu/ver/v004_paused.sql +9 -0
  45. dapu/ver/v005_route_weight.sql +3 -0
  46. dapu/ver/v006_reg_disabled.sql +6 -0
  47. dapu/ver/xv004_route.sql +4 -0
  48. dapu/ver/xv005_to_date.sql +31 -0
  49. dapu/ver/xv006_date.sql +9 -0
  50. dapu/ver/xv007_overflow.sql +78 -0
  51. dapu/versops.py +58 -0
  52. dapu/worker.py +252 -0
  53. dapu-0.7.11.dist-info/METADATA +198 -0
  54. dapu-0.7.11.dist-info/RECORD +57 -0
  55. dapu-0.7.11.dist-info/WHEEL +4 -0
  56. dapu-0.7.11.dist-info/licenses/LICENSE copy.txt +10 -0
  57. dapu-0.7.11.dist-info/licenses/LICENSE.txt +1 -0
dapu/__init__.py ADDED
File without changes
File without changes
dapu/agents/agent.py ADDED
@@ -0,0 +1,144 @@
1
+ from loguru import logger
2
+ import os
3
+ import tempfile
4
+ from dapu.context import DapuContext
5
+ from dapu.perks import make_replacements, read_sql_from_file
6
+ from dapu.placeholder import Placeholder
7
+
8
+ class AgentGeneric:
9
+
10
+ def __init__(self, task_id: str, action: dict, context: DapuContext, route_alias: str):
11
+ self.task_id: str = task_id
12
+ self.action: dict = action
13
+ self.context: DapuContext = context
14
+ self.target_alias: str = self.context.TARGET_ALIAS
15
+ self.route_alias: str = route_alias
16
+ self.temp_dir: str = self.find_temp_dir_path()
17
+ self.task_dir: str|None = self.find_task_dir_path(must_exists=True) # {work_dir}/routes/{route}/{schema}/{table}, where resides haulwork.yaml
18
+
19
+ def do_action(self) -> bool:
20
+ """
21
+ Main action. Must be defined in subclass.
22
+ """
23
+ logger.error("You are using too general class or subclass don't have do_action() implemented")
24
+ return False
25
+
26
+ def collect_files(self, file_element: str='file') -> list[str]:
27
+ """
28
+ Collects files mentioned in self.action from some tag/element (which values are dual-type: string or list of strings)
29
+ File_element name from YAML definition, usually "file". If agents supports files for different purposes then more clear name
30
+ eg. "create", "imput", "pre_shadow", "post_process"
31
+ Order of files (full path) is as in definition file
32
+ """
33
+ existing_files = [] # return list, may remine empty
34
+ if file_element not in self.action:
35
+ logger.debug(f"No element {file_element} in action definition") # don't need to be error (consumer decides)
36
+ return []
37
+ def_files = []
38
+ def_files_part = self.action.get(file_element, None) # short names
39
+ if def_files_part is not None:
40
+ if isinstance(def_files_part, str):
41
+ def_files.append(def_files_part) # teeme stringist listi
42
+ else:
43
+ if isinstance(def_files_part, list):
44
+ def_files = def_files_part
45
+
46
+ for file_name in def_files:
47
+ file_local = os.path.join(self.task_dir or ".", file_name)
48
+ if os.path.exists(file_local):
49
+ existing_files.append(file_local) # full file name
50
+
51
+ return existing_files # samas järjekoras, mis def failis, aga ainult need, mis eksisteerivad ka
52
+
53
+ def apply_files_to_target(self, existing_files: list[str], replacements: list[tuple[str | Placeholder, str]] | None) -> bool:
54
+ """
55
+ Name makes very clear that we never apply changes to source-databases, only to one single target-database
56
+ Files are applied in order they are in list.
57
+ List contains full paths of file names.
58
+ Empty and problematic files return False as failure. First error breaks (returns False)
59
+ Replacements is list of tuples (find -> replace)
60
+ '"""
61
+ for file_full_name in existing_files:
62
+ # Read file and make replacements
63
+ sql: str | None = make_replacements(read_sql_from_file(file_full_name), replacements)# does logging
64
+ if sql is None:
65
+ return False # break on first error
66
+
67
+ try:
68
+ logger.debug(f"Will apply SQL from file {file_full_name}")
69
+ self.context.target(sql, False)
70
+ except Exception as e1:
71
+ logger.error(f"Applying SQL file failed ({file_full_name})")
72
+ #logger.error(f"{sql}") # deep logging or local logging only (too unsafe for remote logging)
73
+ logger.error(e1)
74
+ return False # break on first error
75
+ logger.debug(f"SQL file {file_full_name} applied")
76
+ return True
77
+
78
+ def find_temp_dir_path(self) -> str:
79
+ """
80
+ Encapsulated logic for calculating project temp-dir. It may reside inside target-dir (work_dir),
81
+ # os.path.realpath(os.path.join(self.context.work_dir, 'tmp'))
82
+ which is good for maintenance (quickly finding files) and cleaning process (delete all temp files for target),
83
+ but so this may spoil files for local (dev, which is git-based) and git-based deploy, so conflicts may emerge.
84
+ May-be we stick op.system temp directory and create under it app-named (dapu) and target-named dir
85
+ assuming, that actual deploys are with short living hard-disks (like pods). So we don't need to worry much about
86
+ files remaining there due failures.
87
+ Just remember that our temp-files must be accessible inside one process
88
+ (we dont need to keep them after failure or any other possible ends)
89
+ but one process may execute many independent Agents, so they all must know path same way.
90
+ Files inside temp dir are named as {task_id}.dat -> one temp dir for target, all dat-files inside
91
+ """
92
+ system_temp_dir = tempfile.gettempdir() # https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir
93
+ target_name = os.path.split(self.context.work_dir)[1] # 2nd element is "tail" - everything after last (back)slash
94
+ # temp_path_for_app = os.path.join(system_temp_dir, self.context.APP_NAME)
95
+ # should we need to create intermediate dir? no, we are using os.makedirs later
96
+ temp_path_for_target = os.path.join(system_temp_dir, self.context.APP_NAME, target_name)
97
+ logger.debug(f"Temporary directory path for target is {temp_path_for_target}, if needed")
98
+ return temp_path_for_target
99
+
100
+ def find_task_dir_path(self, must_exists: bool=False) -> str|None:
101
+ """
102
+ Full path from current (internal) task_id (gives 3 directories) and self.context root path (work_dir)
103
+ """
104
+ if not self.task_id:
105
+ logger.error(f"Empty task_id {self.task_id}")
106
+ return None
107
+ if self.context.work_dir is None:
108
+ logger.error(f"Context work_dir is missing for task {self.task_id}")
109
+ return None
110
+ path_way: list = self.task_id.split('.')
111
+ if len(path_way) < 3:
112
+ logger.error(f"Too short task_id {self.task_id}")
113
+ return None
114
+ path: str = self.context.full_name_from_pull(path_way)
115
+ if must_exists and not os.path.exists(path):
116
+ logger.error(f"Path {path} for task '{self.task_id}' not exists in local file system")
117
+ return None
118
+ logger.debug(f"Dir for task {self.task_id} is {path}")
119
+ return path
120
+
121
+ def make_temp_dir(self) -> bool:
122
+ """
123
+ Lets make temp dir on first demand using system-wide logic for path
124
+ Returns False only if creation failed silently
125
+ """
126
+ if os.path.exists(self.temp_dir):
127
+ return True
128
+ # Not exists, lets create
129
+ nix_mode_owner_group: int = 0o770 # Lets the new dir have full rights for process and its group
130
+ #os.mkdir(self.temp_dir, mode=nix_mode_owner_group) -- this one dont make missing dirs
131
+ os.makedirs(self.temp_dir, mode=nix_mode_owner_group, exist_ok=True)
132
+ return os.path.exists(self.temp_dir) # Does it exists after creation
133
+
134
+ def find_task_temp_file_name(self, extension:str ="dat") -> str | None:
135
+ """
136
+ Temporary file for external data outside of git-dir for task_id that current object already knows
137
+ """
138
+ if self.make_temp_dir(): # checks if temp-dir exists, creates
139
+ if extension is None:
140
+ extension = ''
141
+ file_name = self.task_id + '.' + extension
142
+ return os.path.join(self.temp_dir, file_name)
143
+ else:
144
+ return None
@@ -0,0 +1,81 @@
1
+ from loguru import logger
2
+
3
+ from dapu.agents.agent import AgentGeneric
4
+ from dapu.process import logging
5
+ from dapu.perks import split_task_id
6
+
7
+ class Agent(AgentGeneric):
8
+ """
9
+ Dig - dig source database to find possible tables to import and prepare definitions and files (those must put to target database (in meta) as blob with zip
10
+ Needs: task_id => schema.table, eg. dig_asjur5.the_table, dig_asjur5.the_column
11
+ => route (from_asjur5) => route_alias
12
+ first "file" (tabels.sql) -- select for ASA metaquery (columns must correspond to the_table structure)
13
+
14
+ NB! See dig, mis võtab ja panem metadata kirjetena 2-3 tabelisse on tegelikult tehtav tavalise pump abil
15
+ St tuleb teha iga src baasi jaoks 2-3 tabelit ja 2-3 laadimisülesannet
16
+ saame tuvastada muudatused (__hist tabelisse nt, või changes tabelisse)
17
+ -- see võib olla hea enda juurde muudtuste tegemiseks
18
+ Aga dig tuleks teha ümber selleks, et kes võtab õiged metatabelid ja loob failide kupatuse... mille paneme... kuhu? meta.file_like
19
+
20
+
21
+ yaml:
22
+ once: true
23
+ until_first_error: true
24
+ schemas_cmd: schemas.sql
25
+ tables_in_schema_cmd: tables.sql
26
+ columns_in_table_cmd: columns.sql
27
+
28
+ """
29
+
30
+
31
+ def do_action(self) -> bool:
32
+
33
+ _, target_schema, target_table = split_task_id(self.task_id)
34
+ file_element = 'file'
35
+ existing_files = self.collect_files(file_element)
36
+ if not existing_files:
37
+ logger.error(f"No files for tag {file_element} in {self.task_dir}")
38
+ return False
39
+
40
+ source_select = self.read_sql_from_file(existing_files[0], []) # siin on asenduseks nt in schema names -- millistest vaja, või kõik?
41
+
42
+ result_set = self.hub.run(self.route_alias, source_select)
43
+
44
+ if not result_set:
45
+ logger.warning(f"No data from {self.route_alias}")
46
+ return True
47
+
48
+
49
+ prepare_temp = f"""DROP TABLE IF EXISTS current_import"""
50
+ logger.debug(prepare_temp)
51
+ self.hub.run(self.target_alias, prepare_temp, False)
52
+
53
+ create_temp = f"""CREATE TEMP TABLE current_import
54
+ (schema_id integer, schema_name varchar(100), table_id integer, table_name varchar(100), table_comment text)"""
55
+ logger.debug(create_temp)
56
+ self.hub.run(self.target_alias, create_temp, False)
57
+
58
+ for row in result_set:
59
+ sql_ins = f"""INSERT INTO current_import(schema_id, schema_name, table_id, table_name, table_comment)
60
+ VALUES ({row[0]}, '{row[1]}', {row[2]}, '{row[3]}', '{row[4]}') """
61
+ self.hub.run(self.target_alias, sql_ins, False)
62
+
63
+ # compare -- simple variant (delete and insert)
64
+ sql_truncate = f"""DELETE FROM {target_schema}.{target_table} WHERE true"""
65
+ logger.debug(sql_truncate)
66
+ self.hub.run(self.target_alias, sql_truncate, False)
67
+
68
+ sql_refresh = f"""INSERT INTO {target_schema}.{target_table} (schema_id, schema_name, table_id, table_name, table_comment)
69
+ SELECT schema_id, schema_name, table_id, table_name, table_comment FROM current_import"""
70
+ logger.debug(sql_refresh)
71
+ self.hub.run(self.target_alias, sql_refresh, False)
72
+
73
+ logger.debug(prepare_temp)
74
+ self.hub.run(self.target_alias, prepare_temp, False) # polite!
75
+
76
+ logger.info('done')
77
+ return True
78
+
79
+
80
+
81
+
@@ -0,0 +1,69 @@
1
+ from loguru import logger
2
+ from dapu.agents.agent import AgentGeneric
3
+ from dapu.perks import split_task_id
4
+
5
+ class Agent(AgentGeneric):
6
+ """
7
+ drop - töötab tulemtabeliga ja teeb Postgre kiiksuga
8
+ """
9
+
10
+ def do_action(self) -> bool:
11
+ _, target_schema, target_table = split_task_id(self.task_id)
12
+ if not target_schema or not target_table:
13
+ return False
14
+ sql = self.make_drop_command(self.target_alias, target_schema, target_table, with_cascade=True)
15
+ if sql > '':
16
+ try:
17
+ logger.info(f"DROP current structure (in target base)")
18
+ self.context.target(sql, False)
19
+ except Exception as e1:
20
+ msg = f"Failed DROP command: {sql}"
21
+ self.last_error = msg
22
+ logger.error(msg)
23
+ return False
24
+
25
+ # tühja SQL-i ei rakenda ja loeme selle õnnestumiseks
26
+ return True
27
+
28
+ # FIXME: repeats in shadow-agent => find reuse solution
29
+ def make_drop_command(self, target_alias, target_schema: str, target_table: str, with_cascade: bool=False) -> str:
30
+ """
31
+ If object exists in postgre database then return sql command for drop corresponding to object type (table, view, etc)
32
+ Needs database alias
33
+ """
34
+ # Tulembaasis (alati PostrgeSQL) vaja minev tegevus (õiget tüüpi drop käsk)
35
+ # Postgre metainfo päring, kas objekt on olemas ja mis tüüpi
36
+
37
+ command = '' # jääb tühjaks, kui pole vaja droppida
38
+
39
+ sql = f"""
40
+ SELECT
41
+ CASE
42
+ WHEN c.relkind = 'r' THEN 'TABLE'
43
+ WHEN c.relkind = 'v' THEN 'VIEW'
44
+ WHEN c.relkind = 'm' THEN 'MATERIALIZED VIEW'
45
+ ELSE 'ZZ' -- our uncleariness-marker
46
+ END
47
+ , c.relkind -- for debuging unclear state in far future
48
+ FROM pg_class c
49
+ JOIN pg_namespace tns ON tns.oid = c.relnamespace
50
+ WHERE tns.nspname = '{target_schema}' AND c.relname = '{target_table}'
51
+ UNION
52
+ SELECT 'XX' -- our non-existence marker
53
+ , NULL
54
+ ORDER BY 1
55
+ """
56
+ result_set = self.context.target(sql)
57
+
58
+ if result_set:
59
+ unit = (result_set[0][0]).upper() # unit on "TABLE", "VIEW", "MATERIALIZED VIEW"
60
+ if unit not in ('XX', 'ZZ'):
61
+ command = f'DROP {unit} IF EXISTS {target_schema}.{target_table}'
62
+ if with_cascade:
63
+ command = command + ' CASCADE'
64
+ if unit == 'ZZ':
65
+ logger.warning("Unclear tabular type: {result_set[0][1]}")
66
+
67
+ return command
68
+
69
+
@@ -0,0 +1,233 @@
1
+ from loguru import logger
2
+ import openpyxl # kuradi kehv dokumentatsioon on asjal!!
3
+ import os
4
+
5
+ from dapu.agents.agent import AgentGeneric
6
+ from dapu.perks import split_task_id, calculate_file_content_hash, python_value_as_sql_string
7
+ from dapu.placeholder import Placeholder
8
+ from dapu.perks import excel_label_to_number, excel_number_to_label, sanityze, convert_to_iso_date, end_of_month, start_of_month, end_of_year, start_of_year
9
+ from dapu.textops import yaml_string_to_dict
10
+
11
+ class Agent(AgentGeneric):
12
+ """
13
+ -
14
+ do: exceltartu # custom agent.yaml may use word "excel"
15
+ source:
16
+ name: "sample.xlsx" # Excel new format (openpyxl don't work with old format)
17
+ sheet: 1 # positsion of sheet in workbook (1 is first, humanly way)
18
+ start_row: 2 # to eliminate header rows
19
+ management: hash # delete
20
+ delete: month # all, year, month, none, keys_file - do we need delete all data before loading, or periodic by range of excel existing dates
21
+ date_column: arve_algus_dt
22
+ insert: simple # simple, key (key is not implemented)
23
+ key: #### not implemented yet, just idea... to allow more optimized key-based comparision instead of simple insert
24
+ constraint: ak_harrastussport_4_uniq
25
+ update_ts: dwh_updated_ts # võimalus träkkimiseks, veerg peab olema loodud
26
+ deleted_keys: #### not implemented yet, just idea...
27
+ format: excel # csv või text
28
+ file: "katse3deleted.xlsx"
29
+ start_row: 2
30
+ delimiter: "," # või ";" või "; " või " " -- csv jaoks
31
+ cols: # list, kuna võti võib olla mitme-veeruline. vajame allikapoolset (st veerunumbrid excelis)
32
+ - from: 3 # nt isikukood
33
+ - from: 2 # nt arve number
34
+ cols: # tulemtabeli veergude nimed ja kõik muu
35
+ - name: isikukood # just text value
36
+ from: a
37
+ - name: isikukood_hashed
38
+ from: a
39
+ action: hash # some action before saving to database
40
+ - name: arve_algus # just text
41
+ from: b
42
+ - name: arve_algus_dt
43
+ from: b
44
+ action: to_date # transformed to ISO date string
45
+ - name: summa_eur_dec # numeric(11,2)
46
+ from: f
47
+ action: to_dec # transformaed to numeric literal
48
+ - name: dwh_imported_ts
49
+ fixed: current_timestamp # insert ...(current_timestamp) -- as-is
50
+ # value: tere --> insert ..('tere') -- surround with apostrophes
51
+ """
52
+ myname: str = 'exceltartu'
53
+
54
+ def do_action(self) -> bool:
55
+ logger.info(f"start {self.myname} for {self.task_id}")
56
+ route_code, schema_name, table_name = split_task_id(self.task_id)
57
+ if 'source' not in self.action:
58
+ logger.error(f'Missing source (excel file to import data from)');
59
+ return False
60
+ if 'cols' not in self.action:
61
+ logger.error(f'Missing cols (listof columns of target table with instructions how them fullfil from excel)');
62
+ return False
63
+ profile: dict = self.context.get_profile(self.route_alias) or {}
64
+ if not profile:
65
+ logger.error(f"Missing profile {self.route_alias} (for task_id '{self.task_id}')")
66
+ return False
67
+ excel_dir = profile.get('path', None)
68
+ if excel_dir is None:
69
+ logger.error(f"Missing path in profile {self.route_alias} (for task_id '{self.task_id}')")
70
+ return False
71
+ if not os.path.exists(excel_dir):
72
+ logger.error(f"Missing folder '{excel_dir}' (for task_id '{self.task_id}')")
73
+ return False
74
+ if 'name' not in self.action['source']:
75
+ logger.error(f"Missing name of file (action>source>name) in loading definition (for task_id '{self.task_id}')")
76
+ return False
77
+
78
+ excel_file = os.path.join(excel_dir, self.action['source']['name']) # full name
79
+ if not os.path.exists(excel_file):
80
+ logger.info(f"No file '{excel_file}' (for task_id '{self.task_id}'), may be already consumed, end of pulling")
81
+ return True # TRUE since no data since last loading, absolutelly fine
82
+
83
+ file_hash = ""
84
+ file_management = self.action.get('management', 'hash') # hash (compare using last hash, do if different) vs delete (if exists then do)
85
+ if file_management == 'hash': # take last hash from registry
86
+ old_hash = self.context.get_task_hash_from_registry(self.task_id)
87
+ file_hash = calculate_file_content_hash(excel_file)
88
+ if old_hash == file_hash:
89
+ return True # No need to load, ok
90
+
91
+ delete_before = self.action.get('delete', 'month')
92
+ delete_date_column = self.action.get('delete_date_column', None)
93
+ sql_del = None
94
+ periodic = False
95
+ if delete_before == 'all':
96
+ sql_del = f"""DELETE FROM {schema_name}.{table_name} WHERE TRUE"""
97
+ if delete_before in ('day', 'month', 'year') and delete_date_column:
98
+ periodic = True
99
+
100
+ insert_mode = self.action.get('insert', 'simple')
101
+ if insert_mode != 'simple':
102
+ logger.error('Realiseerimata insert meetod, jääb katki')
103
+ return False
104
+
105
+ sheet = self.get_sheet(excel_file, self.action['source'].get('sheet', 1)) # file and human understandable position of sheet (starts with 1)
106
+ min_row, max_row = self.row_limits(sheet)
107
+ if periodic: # if time-based deletion then we must read sheet once to find dates
108
+ (period_start_dt, period_end_dt) = self.calculate_deletion_dates(sheet, min_row, max_row, delete_date_column)
109
+ if period_start_dt is not None and period_end_dt is not None:
110
+ sql_del = f"""DELETE FROM {schema_name}.{table_name} WHERE {delete_date_column} BETWEEN '{period_start_dt}' AND '{period_end_dt}' """
111
+
112
+ if sql_del: # delete if is reason for that
113
+ self.context.target(sql_del, False) # FIXME: change logic -> load into temp table similar to target table, if succeeded than delete and insert
114
+ self.load_sheet(sheet, min_row, max_row, schema_name, table_name, insert_mode)
115
+
116
+ logger.debug(f'Fail imporditud {excel_file}')
117
+ if file_management == 'hash':
118
+ self.context.save_task_hash_to_registry(self.task_id, file_hash)
119
+ else: # 'delete'
120
+ # delete file
121
+ os.remove(excel_file)
122
+ logger.info(f"done {self.myname} for {self.task_id}")
123
+ return True
124
+
125
+ def load_sheet(self, sheet, min_row, max_row, schema_name, table_name, insert_mode):
126
+ logger.debug(f'Laeme sisse read {min_row} .. {max_row}')
127
+ rida = 0
128
+ for row in sheet.iter_rows(min_row=min_row, max_row=max_row, values_only=True):
129
+ # row on tuple tüüpi
130
+ col_names = []
131
+ col_values = []
132
+ rida = rida + 1
133
+ logger.debug(f"uus rida {rida}, analüüsime veerge def faili abil")
134
+ for col in self.action['cols']:
135
+ col_name = col['name'] # baasi tabeli veerunimi -- peab alati olema
136
+ logger.debug(f"veerg: {col_name}")
137
+ col_label_excel = col.get('from', None) # võib ka puududa? nt constant
138
+ if col_label_excel:
139
+ logger.debug(f"veeru silt on: {col_label_excel}")
140
+ if not isinstance(col_label_excel, str):
141
+ logger.debug(type(col_label_excel))
142
+ col_label_excel = str(col_label_excel)
143
+ pos_excel = excel_label_to_number(col_label_excel) # 0-index
144
+ logger.debug(f"veeru indeks on: {pos_excel}")
145
+ logger.debug(f"väärtus on {row[pos_excel]}")
146
+ logger.debug(f"väärtuse tüüp on: {type(row[pos_excel])}")
147
+ col_value = sanityze(row[pos_excel], col.get('transform', None))
148
+ logger.debug(f"väärtus alguses: {col_value}")
149
+ # rega, et on olemas paar
150
+ # (None -> NULL, väärtus -> 'väärtus') + ülakoma varjestus
151
+ if isinstance(col_value, str):
152
+ col_value = col_value.replace("'", "''")
153
+ #else:
154
+ #logger.debug('pole string')
155
+ col_values.append(python_value_as_sql_string(col_value))
156
+ #col_values.append('NULL' if col_value is None else f"'{col_value}'") # võiks veidi tüünida numbrite käitumist, aga PG saab ka nii hakkama
157
+ col_names.append(col_name)
158
+ logger.debug('read läbi')
159
+ if insert_mode == 'simple':
160
+ #sql_ins = f"""INSERT INTO {schema_name}.{table_name} ({', '.join(col_names)}) VALUES ({', '.join(map(str, col_values))})"""
161
+ sql_ins = f"""INSERT INTO {schema_name}.{table_name} ({', '.join(col_names)}) VALUES ({', '.join(col_values)})"""
162
+ else:
163
+ sql_ins = None
164
+ if sql_ins:
165
+ logger.debug(f" rea {rida} lisamine")
166
+ self.context.target(sql_ins, False)
167
+ logger.debug(f" rida {rida} lisatud")
168
+
169
+ def calculate_deletion_dates(self, sheet, min_row, max_row, delete_date_column) -> tuple[str|None, str|None]:
170
+ excel_col = None
171
+ excel_col_action = None
172
+ for col in self.action['cols']:
173
+ if col.get('name', '') == delete_date_column:
174
+ excel_col = col.get('from', None)
175
+ excel_col_action = col.get('transform', None)
176
+ break
177
+ if excel_col is None:
178
+ return (None, None)
179
+
180
+ # on teada täht (tähed), leiame veeru jrk numbri (0-algusega)
181
+ excel_pos = excel_label_to_number(excel_col) # failis utils.py
182
+
183
+ cell_values = [] # list of ISO date strings (2024-01-23, 2024-11-06', ..)
184
+ # läbime selle ÜHE veeru kõigi ridadega (min_col ja max_col on see jrk/pos)
185
+ for jrk, row in enumerate(sheet.iter_rows(min_row=min_row, max_row=max_row, min_col=excel_pos, max_col=excel_pos, values_only=True), min_row):
186
+ cell_value = sanityze(row[0], excel_col_action)
187
+ if cell_value is not None:
188
+ cell_values.append(cell_value) # kuupäev ISO kujul, et tekst oleks mõistlikult sorteeritav
189
+ else:
190
+ # kui kasvõi üks kuupäevaveerus olev väärtus pole kuupäev (või on tühi), siis me keeldume töötamast
191
+ # jamasid saab liiga palju tulla sellest
192
+ logger.debug(f'Kuupäevaveerus leiti mittesobiv väärtus. Rida {jrk} veerg {excel_col}')
193
+ return (None, None)
194
+
195
+ cell_values.sort() # sorteerime kasvavalt
196
+ cell_min_value = cell_values[0] # ja võtame esimese ehk väikseima
197
+ cell_values.sort(reverse=True) # sorteerime kahanevalt
198
+ cell_max_value = cell_values[0] # ja võtame esimese ehk suurima
199
+
200
+ delete_before = self.action.get('delete', 'month')
201
+ # leida min jaoks year/month 1.päev ja max jaoks viimane (iso formaadis)
202
+ period_start_dt: str | None = None
203
+ period_end_dt: str | None = None
204
+ if delete_before == 'year':
205
+ period_start_dt = start_of_year(cell_min_value)
206
+ period_end_dt = end_of_year(cell_max_value)
207
+ if delete_before == 'month':
208
+ period_start_dt = start_of_month(cell_min_value)
209
+ period_end_dt = end_of_month(cell_max_value)
210
+ if delete_before == 'day':
211
+ period_start_dt = cell_min_value
212
+ period_end_dt = cell_max_value
213
+ return (period_start_dt, period_end_dt)
214
+
215
+ def get_sheet(self, excel_file, human_pos):
216
+ """
217
+ Return sheet object
218
+ """
219
+ # lets find out needed sheet
220
+ workbook = openpyxl.load_workbook(excel_file, data_only=True) # data_only
221
+
222
+ # workbookil on olemas list lehtede nimedega (.sheetnames) ja workbook ise on dict, kus võtmeks on lehe nimi
223
+ # list on positsionaalne (loodetavasti ikka sama järjekord, mis inimestel näha)
224
+ # def-failis arendaja poolt määratud sheet 1 peaks ideeliselt olema sheetnames listi element 0
225
+ sheet_names_list = workbook.sheetnames
226
+ needed_name = sheet_names_list[human_pos - 1]
227
+ sheet = workbook[needed_name]
228
+ return sheet
229
+
230
+ def row_limits(self, sheet) -> tuple[int, int]:
231
+ min_row = self.action['source'].get('start_row', 1) # min row = 1 (iter_rows kasutab inimmumbreid)
232
+ max_row = sheet.max_row
233
+ return (min_row, max_row)