ssb-dash-framework 0.2.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/PKG-INFO +1 -1
  2. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/pyproject.toml +1 -1
  3. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/control/control_framework_base.py +92 -64
  4. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_control_view.py +32 -13
  5. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/parquet_editor.py +11 -1
  6. ssb_dash_framework-0.2.0/src/ssb_dash_framework/modules/altinn_control_view copy.py +0 -459
  7. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/LICENSE +0 -0
  8. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/README.md +0 -0
  9. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/__init__.py +0 -0
  10. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/__main__.py +0 -0
  11. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/assets/dashAgGridFunctions.js +0 -0
  12. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/assets/stylesheet.css +0 -0
  13. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/control/__init__.py +0 -0
  14. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/__init__.py +0 -0
  15. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/aarsregnskap.py +0 -0
  16. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/agg_dist_plotter.py +0 -0
  17. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_data_capture.py +0 -0
  18. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/__init__.py +0 -0
  19. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_comment.py +0 -0
  20. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_contact.py +0 -0
  21. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_control.py +0 -0
  22. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_history.py +0 -0
  23. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_main_view.py +0 -0
  24. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_primary_table.py +0 -0
  25. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_submitted_forms.py +0 -0
  26. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_supporting_table.py +0 -0
  27. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_unit_details.py +0 -0
  28. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/bofregistry.py +0 -0
  29. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/building_blocks/__init__.py +0 -0
  30. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/building_blocks/canvas.py +0 -0
  31. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/building_blocks/figuredisplay.py +0 -0
  32. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/building_blocks/map_display.py +0 -0
  33. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/building_blocks/multimodule.py +0 -0
  34. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/building_blocks/tables.py +0 -0
  35. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/control.py +0 -0
  36. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/freesearch.py +0 -0
  37. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/hb_method.py +0 -0
  38. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/macro_module.py +0 -0
  39. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/pi_memorizer.py +0 -0
  40. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/skjemapdfviewer.py +0 -0
  41. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/modules/visualizationbuilder.py +0 -0
  42. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/py.typed +0 -0
  43. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/setup/__init__.py +0 -0
  44. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/setup/app_setup.py +0 -0
  45. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/setup/main_layout.py +0 -0
  46. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/setup/variableselector.py +0 -0
  47. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/__init__.py +0 -0
  48. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/alert_handler.py +0 -0
  49. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/app_logger.py +0 -0
  50. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/core_query_functions.py +0 -0
  51. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/datahelper.py +0 -0
  52. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/debugger_modal.py +0 -0
  53. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/eimerdb_helpers.py +0 -0
  54. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/functions.py +0 -0
  55. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/implementations.py +0 -0
  56. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/module_validation.py +0 -0
  57. {ssb_dash_framework-0.2.0 → ssb_dash_framework-0.2.1}/src/ssb_dash_framework/utils/r_helpers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: ssb-dash-framework
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: SSB Dash Framework
5
5
  License: GPL-3.0
6
6
  Author: Magnus Theodor Engh
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ssb-dash-framework"
3
- version = "0.2.0"
3
+ version = "0.2.1"
4
4
  description = "SSB Dash Framework"
5
5
  authors = [{ name = "Magnus Theodor Engh", email = "dor@ssb.no" }]
6
6
  license = "GPL-3.0"
@@ -26,6 +26,8 @@ def register_control(
26
26
  ) -> Callable[..., Any]:
27
27
  """Decorator used to attach required metadata to control methods.
28
28
 
29
+ It is recommended to have the name of your control method / function start with the prefix 'control_'.
30
+
29
31
  Args:
30
32
  kontrollid (str): The id of the control, preferably a code or shortened name. Must be unique. Note that in the control module controls are sorted alphabetically based on kontrollid, meaning you can add numbers as prefix to control the sorting order in the app.
31
33
  kontrolltype (str): The type of control. Must be 'H' (Hard control), 'S' (Soft control) or 'I' (Informative)
@@ -204,18 +206,19 @@ class ControlFrameworkBase: # TODO: Add some common control methods here for ea
204
206
 
205
207
  df_expanded = pd.DataFrame(combinations, columns=self.applies_to_subset.keys())
206
208
  rows_to_register = row_to_register.merge(df_expanded, how="cross")
207
- rows_to_register = rows_to_register.merge(
208
- registered_controls,
209
- how="outer",
210
- on=[
211
- *self.applies_to_subset.keys(),
212
- *[k for k in control_meta.keys() if k != "kontrollvars"],
213
- ], # TODO: Fix better Dropping kontrollvars as it is included in control_meta but only clutter in the database.
214
- indicator=True,
215
- )
216
- rows_to_register = rows_to_register[
217
- rows_to_register["_merge"] == "left_only"
218
- ].drop(columns=["_merge"])
209
+ if registered_controls is not None:
210
+ rows_to_register = rows_to_register.merge(
211
+ registered_controls,
212
+ how="outer",
213
+ on=[
214
+ *self.applies_to_subset.keys(),
215
+ *[k for k in control_meta.keys() if k != "kontrollvars"],
216
+ ], # TODO: Fix better Dropping kontrollvars as it is included in control_meta but only clutter in the database.
217
+ indicator=True,
218
+ )
219
+ rows_to_register = rows_to_register[
220
+ rows_to_register["_merge"] == "left_only"
221
+ ].drop(columns=["_merge"])
219
222
  if rows_to_register.empty:
220
223
  logger.debug("No new control to register, ending here.")
221
224
  return None
@@ -240,15 +243,24 @@ class ControlFrameworkBase: # TODO: Add some common control methods here for ea
240
243
  self.register_control(control)
241
244
  logger.info("All controls registered.")
242
245
 
243
- def get_current_kontroller(self) -> pd.DataFrame:
246
+ def get_current_kontroller(self) -> pd.DataFrame | None:
244
247
  """Gets the current contents of the 'kontroller' table."""
245
248
  logger.info("Getting current contents of table 'kontroller'")
246
249
  if isinstance(self.conn, EimerDBInstance):
247
250
  conn = ibis.polars.connect()
248
- kontroller = self.conn.query(
249
- "SELECT * FROM kontroller"
250
- ) # maybe add something like this?partition_select=self.applies_to_subset
251
- conn.create_table("kontroller", kontroller)
251
+ try:
252
+ kontroller = self.conn.query(
253
+ "SELECT * FROM kontroller"
254
+ ) # maybe add something like this?partition_select=self.applies_to_subset
255
+ conn.create_table("kontroller", kontroller)
256
+ except (
257
+ ValueError
258
+ ) as e: # TODO permanently fix this. Error caused by running .query on eimerdb table with no contents.
259
+ if str(e) == "max() arg is an empty sequence":
260
+ logger.warning(
261
+ "Did not find any contents in 'kontroller', starting from scratch."
262
+ )
263
+ return None
252
264
  elif conn_is_ibis(self.conn):
253
265
  conn = self.conn
254
266
  else:
@@ -369,14 +381,14 @@ class ControlFrameworkBase: # TODO: Add some common control methods here for ea
369
381
 
370
382
  def get_current_kontrollutslag(
371
383
  self, specific_control: str | None = None
372
- ) -> pd.DataFrame:
384
+ ) -> pd.DataFrame | None:
373
385
  """Method to get current content of the kontrollutslag table.
374
386
 
375
387
  Args:
376
388
  specific_control: Gets the current content of kontrollutslag table for this control. Defaults to None, which returns the data for all controls.
377
389
 
378
390
  Returns:
379
- pd.DataFrame containing the current kontrollutslag table for all controls or just the specified one.
391
+ pd.DataFrame containing the current kontrollutslag table for all controls or just the specified one or None if table empty.
380
392
 
381
393
  Raises:
382
394
  NotImplementedError: If connection is not EimerDBInstance or Ibis connection.
@@ -384,10 +396,22 @@ class ControlFrameworkBase: # TODO: Add some common control methods here for ea
384
396
  logger.info("Getting current kontrollutslag.")
385
397
  if isinstance(self.conn, EimerDBInstance):
386
398
  conn = ibis.polars.connect()
387
- kontrollutslag = self.conn.query(
388
- "SELECT * FROM kontrollutslag"
389
- ) # maybe add something like this?partition_select=self.applies_to_subset
390
- conn.create_table("kontrollutslag", kontrollutslag)
399
+ try:
400
+ kontrollutslag = self.conn.query(
401
+ "SELECT * FROM kontrollutslag"
402
+ ) # maybe add something like this?partition_select=self.applies_to_subset
403
+ conn.create_table("kontrollutslag", kontrollutslag)
404
+ except (
405
+ ValueError
406
+ ) as e: # TODO permanently fix this. Error caused by running .query on eimerdb table with no contents.
407
+ if str(e) == "max() arg is an empty sequence":
408
+ logger.warning(
409
+ "Did not find any contents in 'kontrollutslag', starting from scratch."
410
+ )
411
+ return None
412
+ else:
413
+ raise e
414
+
391
415
  elif conn_is_ibis(self.conn):
392
416
  conn = self.conn
393
417
  else:
@@ -413,39 +437,40 @@ class ControlFrameworkBase: # TODO: Add some common control methods here for ea
413
437
  else:
414
438
  specific_control = None
415
439
  existing_kontrollutslag = self.get_current_kontrollutslag(specific_control)
416
- if existing_kontrollutslag.empty:
417
- logger.debug("No existing rows found.")
418
- merged = control_results.merge(
419
- existing_kontrollutslag,
420
- on=[*self.applies_to_subset.keys(), "kontrollid", "ident", "refnr"],
421
- how="outer",
422
- indicator=True,
423
- )
424
- merged = (
425
- merged[merged["_merge"] == "left_only"][
426
- [
427
- *self.applies_to_subset.keys(),
428
- "kontrollid",
429
- "ident",
430
- "refnr",
431
- "verdi_x",
432
- "utslag_x",
440
+ if existing_kontrollutslag is not None:
441
+ control_results = control_results.merge(
442
+ existing_kontrollutslag,
443
+ on=[*self.applies_to_subset.keys(), "kontrollid", "ident", "refnr"],
444
+ how="outer",
445
+ indicator=True,
446
+ )
447
+ control_results = (
448
+ control_results[control_results["_merge"] == "left_only"][
449
+ [
450
+ *self.applies_to_subset.keys(),
451
+ "kontrollid",
452
+ "ident",
453
+ "refnr",
454
+ "verdi_x",
455
+ "utslag_x",
456
+ ]
433
457
  ]
434
- ]
435
- .rename(columns={"utslag_x": "utslag", "verdi_x": "verdi"})
436
- .dropna()
437
- )
438
- if merged.empty:
458
+ .rename(columns={"utslag_x": "utslag", "verdi_x": "verdi"})
459
+ .dropna()
460
+ )
461
+ else:
462
+ logger.debug("No existing rows found.")
463
+ if control_results.empty:
439
464
  logger.debug("No new rows found, ending here.")
440
465
  return None
441
466
  # Now to insert new rows into the table.
442
- logger.debug(f"Inserting {merged.shape[0]} new rows.")
467
+ logger.debug(f"Inserting {control_results.shape[0]} new rows.")
443
468
  if isinstance(self.conn, EimerDBInstance):
444
- self.conn.insert("kontrollutslag", merged)
469
+ self.conn.insert("kontrollutslag", control_results)
445
470
  elif conn_is_ibis(self.conn):
446
471
  conn = self.conn
447
472
  k = conn.table("kontrollutslag") # type: ignore[attr-defined]
448
- k.insert(merged)
473
+ k.insert(control_results)
449
474
  else:
450
475
  raise NotImplementedError(
451
476
  f"Connection type '{type(self.conn)}' is currently not implemented."
@@ -455,35 +480,38 @@ class ControlFrameworkBase: # TODO: Add some common control methods here for ea
455
480
  def update_existing_records(self, control_results: pd.DataFrame) -> None:
456
481
  """Updates the 'kontrollutslag' table based on results from new run of the method."""
457
482
  logger.debug("Starting process.")
483
+
458
484
  if control_results["kontrollid"].nunique() == 1:
459
485
  specific_control = next(iter(control_results["kontrollid"].unique()))
460
486
  else:
461
487
  specific_control = None
462
488
  existing_kontrollutslag = self.get_current_kontrollutslag(specific_control)
463
- if existing_kontrollutslag.empty:
464
- logger.info("No existing rows found, ending here.")
465
- return None
466
489
  logger.debug(
467
490
  f"control_results:\n{control_results}\nexisting_kontrollutslag:\n{existing_kontrollutslag}"
468
491
  )
469
-
470
- merged = control_results.merge(
471
- existing_kontrollutslag,
472
- on=["kontrollid", "ident", "refnr"],
473
- how="outer",
474
- indicator=True,
475
- ).dropna()
476
- if merged.empty:
492
+ if existing_kontrollutslag is None or existing_kontrollutslag.empty:
493
+ logger.info("No existing rows found, ending here.")
494
+ return None
495
+ else:
496
+ control_results = control_results.merge(
497
+ existing_kontrollutslag,
498
+ on=["kontrollid", "ident", "refnr"],
499
+ how="outer",
500
+ indicator=True,
501
+ ).dropna()
502
+ if control_results.empty:
477
503
  raise ValueError(
478
504
  "Combined results from 'control_results' and 'existing_kontrollutslag' is empty."
479
505
  )
480
- logger.debug(merged)
506
+ logger.debug(control_results)
481
507
  logger.debug(
482
- f"Utslag left:\n{merged['utslag_x'].value_counts()}\nUtslag right:\n{merged['utslag_y'].value_counts()}"
508
+ f"Utslag left:\n{control_results['utslag_x'].value_counts()}\nUtslag right:\n{control_results['utslag_y'].value_counts()}"
509
+ )
510
+ changed = control_results[
511
+ control_results["utslag_x"] != control_results["utslag_y"]
512
+ ][["kontrollid", "ident", "refnr", "verdi_x", "utslag_x"]].rename(
513
+ columns={"utslag_x": "utslag", "verdi_x": "verdi"}
483
514
  )
484
- changed = merged[merged["utslag_x"] != merged["utslag_y"]][
485
- ["kontrollid", "ident", "refnr", "verdi_x", "utslag_x"]
486
- ].rename(columns={"utslag_x": "utslag", "verdi_x": "verdi"})
487
515
  if changed.empty:
488
516
  logger.info("No changed rows, ending here.")
489
517
  return None
@@ -256,19 +256,38 @@ class AltinnControlView(ABC):
256
256
  logger.info("Refreshing view without re-running controls.")
257
257
 
258
258
  if isinstance(self.conn, EimerDBInstance):
259
- conn = ibis.polars.connect()
260
- skjemamottak = self.conn.query(
261
- "SELECT * FROM skjemamottak"
262
- ) # maybe add something like this?partition_select=self.applies_to_subset
263
- conn.create_table("skjemamottak", skjemamottak)
264
- kontroller = self.conn.query(
265
- "SELECT * FROM kontroller"
266
- ) # maybe add something like this?partition_select=self.applies_to_subset
267
- conn.create_table("kontroller", kontroller)
268
- kontrollutslag = self.conn.query(
269
- "SELECT * FROM kontrollutslag"
270
- ) # maybe add something like this?partition_select=self.applies_to_subset
271
- conn.create_table("kontrollutslag", kontrollutslag)
259
+ try:
260
+ conn = ibis.polars.connect()
261
+ skjemamottak = self.conn.query(
262
+ "SELECT * FROM skjemamottak"
263
+ ) # maybe add something like this?partition_select=self.applies_to_subset
264
+ conn.create_table("skjemamottak", skjemamottak)
265
+ kontroller = self.conn.query(
266
+ "SELECT * FROM kontroller"
267
+ ) # maybe add something like this?partition_select=self.applies_to_subset
268
+ conn.create_table("kontroller", kontroller)
269
+ kontrollutslag = self.conn.query(
270
+ "SELECT * FROM kontrollutslag"
271
+ ) # maybe add something like this?partition_select=self.applies_to_subset
272
+ conn.create_table("kontrollutslag", kontrollutslag)
273
+ except (
274
+ ValueError
275
+ ) as e: # TODO permanently fix this. Error caused by running .query on eimerdb table with no contents.
276
+ if str(e) == "max() arg is an empty sequence":
277
+ logger.warning(
278
+ "Did not find any contents in control tables, returning None, None and alert."
279
+ )
280
+ alert_store = [
281
+ create_alert(
282
+ "Finner ingen kontroller i dataene, prøv å kjøre kontroller.",
283
+ "warning",
284
+ ephemeral=True,
285
+ ),
286
+ *alert_store,
287
+ ]
288
+ return None, None, alert_store
289
+ else:
290
+ raise e
272
291
  elif conn_is_ibis(self.conn):
273
292
  conn = self.conn
274
293
  else:
@@ -19,6 +19,7 @@ from dash.dependencies import Input
19
19
  from dash.dependencies import Output
20
20
  from dash.dependencies import State
21
21
  from dash.exceptions import PreventUpdate
22
+ from pandas.io import parquet
22
23
  from ssb_poc_statlog_model.change_data_log import ChangeDataLog
23
24
 
24
25
  from ..setup.variableselector import VariableSelector
@@ -27,6 +28,13 @@ from ..utils.module_validation import module_validator
27
28
 
28
29
  logger = logging.getLogger(__name__)
29
30
 
31
+ def check_for_bucket_path(path):
32
+ """Temporary check to make sure users keep to using '/buckets/' paths.
33
+
34
+ Need to test more with UPath to make sure nothing unexpected happens.
35
+ """
36
+ if not path.startswith("/buckets/"):
37
+ raise NotImplementedError("Due to differences in how files in '/buckets/...' behave compared to files in the cloud buckets this functionality is currently limited to only work with paths that starts with '/buckets'.")
30
38
 
31
39
  class ParquetEditor: # TODO add validation of dataframe, workshop argument names
32
40
  """Simple module with the sole purpose of editing a parquet file.
@@ -67,7 +75,7 @@ class ParquetEditor: # TODO add validation of dataframe, workshop argument name
67
75
  self.module_name = self.__class__.__name__
68
76
  ParquetEditor._id_number += 1
69
77
  self.icon = "✏️" # TODO: Make visible
70
-
78
+ check_for_bucket_path(data_source)
71
79
  if "/inndata/" not in data_source:
72
80
  logger.warning(
73
81
  "Editing is supposed to happen between inndata and klargjort, you might not be following 'Datatilstander'."
@@ -679,6 +687,7 @@ def apply_edits(parquet_path: str | Path) -> pd.DataFrame:
679
687
  Returns:
680
688
  A pd.DataFrame with updated data.
681
689
  """
690
+ check_for_bucket_path(parquet_path)
682
691
  log_path = get_log_path(parquet_path)
683
692
  logger.debug(f"log_path: {log_path}")
684
693
  processlog = read_jsonl_log(log_path)
@@ -714,6 +723,7 @@ def export_from_parqueteditor(
714
723
  FileNotFoundError: if no log file is found.
715
724
  FileExistsError: If any of the files to export already exists and force_overwrite is False.
716
725
  """
726
+ check_for_bucket_path(data_source)
717
727
  log_path = get_log_path(data_source)
718
728
 
719
729
  # Read and update the jsonl log with actual data_target value
@@ -1,459 +0,0 @@
1
- # TODO: Rewrite to window/tab implementation model
2
-
3
- import logging
4
- from abc import ABC
5
- from abc import abstractmethod
6
- from typing import Any
7
- from typing import Literal
8
-
9
- import dash_ag_grid as dag
10
- import dash_bootstrap_components as dbc
11
- from dash import callback
12
- from dash import html
13
- from dash.dependencies import Input
14
- from dash.dependencies import Output
15
- from dash.dependencies import State
16
- from dash.exceptions import PreventUpdate
17
-
18
- from ..setup.variableselector import VariableSelector
19
- from ..utils import TabImplementation
20
- from ..utils import WindowImplementation
21
- from ..utils.alert_handler import create_alert
22
- from ..utils.eimerdb_helpers import create_partition_select
23
- from ..utils.module_validation import module_validator
24
-
25
- logger = logging.getLogger(__name__)
26
-
27
-
28
- default_col_def = {
29
- "filter": True,
30
- "resizable": True,
31
- "sortable": True,
32
- "floatingFilter": True,
33
- "editable": False,
34
- }
35
-
36
-
37
- class AltinnControlView(ABC):
38
- """Provides a layout and functionality for a modal that offers a tabular view of the controls."""
39
-
40
- _id_number: int = 0
41
-
42
- def __init__(
43
- self, time_units: list[str], control_dict: dict[str, Any], conn: object
44
- ) -> None: # TODO add proper annotation for control_dict value
45
- """Initializes the AltinnControlView with time units, control dictionary, and database connection.
46
-
47
- Args:
48
- time_units (list): A list of the time units used.
49
- control_dict (dict): A dictionary with one control class per skjema.
50
- conn (object): The eimerdb connection.
51
- """
52
- logger.warning(
53
- f"{self.__class__.__name__} is under development and may change in future releases."
54
- )
55
- assert hasattr(
56
- conn, "query"
57
- ), "The database connection object must have a 'query' method."
58
- self.module_number = AltinnControlView._id_number
59
- self.module_name = self.__class__.__name__
60
- AltinnControlView._id_number += 1
61
-
62
- self.icon = "⚠️"
63
- self.label = "Kontroll"
64
-
65
- self.control_dict = control_dict
66
- self.conn = conn
67
- self._is_valid()
68
- self.module_layout = self.create_layout()
69
- self.variableselector = VariableSelector(
70
- selected_inputs=time_units, selected_states=[]
71
- )
72
- self.time_units = [
73
- self.variableselector.get_option(x).id.removeprefix("var-")
74
- for x in time_units
75
- ]
76
- self.module_callbacks()
77
- module_validator(self)
78
-
79
- def _is_valid(self) -> None:
80
- VariableSelector([], []).get_option("var-altinnskjema", search_target="id")
81
- VariableSelector([], []).get_option("var-ident", search_target="id")
82
-
83
- def create_layout(self) -> html.Div:
84
- """Generates the layout for the AltinnControlView module.
85
-
86
- Returns:
87
- layout: A Div element containing two tables, kontroller and kontrollutslag.
88
- """
89
- layout = html.Div(
90
- style={"width": "100%"},
91
- children=[
92
- dbc.Row(
93
- [
94
- dbc.Col(
95
- dbc.Button(
96
- "🔄 Refresh",
97
- id="kontrollermodal-refresh",
98
- ),
99
- width="auto",
100
- ),
101
- dbc.Col(
102
- dbc.Button(
103
- "Kjør alle kontrollene",
104
- id="kontrollermodal-kontrollbutton",
105
- ),
106
- width="auto",
107
- ),
108
- dbc.Col(
109
- dbc.Button(
110
- "Insert nye kontrollutslag",
111
- id="kontrollermodal-insertbutton",
112
- ),
113
- width="auto",
114
- ),
115
- dbc.Col(html.P(id="kontrollermodal-vars")),
116
- ],
117
- className="g-2",
118
- ),
119
- dbc.Row(
120
- dag.AgGrid(
121
- id="kontroller-table1",
122
- defaultColDef=default_col_def,
123
- className="ag-theme-alpine header-style-on-filter",
124
- columnSize="responsiveSizeToFit",
125
- dashGridOptions={
126
- "pagination": True,
127
- "rowSelection": "single",
128
- "rowHeight": 28,
129
- },
130
- ),
131
- style={"flexShrink": 0},
132
- ),
133
- dbc.Row(
134
- dbc.Col(
135
- dag.AgGrid(
136
- id="kontroller-table2",
137
- defaultColDef=default_col_def,
138
- className="ag-theme-alpine header-style-on-filter",
139
- columnSize="responsiveSizeToFit",
140
- dashGridOptions={
141
- "pagination": True,
142
- "rowSelection": "single",
143
- "rowHeight": 28,
144
- },
145
- ),
146
- width=12,
147
- )
148
- ),
149
- ],
150
- )
151
- return layout
152
-
153
- @abstractmethod
154
- def layout(self) -> html.Div:
155
- """Defines the layout for the AltinnControlView module.
156
-
157
- This is an abstract method that must be implemented by subclasses to define the module's layout.
158
-
159
- Returns:
160
- html.Div: A Dash HTML Div component representing the layout of the module.
161
- """
162
- pass
163
-
164
- def create_callback_components(
165
- self, input_type: Literal["Input", "State"] = "Input"
166
- ) -> list[str]:
167
- """Generates a list of dynamic Dash Input or State components."""
168
- component = Input if input_type == "Input" else State
169
- logger.warning(
170
- f'{[component(f"var-{unit}", "value") for unit in self.time_units]}'
171
- )
172
- return [component(f"var-{unit}", "value") for unit in self.time_units]
173
-
174
- def module_callbacks(self) -> None:
175
- """Registers Dash callbacks for the AltinnControlView module."""
176
-
177
- @callback( # type: ignore[misc]
178
- Output("kontroller-table1", "rowData"),
179
- Output("kontroller-table1", "columnDefs"),
180
- Input("var-altinnskjema", "value"),
181
- Input("kontrollermodal-refresh", "n_clicks"),
182
- self.variableselector.get_all_inputs(),
183
- # *self.create_callback_components("Input"),
184
- )
185
- def kontrollutslag_antall(
186
- skjema: str, n_clicks: int, *args: Any
187
- ) -> tuple[list[dict[str, Any]], list[dict[str, str | bool]]]:
188
- logger.debug(
189
- f"Args:\n"
190
- f"skjema: {skjema}\n"
191
- f"n_clicks: {n_clicks}\n"
192
- f"args: {args}"
193
- )
194
- partition_args = dict(zip(self.time_units, args, strict=False))
195
- df1 = self.conn.query(
196
- """SELECT
197
- kontroller.skjema,
198
- kontroller.kontrollid,
199
- kontroller.type,
200
- kontroller.beskrivelse,
201
- kontroller.kontrollvar,
202
- kontroller.varsort,
203
- utslag.ant_utslag
204
- FROM kontroller AS kontroller
205
- JOIN (
206
- SELECT kontrollid, COUNT(row_id) AS ant_utslag
207
- FROM kontrollutslag
208
- WHERE utslag = True
209
- GROUP BY kontrollid
210
- ) AS utslag ON kontroller.kontrollid = utslag.kontrollid
211
- """,
212
- create_partition_select(
213
- desired_partitions=self.time_units, skjema=skjema, **partition_args
214
- ),
215
- )
216
- df2 = self.conn.query(
217
- """SELECT k.kontrollid, COUNT(row_id) AS uediterte FROM kontrollutslag AS k
218
- JOIN (
219
- SELECT ident, refnr FROM skjemamottak
220
- WHERE editert = False
221
- ) AS subq ON subq.ident = k.ident AND subq.refnr = k.refnr
222
- WHERE utslag = True
223
- GROUP BY k.kontrollid""",
224
- create_partition_select(
225
- desired_partitions=self.time_units, skjema=skjema, **partition_args
226
- ),
227
- )
228
- df = df1.merge(df2, on="kontrollid", how="left")
229
- columns = [
230
- {
231
- "headerName": col,
232
- "field": col,
233
- "hide": col == "varsort",
234
- "flex": 2 if col == "beskrivelse" else 1,
235
- "tooltipField": col if col == "beskrivelse" else None,
236
- }
237
- for col in df.columns
238
- ]
239
- columns[0]["checkboxSelection"] = True
240
- columns[0]["headerCheckboxSelection"] = True
241
- return df.to_dict("records"), columns
242
-
243
- @callback( # type: ignore[misc]
244
- Output("kontroller-table2", "rowData"),
245
- Output("kontroller-table2", "columnDefs"),
246
- Input("kontroller-table1", "selectedRows"),
247
- self.variableselector.get_all_inputs(),
248
- # *self.create_callback_components("State"),
249
- )
250
- def kontrollutslag_mikro(
251
- current_row: list[dict[str, Any]] | None, *args: Any
252
- ) -> tuple[list[dict[str, Any]], list[dict[str, str | bool]]]:
253
- if current_row is None:
254
- raise PreventUpdate
255
- logger.debug(f"Args:\ncurrent_row: {current_row}\nargs: {args}")
256
- partition_args = dict(zip(self.time_units, args, strict=False))
257
- kontrollid = current_row[0]["kontrollid"]
258
- kontrollvar = current_row[0]["kontrollvar"]
259
- skjema = current_row[0]["skjema"]
260
- varsort = current_row[0]["varsort"]
261
- df = self.conn.query(
262
- f"""
263
- SELECT utslag.ident, utslag.refnr, utslag.kontrollid, utslag.utslag, s.editert, utslag.verdi
264
- FROM kontrollutslag AS utslag
265
- JOIN (
266
- SELECT refnr AS s_refnr, editert, ident
267
- FROM skjemamottak
268
- WHERE aktiv = True
269
- ) AS s ON utslag.refnr = s.s_refnr AND utslag.ident = s.ident
270
- WHERE utslag.kontrollid = '{kontrollid}' AND utslag.utslag = True
271
- ORDER BY editert, utslag.verdi {varsort}
272
- """,
273
- partition_select={
274
- "kontrollutslag": create_partition_select(
275
- desired_partitions=self.time_units,
276
- skjema=skjema,
277
- **partition_args,
278
- ),
279
- "enhetsinfo": create_partition_select(
280
- desired_partitions=self.time_units,
281
- skjema=None,
282
- **partition_args,
283
- ),
284
- "skjemamottak": create_partition_select(
285
- desired_partitions=self.time_units,
286
- skjema=skjema,
287
- **partition_args,
288
- ),
289
- },
290
- ).rename(columns={"verdi": kontrollvar})
291
- columns = [{"headerName": col, "field": col} for col in df.columns]
292
- columns[0]["checkboxSelection"] = True
293
- columns[0]["headerCheckboxSelection"] = True
294
- return df.to_dict("records"), columns
295
-
296
- @callback( # type: ignore[misc]
297
- Output("var-ident", "value", allow_duplicate=True),
298
- Input("kontroller-table2", "selectedRows"),
299
- prevent_initial_call=True,
300
- )
301
- def kontroll_detail_click(input1: list[dict[str, Any]]) -> str:
302
- logger.debug(f"Args:\ninput1: {input1}")
303
- selected = str(input1[0]["ident"])
304
- return selected
305
-
306
- @callback( # type: ignore[misc]
307
- Output("kontrollermodal-vars", "children"),
308
- Input("var-altinnskjema", "value"),
309
- self.variableselector.get_all_inputs(),
310
- # *self.create_callback_components("Input"),
311
- )
312
- def altinnskjema(skjema: str, *args: Any) -> str:
313
- logger.debug(f"Args:\nskjema: {skjema}\nargs: {args}")
314
- partition_args = dict(zip(self.time_units, args, strict=False))
315
- if partition_args is not None and skjema is not None:
316
- valgte_vars = (
317
- f"valgt partisjon: {partition_args}\nvalgt skjema: {skjema}"
318
- )
319
- return valgte_vars
320
-
321
- @callback( # type: ignore[misc]
322
- Output("alert_store", "data", allow_duplicate=True),
323
- Input("kontrollermodal-kontrollbutton", "n_clicks"),
324
- State("var-altinnskjema", "value"),
325
- State("alert_store", "data"),
326
- self.variableselector.get_all_inputs(),
327
- # *self.create_callback_components("State"),
328
- prevent_initial_call=True,
329
- )
330
- def kontrollkjøring(
331
- n_clicks: int, skjema: str, alert_store: list[dict[str, Any]], *args: Any
332
- ) -> list[dict[str, Any]]:
333
- logger.debug(
334
- f"Args:\n"
335
- f"n_clicks: {n_clicks}\n"
336
- f"skjema: {skjema}\n"
337
- f"alert_store: {alert_store}\n"
338
- f"args: {args}"
339
- )
340
- partition_args = dict(zip(self.time_units, args, strict=False))
341
- partitions = create_partition_select(
342
- desired_partitions=self.time_units, skjema=None, **partition_args
343
- )
344
- partitions_skjema = create_partition_select(
345
- desired_partitions=self.time_units, skjema=skjema, **partition_args
346
- )
347
- if n_clicks > 0:
348
- try:
349
- control_class = self.control_dict[skjema](
350
- partitions, partitions_skjema, self.conn
351
- )
352
- n_updates = control_class.execute_controls()
353
- alert_store = [
354
- create_alert(
355
- f"Kontrollkjøringa er ferdig. {n_updates} rader oppdatert.",
356
- "info",
357
- ephemeral=True,
358
- ),
359
- *alert_store,
360
- ]
361
- except Exception as e:
362
- logger.debug(f"Executing controls failed!\n{e}")
363
- alert_store = [
364
- create_alert(
365
- f"Kontrollkjøringa feilet. {e!s}",
366
- "danger",
367
- ephemeral=True,
368
- ),
369
- *alert_store,
370
- ]
371
- return alert_store
372
- logger.debug("kontrollkjøring: PreventUpdate raised")
373
- raise PreventUpdate
374
-
375
- @callback( # type: ignore[misc]
376
- Output("alert_store", "data", allow_duplicate=True),
377
- Input("kontrollermodal-insertbutton", "n_clicks"),
378
- State("var-altinnskjema", "value"),
379
- State("alert_store", "data"),
380
- self.variableselector.get_all_inputs(),
381
- # *self.create_callback_components("State"),
382
- prevent_initial_call=True,
383
- )
384
- def kontrollkjøring_insert(
385
- n_clicks: int, skjema: str, alert_store: list[dict[str, Any]], *args: Any
386
- ) -> list[dict[str, Any]]:
387
- logger.debug(
388
- f"Args:\n"
389
- f"n_clicks: {n_clicks}\n"
390
- f"skjema: {skjema}\n"
391
- f"alert_store: {alert_store}\n"
392
- f"args: {args}"
393
- )
394
- partition_args = dict(zip(self.time_units, args, strict=False))
395
- partitions = create_partition_select(
396
- desired_partitions=self.time_units, skjema=None, **partition_args
397
- )
398
- partitions_skjema = create_partition_select(
399
- desired_partitions=self.time_units, skjema=skjema, **partition_args
400
- )
401
- if n_clicks > 0:
402
- try:
403
- control_class = self.control_dict[skjema](
404
- partitions, partitions_skjema, self.conn
405
- )
406
- n_inserts = control_class.insert_new_rows()
407
- alert_store = [
408
- create_alert(
409
- f"Kontrollkjøringa er ferdig. {n_inserts} nye rader er lastet inn.",
410
- "info",
411
- ephemeral=True,
412
- ),
413
- *alert_store,
414
- ]
415
- except Exception as e:
416
- logger.debug(f"Inserting failed!\n{e}", exc_info=True)
417
- alert_store = [
418
- create_alert(
419
- f"Kontrollkjøringa feilet. {e!s}",
420
- "danger",
421
- ephemeral=True,
422
- ),
423
- *alert_store,
424
- ]
425
- return alert_store
426
- logger.debug("PreventUpdate raised")
427
- raise PreventUpdate
428
-
429
-
430
- class AltinnControlViewTab(TabImplementation, AltinnControlView):
431
- """AltinnControlView implemented as a tab."""
432
-
433
- def __init__(
434
- self, time_units: list[str], control_dict: dict[str, Any], conn: object
435
- ) -> None:
436
- """Initializes the AltinnControlViewTab module."""
437
- AltinnControlView.__init__(
438
- self,
439
- time_units=time_units,
440
- control_dict=control_dict,
441
- conn=conn,
442
- )
443
- TabImplementation.__init__(self)
444
-
445
-
446
- class AltinnControlViewWindow(WindowImplementation, AltinnControlView):
447
- """AltinnControlView implemented as a window."""
448
-
449
- def __init__(
450
- self, time_units: list[str], control_dict: dict[str, Any], conn: object
451
- ) -> None:
452
- """Initializes the AltinnControlViewWindow module."""
453
- AltinnControlView.__init__(
454
- self,
455
- time_units=time_units,
456
- control_dict=control_dict,
457
- conn=conn,
458
- )
459
- WindowImplementation.__init__(self)