dipex 5.5.2__tar.gz → 5.7.0__tar.gz

{dipex-5.5.2 → dipex-5.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: DIPEX
-Version: 5.5.2
+Version: 5.7.0
 Summary: OS2mo-data-import-and-export
 Home-page: https://magenta.dk/
 License: MPL-2.0

dipex-5.7.0/integrations/SD_Lon/alembic/env.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Magenta ApS <https://magenta.dk>
+# SPDX-License-Identifier: MPL-2.0
+# Initial version generated using `poetry run alembic init alembic`
+from logging.config import fileConfig
+
+from alembic import context
+from payload_db.engine import get_db_url
+from payload_db.engine import get_engine
+from payload_db.models import Base
+
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    context.configure(
+        url=get_db_url(),
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = get_engine()
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

dipex-5.7.0/integrations/SD_Lon/alembic/versions/26d00ae9c67f_add_payload_table.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: Magenta ApS
+# SPDX-License-Identifier: MPL-2.0
+"""Add Payload table
+
+Revision ID: 26d00ae9c67f
+Revises:
+Create Date: 2023-09-29 12:31:28.477508
+
+"""
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "26d00ae9c67f"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "payload",
+        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column(
+            "timestamp",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("full_url", sa.TEXT(), nullable=True),
+        sa.Column("params", sa.TEXT(), nullable=True),
+        sa.Column("response", sa.TEXT(), nullable=True),
+        sa.Column("status_code", sa.Integer(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("payload")
+    # ### end Alembic commands ###

dipex-5.7.0/integrations/SD_Lon/payload_db/__init__.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: Magenta ApS
+# SPDX-License-Identifier: MPL-2.0
+from uuid import UUID
+
+from sqlalchemy.orm import sessionmaker
+
+from .engine import get_engine
+from .models import Payload
+
+
+Session = sessionmaker()
+
+
+def log_payload(
+    request_uuid: UUID,
+    full_url: str,
+    params: str,
+    response: str,
+    status_code: int,
+) -> None:
+    """Log a given SD payload to the payload database
+
+    Args:
+        request_uuid: Unique UUID of this API request. Must be generated by the caller.
+        full_url: Full URL of API endpoint called, e.g. containing the SD server name.
+        params: Stringified version of the `payload` variable in `sd_lookup`.
+        response: The raw/unparsed XML response from the SD API endpoint called.
+        status_code: The HTTP status code from the SD API endpoint called.
+    """
+    Session.configure(bind=get_engine())
+    session = Session()
+    payload = Payload(
+        id=request_uuid,
+        full_url=full_url,
+        params=params,
+        response=response,
+        status_code=status_code,
+    )
+    session.add(payload)
+    session.commit()

dipex-5.7.0/integrations/SD_Lon/payload_db/engine.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Magenta ApS
+# SPDX-License-Identifier: MPL-2.0
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+
+
+def get_db_url() -> str:
+    return f"postgresql+psycopg2://{os.environ['APP_DBUSER']}:{os.environ['APP_DBPASSWORD']}@{os.environ['PGHOST']}/{os.environ['APP_DATABASE']}"
+
+
+def get_engine() -> Engine:
+    return create_engine(get_db_url())

dipex-5.7.0/integrations/SD_Lon/payload_db/models.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Magenta ApS
+# SPDX-License-Identifier: MPL-2.0
+from sqlalchemy import Column
+from sqlalchemy import DateTime
+from sqlalchemy import Integer
+from sqlalchemy.dialects.postgresql import TEXT
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.sql import func
+
+
+Base = declarative_base()
+
+
+class Payload(Base):  # type: ignore
+    __tablename__ = "payload"
+
+    id = Column(UUID(as_uuid=True), primary_key=True)
+    timestamp = Column(DateTime(timezone=True), server_default=func.now())
+    full_url = Column(TEXT)
+    params = Column(TEXT)
+    response = Column(TEXT)
+    status_code = Column(Integer)

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/fix_departments.py

@@ -1,10 +1,12 @@
 import datetime
 import json
+import uuid
 from typing import Any
 from typing import Optional
 from typing import List
 from typing import OrderedDict
 from uuid import UUID
+from uuid import uuid4
 
 import requests
 from os2mo_helpers.mora_helpers import MoraHelper
@@ -77,8 +79,13 @@ class FixDepartments:
         """
         inst_id = self.settings.sd_institution_identifier
         params = {"UUIDIndicator": "true", "InstitutionIdentifier": inst_id}
+        request_uuid = uuid4()
+        logger.info("get_institution", request_uuid=request_uuid)
         institution_info = sd_lookup(
-            "GetInstitution20111201", settings=self.settings, params=params
+            "GetInstitution20111201",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
         )
         institution = institution_info["Region"]["Institution"]
         institution_uuid = institution["InstitutionUUIDIdentifier"]
@@ -262,8 +269,13 @@ class FixDepartments:
         if uuid is None and shortname is None:
             raise Exception("Provide either uuid or shortname")
 
+        request_uuid = uuid4()
+        logger.info("get_department", request_uuid=request_uuid)
         department_info = sd_lookup(
-            "GetDepartment20111201", settings=self.settings, params=params
+            "GetDepartment20111201",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
         )
         department = department_info.get("Department")
         if department is None:
@@ -343,9 +355,17 @@ class FixDepartments:
         for time_delta in time_deltas:
             effective_date = validity_date + datetime.timedelta(days=time_delta)
             params["EffectiveDate"] = (effective_date.strftime("%d.%m.%Y"),)
-
+            request_uuid = uuid4()
+            logger.info(
+                "_read_department_engagements",
+                request_uuid=request_uuid,
+                time_delta=time_delta,
+            )
             employments = sd_lookup(
-                "GetEmployment20111201", settings=self.settings, params=params
+                "GetEmployment20111201",
+                settings=self.settings,
+                params=params,
+                request_uuid=request_uuid,
             )
             people = employments.get("Person", [])
             if not isinstance(people, list):
@@ -467,8 +487,13 @@ class FixDepartments:
             "EffectiveDate": validity_date.strftime("%d.%m.%Y"),
             "DepartmentUUIDIdentifier": unit_uuid,
         }
+        request_uuid = uuid4()
+        logger.info("get_parent", request_uuid=request_uuid)
         parent_response = sd_lookup(
-            "GetDepartmentParent20190701", settings=self.settings, params=params
+            "GetDepartmentParent20190701",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
         )
         if "DepartmentParent" not in parent_response:
             logger.error(

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/sd_changed_at.py

@@ -3,6 +3,7 @@ import logging
 import pathlib
 import sqlite3
 import sys
+import uuid
 from functools import lru_cache
 from itertools import tee
 from operator import itemgetter
@@ -314,7 +315,15 @@ class ChangeAtSD:
                     "DeactivationDate": "31.12.9999",
                 }
             )
-        response = sd_lookup(url, settings=self.settings, params=params)
+
+        request_uuid = uuid.uuid4()
+        logger.info("read_employment_changed", request_uuid=request_uuid)
+        response = sd_lookup(
+            url,
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
+        )
 
         employment_response = ensure_list(response.get("Person", []))
 
@@ -352,8 +361,15 @@ class ChangeAtSD:
             params["DeactivationDate"] = to_date.strftime("%d.%m.%Y")
             params["DeactivationTime"] = to_date.strftime("%H:%M")
 
+        request_uuid = uuid.uuid4()
+        logger.info("get_sd_persons_changed", request_uuid=request_uuid)
         url = "GetPersonChangedAtDate20111201"
-        response = sd_lookup(url, settings=self.settings, params=params)
+        response = sd_lookup(
+            url,
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
+        )
         persons_changed = ensure_list(response.get("Person", []))
         return persons_changed
 
@@ -380,7 +396,14 @@ class ChangeAtSD:
             "PostalAddressIndicator": "false",
         }
         url = "GetPerson20111201"
-        response = sd_lookup(url, settings=self.settings, params=params)
+        request_uuid = uuid.uuid4()
+        logger.info("get_sd_person", request_uuid=request_uuid)
+        response = sd_lookup(
+            url,
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
+        )
         person = ensure_list(response.get("Person", []))
         return person
 
@@ -969,7 +992,14 @@ class ChangeAtSD:
                     "UUIDIndicator": "true",
                     "DepartmentIdentifier": department["DepartmentIdentifier"],
                 }
-                response = sd_lookup(url, settings=self.settings, params=params)
+                request_uuid = uuid.uuid4()
+                logger.info("edit_engagement_department", request_uuid=request_uuid)
+                response = sd_lookup(
+                    url,
+                    settings=self.settings,
+                    params=params,
+                    request_uuid=request_uuid,
+                )
                 logger.warning("GetDepartment returned", response=response)
                 org_unit = response["Department"]["DepartmentUUIDIdentifier"]
                 if org_unit is None:

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/sd_changed_at_redo.py

@@ -6,14 +6,19 @@
 # angiv uuider for de berørte på kommandolinien
 import datetime
 import sys
+import uuid
 
 import click
 
 from .config import get_changed_at_settings
+from .log import get_logger
 from .sd_changed_at import ChangeAtSD
 from .sd_common import sd_lookup
 
 
+logger = get_logger()
+
+
 def read_person(cat, cpr):
     params = {
         "EffectiveDate": cat.from_date.strftime("%d.%m.%Y"),
@@ -24,7 +29,9 @@ def read_person(cat, cpr):
         "PostalAddressIndicator": "false",
     }
     url = "GetPerson20111201"
-    response = sd_lookup(url, params=params)
+    request_uuid = uuid.uuid4()
+    logger.info("read_person", request_uuid=request_uuid)
+    response = sd_lookup(url, params=params, request_uuid=request_uuid)
     person = response.get("Person", [])
 
     if not isinstance(person, list):

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/sd_common.py

@@ -12,6 +12,7 @@ from typing import Union
 
 import requests
 import xmltodict
+from payload_db import log_payload
 from ra_utils.load_settings import load_settings
 
 from .config import CommonSettings
@@ -43,6 +44,7 @@ def sd_lookup(
     url: str,
     settings: Optional[CommonSettings] = None,
     params: Optional[Dict[str, Any]] = None,
+    request_uuid: Optional[uuid.UUID] = None,
 ) -> OrderedDict:
     """Fire a requests against SD."""
     logger.info("Retrieve: {}".format(url))
@@ -75,9 +77,16 @@ def sd_lookup(
         auth=auth,
     )
 
-    # This line logs CPR numbers - instead we will persist the SD payloads
-    # in a DB (see Redmine #57093)
-    # logger.debug("Response: {}".format(response.text))
+    try:
+        log_payload(
+            request_uuid=request_uuid,
+            full_url=full_url,
+            params=str(payload),
+            response=response.text,
+            status_code=response.status_code,
+        )
+    except Exception:
+        logger.exception("could not save SD response to payload database")
 
     dict_response = xmltodict.parse(response.text)
 
@@ -279,5 +288,12 @@ def read_employment_at(
     if employment_id:
         params.update({"EmploymentIdentifier": employment_id})
 
-    response = sd_lookup(url, settings=settings, params=params)
+    request_uuid = uuid.uuid4()
+    logger.info("read_employment_at", request_uuid=request_uuid)
+    response = sd_lookup(
+        url,
+        settings=settings,
+        params=params,
+        request_uuid=request_uuid,
+    )
     return response.get("Person")

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/sd_fixup.py

@@ -1,4 +1,5 @@
 import datetime
+import uuid
 from datetime import date
 from functools import partial
 from operator import itemgetter
@@ -25,6 +26,7 @@ from tqdm import tqdm
 
 from . import sd_payloads
 from .config import get_importer_settings
+from .log import get_logger
 from .sd_changed_at import ChangeAtSD
 from .sd_common import EmploymentStatus
 from .sd_common import mora_assert
@@ -32,6 +34,9 @@ from .sd_common import primary_types
 from .sd_common import sd_lookup
 
 
+logger = get_logger()
+
+
 def fetch_user_employments(cpr: str) -> List:
     # Notice, this will not get future engagements
     params = {
@@ -47,8 +52,14 @@ def fetch_user_employments(cpr: str) -> List:
         "SalaryCodeGroupIndicator": "false",
         "EffectiveDate": date.today().strftime("%d.%m.%Y"),
     }
-
-    sd_employments_response = sd_lookup("GetEmployment20111201", None, params)
+    request_uuid = uuid.uuid4()
+    logger.info("fetch_user_employments", request_uuid=request_uuid)
+    sd_employments_response = sd_lookup(
+        "GetEmployment20111201",
+        settings=None,
+        params=params,
+        request_uuid=request_uuid,
+    )
     if "Person" not in sd_employments_response:
         return []
 

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/sd_importer.py

@@ -11,6 +11,7 @@ from operator import itemgetter
 from typing import Any, OrderedDict
 from typing import Dict
 from typing import Optional
+from uuid import uuid4
 
 import click
 from anytree import Node
@@ -207,8 +208,13 @@ class SdImport:
             "UUIDIndicator": "true",
             "EmploymentDepartmentIndicator": "false",
         }
+        request_uuid = uuid4()
+        logger.info("_read_department_info", request_uuid=request_uuid)
         departments = sd_lookup(
-            "GetDepartment20111201", settings=self.settings, params=params
+            "GetDepartment20111201",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
         )
 
         for department in departments["Department"]:
@@ -388,7 +394,14 @@ class SdImport:
             "InstitutionIdentifier": self.settings.sd_institution_identifier,
             "UUIDIndicator": "true",
         }
-        r = sd_lookup("GetInstitution20111201", settings=self.settings, params=params)
+        request_uuid = uuid4()
+        logger.info("_get_institution", request_uuid=request_uuid)
+        r = sd_lookup(
+            "GetInstitution20111201",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
+        )
         return uuid.UUID(r["Region"]["Institution"]["InstitutionUUIDIdentifier"])
 
     def _create_org_tree_structure(self):
@@ -463,16 +476,26 @@ class SdImport:
             "PostalAddressIndicator": "false",
             "EffectiveDate": self.import_date,
         }
+        request_uuid = uuid4()
+        logger.info("add_people: active_people", request_uuid=request_uuid)
         active_people = sd_lookup(
-            "GetPerson20111201", settings=self.settings, params=params
+            "GetPerson20111201",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
         )
         if not isinstance(active_people["Person"], list):
             active_people["Person"] = [active_people["Person"]]
 
         params["StatusActiveIndicator"] = False
         params["StatusPassiveIndicator"] = True
+        request_uuid = uuid4()
+        logger.info("add_people: passive_people", request_uuid=request_uuid)
         passive_people = sd_lookup(
-            "GetPerson20111201", settings=self.settings, params=params
+            "GetPerson20111201",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
         )
         if not isinstance(passive_people["Person"], list):
             passive_people["Person"] = [passive_people["Person"]]
@@ -569,8 +592,13 @@ class SdImport:
             "DeactivationDate": self.import_date,
             "UUIDIndicator": "true",
         }
+        request_uuid = uuid4()
+        logger.info("create_ou_tree", request_uuid=request_uuid)
         organisation = sd_lookup(
-            "GetOrganization20111201", settings=self.settings, params=params
+            "GetOrganization20111201",
+            settings=self.settings,
+            params=params,
+            request_uuid=request_uuid,
         )
         departments = organisation["Organization"]["DepartmentReference"]
 

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/sync_job_id.py

@@ -1,5 +1,6 @@
 import logging
 import sys
+import uuid
 
 import click
 import requests
@@ -99,8 +100,15 @@ class JobIdSync:
         params = {
             "JobPositionIdentifier": job_pos_id,
         }
+        request_uuid = uuid.uuid4()
+        logger.info("_get_job_pos_id_from_sd", request_uuid=request_uuid)
         try:
-            job_pos_response = sd_lookup("GetProfession20080201", self.settings, params)
+            job_pos_response = sd_lookup(
+                "GetProfession20080201",
+                settings=self.settings,
+                params=params,
+                request_uuid=request_uuid,
+            )
         except Exception:  # TODO: Be specific here
             logger.info("This job_position could not be found in SD")
             return None

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/test_mo_against_sd.py

@@ -1,4 +1,5 @@
 import json
+import uuid
 from datetime import datetime
 from itertools import starmap
 from operator import itemgetter
@@ -7,11 +8,15 @@ import click
 from more_itertools import flatten
 from os2mo_helpers.mora_helpers import MoraHelper
 
+from .log import get_logger
 from .sd_common import EmploymentStatus
 from .sd_common import load_settings
 from .sd_common import sd_lookup
 
 
+logger = get_logger()
+
+
 def progress_iterator(elements, outputter, mod=10):
     total = len(elements)
     for i, element in enumerate(elements, start=1):
@@ -150,7 +155,13 @@ class TestMOAgainstSd(object):
             "SalaryCodeGroupIndicator": "false",
             "EffectiveDate": self.date.strftime("%d.%m.%Y"),
         }
-        sd_employments_response = sd_lookup("GetEmployment20111201", params)
+        request_uuid = uuid.uuid4()
+        logger.info("check_user", request_uuid=request_uuid)
+        sd_employments_response = sd_lookup(
+            "GetEmployment20111201",
+            params=params,
+            request_uuid=request_uuid,
+        )
         if "Person" not in sd_employments_response:
             return mo_uuid, {
                 "uuid": mo_uuid,

{dipex-5.5.2 → dipex-5.7.0}/integrations/SD_Lon/sdlon/test_sd_connectivity.py

@@ -1,13 +1,18 @@
 import logging
+import uuid
 from uuid import UUID
 
 import click
 from pydantic import ValidationError
 
 from .config import get_changed_at_settings
+from .log import get_logger
 from .sd_common import sd_lookup
 
 
+logger = get_logger()
+
+
 class TestSdConnectivity(object):
     def __init__(self):
         self.validation_error = None
@@ -36,11 +41,14 @@ class TestSdConnectivity(object):
             "UUIDIndicator": "true",
             "InstitutionIdentifier": self.settings.sd_institution_identifier,
         }
+        request_uuid = uuid.uuid4()
+        logger.info("_check_contact_to_sd", request_uuid=request_uuid)
         try:
             institution_info = sd_lookup(
                 "GetInstitution20111201",
                 settings=self.settings,
                 params=params,
+                request_uuid=request_uuid,
             )
         except Exception:
             logging.exception("Fejl i kontakt til SD Løn")