dipex 5.7.0__tar.gz → 5.7.2__tar.gz

{dipex-5.7.0 → dipex-5.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: DIPEX
-Version: 5.7.0
+Version: 5.7.2
 Summary: OS2mo-data-import-and-export
 Home-page: https://magenta.dk/
 License: MPL-2.0

{dipex-5.7.0 → dipex-5.7.2}/exporters/sql_export/lora_cache.py

@@ -19,8 +19,8 @@ PICKLE_PROTOCOL = pickle.DEFAULT_PROTOCOL
 
 
 def get_cache(resolve_dar=True, full_history=False, skip_past=False, settings=None):
-    if GqlLoraCacheSettings().sentry_dsn:
-        sentry_sdk.init(dsn=GqlLoraCacheSettings().sentry_dsn)
+    if GqlLoraCacheSettings().job_settings.sentry_dsn:
+        sentry_sdk.init(dsn=GqlLoraCacheSettings().job_settings.sentry_dsn)
 
     if GqlLoraCacheSettings().use_new_cache:
         return GQLLoraCache(

{dipex-5.7.0 → dipex-5.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "DIPEX"
-version = "5.7.0"
+version = "5.7.2"
 authors = ["Magenta ApS <info@magenta.dk>"]
 description = "OS2mo-data-import-and-export"
 license = "MPL-2.0"

dipex-5.7.0/integrations/SD_Lon/alembic/env.py

@@ -1,68 +0,0 @@
-# SPDX-FileCopyrightText: Magenta ApS <https://magenta.dk>
-# SPDX-License-Identifier: MPL-2.0
-# Initial version generated using `poetry run alembic init alembic`
-from logging.config import fileConfig
-
-from alembic import context
-from payload_db.engine import get_db_url
-from payload_db.engine import get_engine
-from payload_db.models import Base
-
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = Base.metadata
-
-
-def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
-
-    """
-    context.configure(
-        url=get_db_url(),
-        target_metadata=target_metadata,
-        literal_binds=True,
-        dialect_opts={"paramstyle": "named"},
-    )
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def run_migrations_online() -> None:
-    """Run migrations in 'online' mode.
-
-    In this scenario we need to create an Engine
-    and associate a connection with the context.
-
-    """
-    connectable = get_engine()
-    with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
-        with context.begin_transaction():
-            context.run_migrations()
-
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()

dipex-5.7.0/integrations/SD_Lon/alembic/versions/26d00ae9c67f_add_payload_table.py

@@ -1,47 +0,0 @@
-# SPDX-FileCopyrightText: Magenta ApS
-# SPDX-License-Identifier: MPL-2.0
-"""Add Payload table
-
-Revision ID: 26d00ae9c67f
-Revises:
-Create Date: 2023-09-29 12:31:28.477508
-
-"""
-from typing import Sequence
-from typing import Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision: str = "26d00ae9c67f"
-down_revision: Union[str, None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "payload",
-        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column(
-            "timestamp",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=True,
-        ),
-        sa.Column("full_url", sa.TEXT(), nullable=True),
-        sa.Column("params", sa.TEXT(), nullable=True),
-        sa.Column("response", sa.TEXT(), nullable=True),
-        sa.Column("status_code", sa.Integer(), nullable=True),
-        sa.PrimaryKeyConstraint("id"),
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table("payload")
-    # ### end Alembic commands ###

dipex-5.7.0/integrations/SD_Lon/payload_db/__init__.py

@@ -1,40 +0,0 @@
-# SPDX-FileCopyrightText: Magenta ApS
-# SPDX-License-Identifier: MPL-2.0
-from uuid import UUID
-
-from sqlalchemy.orm import sessionmaker
-
-from .engine import get_engine
-from .models import Payload
-
-
-Session = sessionmaker()
-
-
-def log_payload(
-    request_uuid: UUID,
-    full_url: str,
-    params: str,
-    response: str,
-    status_code: int,
-) -> None:
-    """Log a given SD payload to the payload database
-
-    Args:
-        request_uuid: Unique UUID of this API request. Must be generated by the caller.
-        full_url: Full URL of API endpoint called, e.g. containing the SD server name.
-        params: Stringified version of the `payload` variable in `sd_lookup`.
-        response: The raw/unparsed XML response from the SD API endpoint called.
-        status_code: The HTTP status code from the SD API endpoint called.
-    """
-    Session.configure(bind=get_engine())
-    session = Session()
-    payload = Payload(
-        id=request_uuid,
-        full_url=full_url,
-        params=params,
-        response=response,
-        status_code=status_code,
-    )
-    session.add(payload)
-    session.commit()

dipex-5.7.0/integrations/SD_Lon/payload_db/engine.py

@@ -1,14 +0,0 @@
-# SPDX-FileCopyrightText: Magenta ApS
-# SPDX-License-Identifier: MPL-2.0
-import os
-
-from sqlalchemy import create_engine
-from sqlalchemy.engine import Engine
-
-
-def get_db_url() -> str:
-    return f"postgresql+psycopg2://{os.environ['APP_DBUSER']}:{os.environ['APP_DBPASSWORD']}@{os.environ['PGHOST']}/{os.environ['APP_DATABASE']}"
-
-
-def get_engine() -> Engine:
-    return create_engine(get_db_url())

dipex-5.7.0/integrations/SD_Lon/payload_db/models.py

@@ -1,23 +0,0 @@
-# SPDX-FileCopyrightText: Magenta ApS
-# SPDX-License-Identifier: MPL-2.0
-from sqlalchemy import Column
-from sqlalchemy import DateTime
-from sqlalchemy import Integer
-from sqlalchemy.dialects.postgresql import TEXT
-from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.orm import declarative_base
-from sqlalchemy.sql import func
-
-
-Base = declarative_base()
-
-
-class Payload(Base):  # type: ignore
-    __tablename__ = "payload"
-
-    id = Column(UUID(as_uuid=True), primary_key=True)
-    timestamp = Column(DateTime(timezone=True), server_default=func.now())
-    full_url = Column(TEXT)
-    params = Column(TEXT)
-    response = Column(TEXT)
-    status_code = Column(Integer)

dipex-5.7.0/integrations/SD_Lon/sdlon/config.py

@@ -1,201 +0,0 @@
-#!/usr/bin/env python3
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-from datetime import date
-from functools import lru_cache
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-from typing import Type
-
-from pydantic import AnyHttpUrl
-from pydantic import BaseSettings
-from pydantic import conint
-from pydantic import constr
-from pydantic import Extra
-from pydantic import Field
-from pydantic import PositiveInt
-from pydantic import SecretStr
-from pydantic import UUID4
-from ra_utils.load_settings import load_settings
-from ra_utils.job_settings import JobSettings
-
-from .log import LogLevel
-from .models import JobFunction
-
-
-class CommonSettings(BaseSettings):  # type: ignore
-    """
-    Settings common to both the SD importer and SD-changed-at
-    """
-
-    mora_base: AnyHttpUrl = Field("http://mo-service:5000")
-    mox_base: AnyHttpUrl = Field("http://mo-service:5000/lora")
-
-    sd_employment_field: Optional[str] = Field(default=None, regex="extension_[0-9]+")
-    sd_global_from_date: date
-    sd_import_run_db: str
-    sd_import_too_deep: List[str] = []
-    sd_institution_identifier: str
-    sd_password: SecretStr
-    sd_user: str
-    sd_job_function: JobFunction
-    sd_monthly_hourly_divide: PositiveInt
-
-    # List of SD JobPositionIdentifiers which should not result in creation
-    # of engagements
-    sd_skip_employment_types: List[str] = []
-
-    # If true, the <TelephoneNumberIdentifier> in <ContactInformation> in
-    # <Employment> in the <Person> tag will be used to set a value that can
-    # trigger AD-creation of the user
-    # (see https://redmine.magenta-aps.dk/issues/56089)
-    sd_phone_number_id_for_ad_creation: bool = False
-    # SD can only set <TelephoneNumberIdentifier> to an integer value, hence
-    # the value 14 is used (letters A=1 and D=4 for "AD")
-    sd_phone_number_id_trigger: str = "14"
-    sd_phone_number_id_for_ad_string: str = "AD-bruger fra SD"
-
-    cpr_uuid_map_path: str = (
-        "/opt/dipex/os2mo-data-import-and-export/settings/cpr_uuid_map.csv"
-    )
-
-    log_level: LogLevel = LogLevel.DEBUG
-
-    job_settings: JobSettings = JobSettings()
-
-
-def gen_json_file_settings_func(settings_class: Type[CommonSettings]):
-    def json_file_settings(settings: BaseSettings) -> Dict[str, Any]:  # type: ignore
-        try:
-            json_settings = load_settings()
-        except FileNotFoundError:
-            return dict()
-
-        # Remove the "integrations.SD_Lon." part of the key name
-        json_settings = {
-            key.replace("integrations.SD_Lon.", "sd_"): value
-            for key, value in json_settings.items()
-        }
-
-        # Remove any double "sd_sd_" in the keys
-        json_settings = {
-            key.replace("sd_sd_", "sd_"): value for key, value in json_settings.items()
-        }
-
-        # Replace dots with underscores to be Pydantic compliant
-        json_settings = {
-            key.replace(".", "_"): value for key, value in json_settings.items()
-        }
-
-        # Remove settings forbidden according to the Settings model
-        properties = settings_class.schema()["properties"].keys()
-        json_settings = {
-            key: value for key, value in json_settings.items() if key in properties
-        }
-
-        return json_settings
-
-    return json_file_settings
-
-
-class SDCommonSettings(CommonSettings):
-    class Config:
-        extra = Extra.forbid
-
-        @classmethod
-        def customise_sources(
-            cls,
-            init_settings,
-            env_settings,
-            file_secret_settings,
-        ):
-            return (
-                init_settings,
-                env_settings,
-                gen_json_file_settings_func(SDCommonSettings),
-                file_secret_settings,
-            )
-
-
-class ImporterSettings(CommonSettings):
-    municipality_code: conint(ge=100, le=999)  # type: ignore
-    municipality_name: str
-    sd_importer_create_associations: bool = True
-    sd_importer_create_email_addresses: bool = True
-
-    # Deprecated
-    sd_importer_employment_date_as_engagement_start_date: bool = False
-
-    sd_no_salary_minimum_id: Optional[int] = None
-    sd_use_ad_integration: bool = True
-
-    class Config:
-        extra = Extra.forbid
-
-        @classmethod
-        def customise_sources(
-            cls,
-            init_settings,
-            env_settings,
-            file_secret_settings,
-        ):
-            return (
-                init_settings,
-                env_settings,
-                gen_json_file_settings_func(ImporterSettings),
-                file_secret_settings,
-            )
-
-
-class ChangedAtSettings(CommonSettings):
-    sd_fix_departments_root: Optional[UUID4] = None
-    sd_no_salary_minimum_id: Optional[int] = None
-    sd_overwrite_existing_employment_name = True
-
-    # List of CRPs to either include OR exclude in the run
-    sd_cprs: List[constr(regex="^[0-9]{10}$")] = []  # type: ignore # noqa
-
-    # If true, the sd_cprs will be excluded and if false, only the
-    # CPRs in sd_cprs will be included in the run
-    sd_exclude_cprs_mode: bool = True
-
-    sd_read_forced_uuids: bool = True
-    sd_update_primary_engagement: bool = True
-    sd_use_ad_integration: bool = True
-
-    class Config:
-        extra = Extra.forbid
-
-        @classmethod
-        def customise_sources(
-            cls,
-            init_settings,
-            env_settings,
-            file_secret_settings,
-        ):
-            return (
-                init_settings,
-                env_settings,
-                gen_json_file_settings_func(ChangedAtSettings),
-                file_secret_settings,
-            )
-
-
-@lru_cache()
-def get_common_settings(*args, **kwargs) -> SDCommonSettings:
-    return SDCommonSettings(*args, **kwargs)
-
-
-@lru_cache()
-def get_importer_settings(*args, **kwargs) -> ImporterSettings:
-    return ImporterSettings(*args, **kwargs)
-
-
-@lru_cache()
-def get_changed_at_settings(*args, **kwargs) -> ChangedAtSettings:
-    return ChangedAtSettings(*args, **kwargs)

dipex-5.7.0/integrations/SD_Lon/sdlon/date_utils.py

@@ -1,181 +0,0 @@
-import logging
-import re
-from datetime import date
-from datetime import datetime
-from datetime import timedelta
-from itertools import chain
-from itertools import takewhile
-from typing import OrderedDict
-from typing import Union
-from typing import Iterator
-from typing import Optional
-from typing import Tuple
-
-from more_itertools import pairwise
-from more_itertools import tabulate
-
-from .log import get_logger
-from .sd_common import EmploymentStatus
-
-# TODO: move constants elsewhere
-# TODO: set back to "infinity" when MO can handle this
-# MO_INFINITY: str = "infinity"
-
-MO_INFINITY = None
-SD_INFINITY: str = "9999-12-31"
-DATE_REGEX_STR = "[0-9]{4}-(0[1-9]|1[0-2])-([0-2][0-9]|3[0-1])"
-
-logger = get_logger()
-
-
-def date_to_datetime(d: date) -> datetime:
-    return datetime(d.year, d.month, d.day)
-
-
-def format_date(d: datetime | date) -> str:
-    return d.strftime("%Y-%m-%d").zfill(10)
-
-
-def parse_datetime(date_str: str) -> datetime:
-    return datetime.strptime(date_str, "%Y-%m-%d")
-
-
-def datetime_to_sd_date(date_time: Union[datetime, date]) -> str:
-    return date_time.strftime("%d.%m.%Y")
-
-
-def _get_employment_from_date(employment: OrderedDict) -> datetime:
-    """
-    Get the latest date of all the dates in the payload from the SD
-    GetEmployment endpoint (see https://redmine.magenta-aps.dk/issues/51898)
-
-    Args:
-        employment: the SD employment
-
-    Returns: the maximum of all the date in the SD payload
-
-    """
-    MIN_DATE = format_date(datetime.min)
-
-    employment_date = employment.get("EmploymentDate", MIN_DATE)
-    employment_department_date = employment.get("EmploymentDepartment", dict()).get(
-        "ActivationDate", MIN_DATE
-    )
-    employment_status_date = employment.get("EmploymentStatus", dict()).get(
-        "ActivationDate", MIN_DATE
-    )
-    profession_date = employment.get("Profession", dict()).get(
-        "ActivationDate", MIN_DATE
-    )
-    working_time_date = employment.get("WorkingTime", dict()).get(
-        "ActivationDate", MIN_DATE
-    )
-
-    return max(
-        parse_datetime(employment_date),
-        parse_datetime(employment_department_date),
-        parse_datetime(employment_status_date),
-        parse_datetime(profession_date),
-        parse_datetime(working_time_date),
-    )
-
-
-def get_employment_datetimes(employment: OrderedDict) -> Tuple[datetime, datetime]:
-    """
-    Get the "from" and "to" date from the SD employment
-
-    Args:
-        employment: The SD employment
-
-    Returns:
-        Tuple containing the "from" and "to" dates.
-    """
-
-    status = EmploymentStatus(employment["EmploymentStatus"]["EmploymentStatusCode"])
-
-    if status in EmploymentStatus.let_go():
-        datetime_from = parse_datetime(employment["EmploymentDate"])
-        termination_date = str(
-            sd_to_mo_termination_date(employment["EmploymentStatus"]["ActivationDate"])
-        )
-        datetime_to = parse_datetime(termination_date)
-        return datetime_from, datetime_to
-
-    datetime_from = _get_employment_from_date(employment)
-    datetime_to = parse_datetime(employment["EmploymentStatus"]["DeactivationDate"])
-
-    return datetime_from, datetime_to
-
-
-# TODO: Create "MoValidity" and "SdValidity" classes based on the RA Models
-#  "Validity" class and use these as input to the function below
-
-
-def sd_to_mo_termination_date(sd_date: str) -> Optional[str]:
-    """
-    Convert SD termination date to MO termination date.
-
-    Args:
-        sd_date: SD termination date formatted as "YYYY-MM-DD"
-
-    Returns:
-        MO termination date formatted as "YYYY-MM-DD"
-    """
-
-    assert isinstance(sd_date, str)
-    assert re.compile(DATE_REGEX_STR).match(sd_date)
-
-    if sd_date == SD_INFINITY:
-        return MO_INFINITY
-
-    return sd_date
-
-
-def to_midnight(dt: datetime) -> datetime:
-    """Get previous midnight from datetime."""
-    return dt.replace(hour=0, minute=0, second=0, microsecond=0)
-
-
-def is_midnight(dt: datetime) -> bool:
-    """Check if datetime is at midnight."""
-    return dt == to_midnight(dt)
-
-
-def gen_cut_dates(from_datetime: datetime, to_datetime: datetime) -> Iterator[datetime]:
-    """Get iterator of cut-dates between from_datetime and to_datetime.
-
-    Args:
-        from_datetime: the start date
-        to_datetime: the end date
-
-    Yields:
-        The from_datetime, then all intermediate midnight datetimes and the to_datetime.
-    """
-    assert from_datetime < to_datetime
-
-    # Tabulate to infinite iterator of midnights starting after from_datetime
-    def midnight_at_offset(offset: int) -> datetime:
-        return to_midnight(from_datetime) + timedelta(days=offset)
-
-    midnights = takewhile(
-        lambda midnight: midnight < to_datetime, tabulate(midnight_at_offset, start=1)
-    )
-    return chain([from_datetime], midnights, [to_datetime])
-
-
-def gen_date_intervals(
-    from_datetime: datetime, to_datetime: datetime
-) -> Iterator[Tuple[datetime, datetime]]:
-    """
-    Get iterator capable of generating a sequence of datetime pairs
-    incrementing one day at a time. The latter date in a pair is
-    advanced by exactly one day compared to the former date in the pair.
-
-    Args:
-        from_datetime: the start date
-        to_datetime: the end date
-
-    Yields:
-        The next date pair in the sequence of pairs
-    """
-    return pairwise(gen_cut_dates(from_datetime, to_datetime))

dipex-5.7.0/integrations/SD_Lon/sdlon/employees.py

@@ -1,51 +0,0 @@
-from uuid import UUID
-
-from gql import gql
-from more_itertools import one
-from raclients.graph.client import GraphQLClient
-from sdlon.models import MOBasePerson
-
-QUERY_GET_EMPLOYEE = gql(
-    """
-        query GetEmployee($cpr: [CPR!]!) {
-            employees(cpr_numbers: $cpr) {
-                objects {
-                    current {
-                        name
-                        givenname
-                        surname
-                        uuid
-                    }
-                }
-            }
-        }
-    """
-)
-
-
-def get_employee(gql_client: GraphQLClient, cpr: str) -> MOBasePerson | None:
-    """
-    Get employee from MO
-
-    Args:
-        gql_client: The GraphQl client for calling MO
-        cpr: The CPR number of the employee
-
-    Returns:
-        The MO employee or None if the employee does not exist in MO
-    """
-    r = gql_client.execute(QUERY_GET_EMPLOYEE, variable_values={"cpr": cpr})
-
-    obj = r["employees"]["objects"]
-    if not obj:
-        return None
-
-    employee = one(obj)["current"]
-
-    return MOBasePerson(
-        cpr=cpr,
-        givenname=employee["givenname"],
-        surname=employee["surname"],
-        name=employee["name"],
-        uuid=UUID(employee["uuid"]),
-    )