ms-salesforce-api 3.3.0__tar.gz → 3.5.0__tar.gz

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ms-salesforce-api
-Version: 3.3.0
+Version: 3.5.0
 Summary: Python library used to extract data from Salesforce API and migrate it to Bigquery and Postgres.
 License-File: LICENSE
 Author: Making Science
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: flake8 (>=6.0.0,<7.0.0)
-Requires-Dist: gc-google-services-api (>=1.2.7,<2.0.0)
+Requires-Dist: gc-google-services-api (==2.4.0)
 Requires-Dist: isort (>=5.12.0,<6.0.0)
 Requires-Dist: pre-commit (>=3.1.1,<4.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.6,<3.0.0)

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/account/constants.py

@@ -7,8 +7,8 @@ SELECT
     BillingPostalCode,
     BillingStateCode,
     BillingStreet,
-    BQ_CustomerGroupId__c,
-    BQ_CustomerSubgroupId__c,
+    LKP_CustomerSubgroup__r.LKP_CustomerGroup__c,
+    LKP_CustomerSubgroup__c,
     Business_Name__c,
     CIF__c,
     CreatedDate,

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/account/dto/AccountDTO.py

@@ -262,6 +262,14 @@ class AccountDTO(object):
             except Exception:
                 return ""
 
+        def _get_customer_group_id():
+            try:
+                return normalize_value(
+                    record["LKP_CustomerSubgroup__r"]["LKP_CustomerGroup__c"]
+                )
+            except (TypeError, KeyError):
+                return None
+
         return AccountDTO(
             id=record["Id"],
             accelerator=record["Accelerator__c"],
@@ -304,9 +312,9 @@ class AccountDTO(object):
             billing_state_code=normalize_value(record["BillingStateCode"]),
             mail_invoincing=normalize_value(record["MAIL_Invoicing__c"]),
             invoincing_email=normalize_value(record["Invoicing_Email__c"]),
-            customer_group_id=normalize_value(record["BQ_CustomerGroupId__c"]),
+            customer_group_id=_get_customer_group_id(),
             customer_subgroup_id=normalize_value(
-                record["BQ_CustomerSubgroupID__c"]
+                record["LKP_CustomerSubgroup__c"]
             ),
             customer_subgroup_name=normalize_value(_get_subgroup_name()),
             customer_subgroup_dt_start=normalize_value(_get_subgroup_start()),

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/account/export_data/Bigquery.py

@@ -55,8 +55,8 @@ class BigQueryExporter:
                 "billing_state_code": "STRING",
                 "mail_invoincing": "STRING",
                 "invoincing_email": "STRING",
-                "customer_group_id": "INTEGER",
-                "customer_subgroup_id": "INTEGER",
+                "customer_group_id": "STRING",
+                "customer_subgroup_id": "STRING",
                 "customer_subgroup_name": "STRING",
                 "customer_subgroup_dt_start": "DATE",
                 "customer_subgroup_dt_end": "DATE",

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/opportunity/constants.py

@@ -83,6 +83,7 @@ SELECT
     TXT_Comments__c,
     Year_Created__c,
     LKP_ParentOpportunity__c,
+    TECH_Old_NewBusiness__c,
     (
         SELECT
             Description,

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/opportunity/dto/OpportunityDTO.py

@@ -213,6 +213,7 @@ class OpportunityDTO(object):
         billing_info,
         parent_opportunity,
         project_name,
+        tech_old_new_business,
     ):
         self.opportunity_id = opportunity_id
         self.account_billing_country = account_billing_country
@@ -296,6 +297,7 @@ class OpportunityDTO(object):
         self.billing_info = billing_info
         self.parent_opportunity = parent_opportunity
         self.project_name = project_name
+        self.tech_old_new_business = tech_old_new_business
 
     @staticmethod
     def from_salesforce_record(record):
@@ -530,6 +532,7 @@ class OpportunityDTO(object):
             billing_info=record["TXT_BillingInfo__c"],
             parent_opportunity=record["LKP_ParentOpportunity__c"],
             project_name=record["FRM_ProjectName__c"],
+            tech_old_new_business=record["TECH_Old_NewBusiness__c"],
         )
 
     def to_dict(self):
@@ -616,4 +619,5 @@ class OpportunityDTO(object):
             "billing_info": self.billing_info,
             "parent_opportunity": self.parent_opportunity,
             "project_name": self.project_name,
+            "tech_old_new_business": self.tech_old_new_business,
         }

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/opportunity/export_data/Bigquery.py

@@ -106,6 +106,7 @@ class BigQueryExporter:
                 "billing_info": "STRING",
                 "parent_opportunity": "STRING",
                 "project_name": "STRING",
+                "tech_old_new_business": "Boolean",
             },
             "opportunity_line_item": {
                 "opportunity_id": "STRING",

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/project/__tests__/test_Project.py

@@ -397,8 +397,8 @@ class TestProject(unittest.TestCase):
             "account_company_invoicing": "",
             "account_created_date": "2020-03-18 15:32:15",
             "account_currency_code": "EUR",
-            "account_customer_groupId": "1",
-            "account_customer_subgroupId": "5",
+            "account_customer_groupId": "",
+            "account_customer_subgroupId": "",
             "account_fax": None,
             "account_invoicing_email": "client1@test.com",
             "account_mail_invoicing": "client1@test.com",
@@ -452,7 +452,6 @@ class TestProject(unittest.TestCase):
             "group_pck_type": "Key Account",
             "group_start_date": "2023-06-01",
             "group_supervisor_email": "employee2@test.com",
-            "international_deparments": False,
             "invoicing_country_code": "US",
             "jira_task_url": "<a "
             "href=https://makingscience.atlassian.net/browse/ESMSBD0001-7168 "
@@ -523,7 +522,6 @@ class TestProject(unittest.TestCase):
             "subgroup_subgroupid": "a19AX0000004simYAA",
         }
         self.assertIsInstance(opportunity, dict)
-
         self.assertDictEqual(
             opportunity,
             opportunity_result,

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/project/constants.py

@@ -6,7 +6,6 @@ SELECT
     CurrencyIsoCode,
     End_Date__c,
     Id,
-    International_Departments__c,
     LastModifiedDate,
     LKP_CostCenter__r.Name,
     LKP_MSCompanyInvoicing__r.PCK_Prefix__c,
@@ -38,8 +37,8 @@ SELECT
     Project_Account__r.BillingPostalCode,
     Project_Account__r.BillingStateCode,
     Project_Account__r.BillingStreet,
-    Project_Account__r.BQ_CustomerGroupId__c,
-    Project_Account__r.BQ_CustomerSubgroupId__c,
+    Project_Account__r.LKP_CustomerSubgroup__r.LKP_CustomerGroup__c,
+    Project_Account__r.LKP_CustomerSubgroup__c,
     Project_Account__r.Business_Name__c,
     Project_Account__r.CIF__c,
     Project_Account__r.CreatedDate,

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/project/dto/OpportunityDTO.py

@@ -77,7 +77,6 @@ class OpportunityDTO(object):
         autorenewal,
         comments,
         end_date,
-        international_deparments,
         opportunity_extension,
         maintance_project,
         ms_project_id,
@@ -167,7 +166,6 @@ class OpportunityDTO(object):
         self.autorenewal = autorenewal
         self.comments = comments
         self.end_date = end_date
-        self.international_deparments = international_deparments
         self.opportunity_extension = opportunity_extension
         self.maintance_project = maintance_project
         self.ms_project_id = ms_project_id
@@ -442,7 +440,9 @@ class OpportunityDTO(object):
 
         def _get_account_customer_groupid():
             try:
-                data = record["Project_Account__r"]["BQ_CustomerGroupId__c"]
+                data = record["Project_Account__r"]["LKP_CustomerSubgroup__r"][
+                    "LKP_CustomerGroup__c"
+                ]
                 if data:
                     return normalize_value(data)
             except (TypeError, KeyError):
@@ -450,7 +450,7 @@ class OpportunityDTO(object):
 
         def _get_account_customer_subgroupid():
             try:
-                data = record["Project_Account__r"]["BQ_CustomerSubgroupID__c"]
+                data = record["Project_Account__r"]["LKP_CustomerSubgroup__c"]
                 if data:
                     return normalize_value(data)
             except (TypeError, KeyError):
@@ -682,12 +682,6 @@ class OpportunityDTO(object):
             except KeyError:
                 return ""
 
-        def _get_international_deparments():
-            try:
-                return record["International_Departments__c"]
-            except KeyError:
-                return False
-
         def _get_opportunity_extension():
             try:
                 return record["LKP_OpportunityExtension__c"]
@@ -865,7 +859,6 @@ class OpportunityDTO(object):
             autorenewal=_get_autorenewal(),
             comments=_get_comments(),
             end_date=_get_end_date(),
-            international_deparments=_get_international_deparments(),
             opportunity_extension=_get_opportunity_extension(),
             maintance_project=_get_maintance_project(),
             ms_project_id=_get_ms_project_id(),
@@ -960,7 +953,6 @@ class OpportunityDTO(object):
             "autorenewal": self.autorenewal,
             "comments": self.comments,
             "end_date": self.end_date,
-            "international_deparments": self.international_deparments,
             "opportunity_extension": self.opportunity_extension,
             "maintance_project": self.maintance_project,
             "ms_project_id": self.ms_project_id,

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/ms_salesforce_api/salesforce/api/project/export_data/Bigquery.py

@@ -48,7 +48,6 @@ class BigQueryExporter:
                 "autorenewal": "BOOLEAN",
                 "comments": "STRING",
                 "end_date": "DATE",
-                "international_deparments": "BOOLEAN",
                 "opportunity_extension": "STRING",
                 "maintance_project": "BOOLEAN",
                 "ms_project_id": "STRING",
@@ -136,8 +135,8 @@ class BigQueryExporter:
                 "tax_id_type": "STRING",
                 "tier": "STRING",
                 "website": "STRING",
-                "customer_groupId": "INTEGER",
-                "customer_subgroupId": "INTEGER",
+                "customer_groupId": "STRING",
+                "customer_subgroupId": "STRING",
             },
             "groups": {
                 "project_id": "STRING",

ms_salesforce_api-3.5.0/ms_salesforce_api/salesforce/api/projects_with_backup/__init__.py

@@ -0,0 +1,55 @@
+import logging
+
+from ms_salesforce_api.salesforce.api.projects_with_backup.constants import (
+    DEFAULT_PROJECTS_QUERY,
+)
+from ms_salesforce_api.salesforce.api.projects_with_backup.dto.ProjectDTO import (  # noqa: E501
+    ProjectDTO,
+)
+from ms_salesforce_api.salesforce.SalesforceQueryExecutor import (
+    SalesforceQueryExecutor,
+)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
+
+MAX_PROJECT_IDS_PER_QUERY = 200
+
+
+class Project(SalesforceQueryExecutor):
+    def get_batches(self, lst, n):
+        """Yield successive n-sized chunks from lst."""
+        for i in range(0, len(lst), n):
+            yield lst[i : i + n]  # noqa: E203
+
+    def get_all(
+        self,
+        query: str = DEFAULT_PROJECTS_QUERY,
+        format: str = "json",
+    ):
+        data = self.fetch_data(query)
+        if data is None:
+            logging.error(
+                "[ERROR - SalesforceAPI]: No projects data return from Salesforce API"  # noqa: E501
+            )
+            return []
+
+        try:
+            projects = [
+                ProjectDTO.from_salesforce_record(record) for record in data
+            ]
+            projects_list = list(projects)
+
+            if format == "json":
+                projects_list = [
+                    project.to_dict() for project in projects_list
+                ]
+
+            return projects_list
+        except Exception as e:
+            logging.error(
+                f"[ERROR - get_all]: Failed to get all projects: {e}"
+            )
+            return []

ms_salesforce_api-3.5.0/ms_salesforce_api/salesforce/api/projects_with_backup/constants.py

@@ -0,0 +1,16 @@
+DEFAULT_PROJECTS_QUERY = """
+SELECT
+    Id,
+    Account.LKP_CustomerSubgroup__r.LKP_CustomerGroup__r.Name,
+    Account.Name,
+    FRM_ProjectName__c,
+    FRM_ProjectCode__c,
+    Tolabel(PCK_ProjectStatus__c),
+    DT_ProjectStartDate__c,
+    DT_ProjectEndDate__c,
+    LKP_AccountManager__r.Name,
+    Account_Billing_Country__c,
+    PCK_Division__c
+FROM
+    Opportunity
+"""

ms_salesforce_api-3.5.0/ms_salesforce_api/salesforce/api/projects_with_backup/dto/ProjectDTO.py

@@ -0,0 +1,86 @@
+from ms_salesforce_api.salesforce.helpers.string import normalize_value
+
+
+class ProjectDTO(object):
+    def __init__(
+        self,
+        id,
+        account_customer_group_name,
+        account_name,
+        project_name,
+        project_code,
+        project_status,
+        project_startdate,
+        project_enddate,
+        account_manager_name,
+        account_billing_country,
+        division,
+    ):
+        self.id = id
+        self.account_customer_group_name = account_customer_group_name
+        self.account_name = account_name
+        self.project_name = project_name
+        self.project_code = project_code
+        self.project_status = project_status
+        self.project_startdate = project_startdate
+        self.project_enddate = project_enddate
+        self.account_manager_name = account_manager_name
+        self.account_billing_country = account_billing_country
+        self.division = division
+
+    @staticmethod
+    def from_salesforce_record(record):
+        def _get_account_customer_group_name():
+            try:
+                return normalize_value(
+                    record["Account"]["LKP_CustomerSubgroup__r"][
+                        "LKP_CustomerGroup__r"
+                    ][
+                        "Name"
+                    ]  # noqa: E501
+                )
+            except (KeyError, TypeError):
+                return None
+
+        def _get_account_name():
+            try:
+                return normalize_value(record["Account"]["Name"])  # noqa: E501
+            except (KeyError, TypeError):
+                return None
+
+        def _get_account_manager_name():
+            try:
+                return normalize_value(
+                    record["LKP_AccountManager__r"]["Name"]  # noqa: E501
+                )
+            except (KeyError, TypeError):
+                return None
+
+        return ProjectDTO(
+            id=record["Id"],
+            account_customer_group_name=_get_account_customer_group_name(),
+            account_name=_get_account_name(),
+            project_name=record["FRM_ProjectName__c"],
+            project_code=record["FRM_ProjectCode__c"],
+            project_status=record["PCK_ProjectStatus__c"],
+            project_startdate=record["DT_ProjectStartDate__c"],
+            project_enddate=record["DT_ProjectEndDate__c"],
+            account_manager_name=_get_account_manager_name(),
+            account_billing_country=record["Account_Billing_Country__c"],
+            division=record["PCK_Division__c"],
+        )
+
+    def to_dict(self):
+        return {
+            "id": self.id,
+            "account_customer_group_name": self.account_customer_group_name,
+            "account_name": self.account_name,
+            "project_name": self.project_name,
+            "project_code": self.project_code,
+            "project_status": self.project_status,
+            "project_startdate": self.project_startdate,
+            "project_enddate": self.project_enddate,
+            "account_manager_name": self.account_manager_name,
+            "account_billing_country": self.account_billing_country,
+            "division": self.division,
+        }

ms_salesforce_api-3.5.0/ms_salesforce_api/salesforce/api/projects_with_backup/export_data/Bigquery.py

@@ -0,0 +1,75 @@
+import logging
+
+from gc_google_services_api.bigquery import BigQueryManager
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+
+class BigQueryExporter:
+    """
+    Initializes the Bigquery exporter with the given project ID and dataset ID.
+
+    Args:
+        project_id (str): The ID of the Google Cloud project.
+        dataset_id (str): The ID of the BigQuery dataset.
+    """
+
+    def __init__(self, project_id, dataset_id):
+        self.project_id = project_id
+        self.dataset_id = dataset_id
+        self.client = BigQueryManager(
+            project_id=project_id,
+            dataset_id=dataset_id,
+        )
+        self.batch_size = 200
+        self.schemas = {
+            "projects_with_backup": {
+                "id": "STRING",
+                "account_customer_group_name": "STRING",
+                "account_name": "STRING",
+                "project_name": "STRING",
+                "project_code": "STRING",
+                "project_status": "STRING",
+                "project_startdate": "STRING",
+                "project_enddate": "STRING",
+                "account_manager_name": "STRING",
+                "account_billing_country": "STRING",
+                "division": "STRING",
+            },
+        }
+
+        for table_name, table_schema in self.schemas.items():
+            self.client.create_table_if_not_exists(table_name, table_schema)
+
+    def _execute_query(self, query, log_id, default_error_value=None):
+        custom_error_value = f"{log_id}_custom_error"
+
+        result = self.client.execute_query(
+            query,
+            custom_error_value,
+        )
+
+        if result == custom_error_value:
+            logging.error(
+                f"[ERROR - _execute_query]: Error executing query for {log_id} in BigQuery."  # noqa: E501
+            )
+            result = default_error_value
+
+        return result
+
+    def export_data(self, profit_centers):
+        self.client.load_massive_data(
+            rows_to_insert=profit_centers,
+            table_name="projects_with_backup",
+        )
+
+    def delete_all_rows(self):
+        table_names = self.schemas.keys()
+        for table_name in table_names:
+            delete_query_table = f"DELETE FROM `{self.project_id}.{self.dataset_id}.{table_name}` WHERE true"  # noqa: E501
+            self._execute_query(
+                query=delete_query_table,
+                log_id=f"delete_table_{table_name}",
+            )

{ms_salesforce_api-3.3.0 → ms_salesforce_api-3.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ms-salesforce-api"
-version = "3.3.0"
+version = "3.5.0"
 description = "Python library used to extract data from Salesforce API and migrate it to Bigquery and Postgres."
 authors = ["Making Science"]
 readme = "README.md"
@@ -15,7 +15,7 @@ pre-commit = "^3.1.1"
 pyjwt = {extras = ["crypto"], version = "^2.6.0"}
 requests = "^2.28.2"
 psycopg2-binary = "^2.9.6"
-gc-google-services-api = "^1.2.7"
+gc-google-services-api = "2.4.0"
 
 [build-system]
 requires = ["poetry-core"]