ipulse-shared-core-ftredge 2.57__tar.gz → 3.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ipulse-shared-core-ftredge might be problematic. Click here for more details.
- {ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-3.1.1}/PKG-INFO +2 -4
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/setup.py +2 -4
- ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge/__init__.py +11 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge/models/__init__.py +0 -1
- ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge/models/organisation.py +71 -0
- ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge/models/resource_catalog_item.py +115 -0
- ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge/utils/__init__.py +3 -0
- ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge/utils/utils_common.py +10 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO +2 -4
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt +2 -12
- ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge.egg-info/requires.txt +4 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/tests/test_utils_gcp.py +1 -1
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/__init__.py +0 -20
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/enums/__init__.py +0 -29
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/enums/enums_common_utils.py +0 -177
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/enums/enums_data_eng.py +0 -44
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/enums/enums_module_fincore.py +0 -58
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/enums/enums_modules.py +0 -33
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/models/audit_log_firestore.py +0 -12
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/models/organisation.py +0 -65
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/models/pulse_enums.py +0 -196
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/models/resource_catalog_item.py +0 -189
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/utils_custom_logs.py +0 -201
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/utils_gcp.py +0 -314
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/utils_gcp_for_pipelines.py +0 -201
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/utils_pipelinemon.py +0 -362
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py +0 -153
- ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge.egg-info/requires.txt +0 -6
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/LICENCE +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/README.md +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/pyproject.toml +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/setup.cfg +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge/models/user_auth.py +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge/models/user_profile.py +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge/models/user_profile_update.py +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge/models/user_status.py +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge.egg-info/dependency_links.txt +0 -0
- {ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/src/ipulse_shared_core_ftredge.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: ipulse_shared_core_ftredge
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.1.1
|
|
4
4
|
Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
|
|
5
5
|
Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
|
|
6
6
|
Author: Russlan Ramdowar
|
|
@@ -8,6 +8,4 @@ License-File: LICENCE
|
|
|
8
8
|
Requires-Dist: pydantic[email]~=2.5
|
|
9
9
|
Requires-Dist: python-dateutil~=2.8
|
|
10
10
|
Requires-Dist: pytest~=7.1
|
|
11
|
-
Requires-Dist:
|
|
12
|
-
Requires-Dist: google-cloud-error-reporting~=1.11.0
|
|
13
|
-
Requires-Dist: google-cloud-bigquery~=3.24.0
|
|
11
|
+
Requires-Dist: ipulse_shared_base_ftredge>=2.2.2
|
|
@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
|
|
|
3
3
|
|
|
4
4
|
setup(
|
|
5
5
|
name='ipulse_shared_core_ftredge',
|
|
6
|
-
version='
|
|
6
|
+
version='3.1.1',
|
|
7
7
|
package_dir={'': 'src'}, # Specify the source directory
|
|
8
8
|
packages=find_packages(where='src'), # Look for packages in 'src'
|
|
9
9
|
install_requires=[
|
|
@@ -11,9 +11,7 @@ setup(
|
|
|
11
11
|
'pydantic[email]~=2.5',
|
|
12
12
|
'python-dateutil~=2.8',
|
|
13
13
|
'pytest~=7.1',
|
|
14
|
-
'
|
|
15
|
-
'google-cloud-error-reporting~=1.11.0',
|
|
16
|
-
'google-cloud-bigquery~=3.24.0'
|
|
14
|
+
'ipulse_shared_base_ftredge>=2.2.2',
|
|
17
15
|
],
|
|
18
16
|
author='Russlan Ramdowar',
|
|
19
17
|
description='Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.',
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# pylint: disable=missing-module-docstring
|
|
2
|
+
# pylint: disable=missing-function-docstring
|
|
3
|
+
# pylint: disable=missing-class-docstring
|
|
4
|
+
# pylint: disable=broad-exception-caught
|
|
5
|
+
# pylint: disable=line-too-long
|
|
6
|
+
# pylint: disable=unused-variable
|
|
7
|
+
# pylint: disable=broad-exception-caught
|
|
8
|
+
# from pydantic import BaseModel, validator, ValidationError, Field
|
|
9
|
+
# from typing import Set, Optional
|
|
10
|
+
# import uuid
|
|
11
|
+
# from datetime import datetime
|
|
12
|
+
# import dateutil.parser
|
|
13
|
+
|
|
14
|
+
# CLASS_VERSION= 1.0
|
|
15
|
+
# MODULE= "core"
|
|
16
|
+
# CLASS_REF="orgn"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# class Organisation(BaseModel):
|
|
20
|
+
# puid: str = Field(default_factory=f"{datetime.utcnow().strftime('%Y%m%d%H%M')}{uuid.uuid4().hex[:8]}_{MODULE}{CLASS_REF}".lower())
|
|
21
|
+
# name: str
|
|
22
|
+
# creat_date: datetime = Field(default_factory=datetime.utcnow())
|
|
23
|
+
# updt_date: datetime = Field(default_factory=datetime.utcnow())
|
|
24
|
+
# creat_by_user: Optional[str] = None
|
|
25
|
+
# updt_by_user: Optional[str] = None
|
|
26
|
+
# relations: Optional[Set[str]]=None
|
|
27
|
+
# description: Optional[str] = None # Updated to use Optional
|
|
28
|
+
# industries: Optional[Set[str]] = None # Updated to use Optional
|
|
29
|
+
# website: Optional[str] = None # Updated to use Optional
|
|
30
|
+
# org_admin_user_uids: Optional[Set[str]] = None # Updated to use Optional
|
|
31
|
+
# class Config:
|
|
32
|
+
# extra = "forbid"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# @validator('relations', pre=True, always=True)
|
|
36
|
+
# def validate_relations(cls, relations):
|
|
37
|
+
# if not set(relations).issubset(enums.organisation_relations):
|
|
38
|
+
# raise ValueError("Invalid relation values provided.")
|
|
39
|
+
# return relations
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# @validator('industries', pre=True, always=True)
|
|
43
|
+
# def validate_industries(cls, industries):
|
|
44
|
+
# if industries is not None and not set(industries).issubset(enums.organisation_industries):
|
|
45
|
+
# raise ValueError("Invalid industry values provided.")
|
|
46
|
+
# return industries
|
|
47
|
+
|
|
48
|
+
# @validator('creat_date', 'updt_date', pre=True)
|
|
49
|
+
# def parse_date(cls, value):
|
|
50
|
+
# if value is None:
|
|
51
|
+
# return value
|
|
52
|
+
# if isinstance(value, datetime):
|
|
53
|
+
# return value
|
|
54
|
+
# try:
|
|
55
|
+
# # Assuming Firestore returns an ISO 8601 string, adjust if necessary
|
|
56
|
+
# print("Putting Updt or Creat date in a valid format in a Validator when creating Organisation object")
|
|
57
|
+
# return dateutil.parser.isoparse(value)
|
|
58
|
+
# except (TypeError, ValueError):
|
|
59
|
+
# raise ValidationError(f"Invalid datetime format inside Organisation: {value}")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ### Description, Industries, and Website are optional for Retail Customer and mandatory for Non Retail Customer
|
|
63
|
+
# @validator('description', 'industries', 'website', pre=True, always=True)
|
|
64
|
+
# def validate_optional_fields(cls, value, values):
|
|
65
|
+
# if values.get('name') == 'Retail Customer' and values.get('relations') == {"retail_customer"} or values.get('relations') == ["retail_customer"]:
|
|
66
|
+
# if value is not None:
|
|
67
|
+
# raise ValueError("For 'Retail Customer' with only 'retail_customer' relations, description, industries, and website should not be provided.")
|
|
68
|
+
# else:
|
|
69
|
+
# if value is None:
|
|
70
|
+
# raise ValueError("For Non Retail Customer, description, industries, and website are mandatory.")
|
|
71
|
+
# return value
|
ipulse_shared_core_ftredge-3.1.1/src/ipulse_shared_core_ftredge/models/resource_catalog_item.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# import uuid
|
|
2
|
+
# from datetime import datetime
|
|
3
|
+
# from pydantic import BaseModel, validator, ValidationError
|
|
4
|
+
# from typing import Dict, Any, Set, Optional
|
|
5
|
+
|
|
6
|
+
# import dateutil.parser
|
|
7
|
+
|
|
8
|
+
# CLASS_VERSION = 1.0
|
|
9
|
+
# CLASS_REF = "resdes"
|
|
10
|
+
# MODULE = "core"
|
|
11
|
+
|
|
12
|
+
# class ResourceCatalogItem(BaseModel):
|
|
13
|
+
|
|
14
|
+
# resr_puid_or_name: str #Ex: username
|
|
15
|
+
# resr_path: str #Ex: ipulse-401013/cloud/firesotre/Users/{user_uid}/username
|
|
16
|
+
# resr_name: str #Ex: username
|
|
17
|
+
# resr_pulse_module: str #Ex: core
|
|
18
|
+
# resr_type: str
|
|
19
|
+
# resr_classifications: Set[str]
|
|
20
|
+
# resr_contents:Set[str]
|
|
21
|
+
# resr_original_or_processed: str
|
|
22
|
+
# resr_origin: str
|
|
23
|
+
# resr_origin_organisations_uids: Set[str]
|
|
24
|
+
# resr_origin_description: str
|
|
25
|
+
# resr_licences_types: Set[str]
|
|
26
|
+
# resr_description_details: str
|
|
27
|
+
# resr_updtbl_by_non_staff: bool
|
|
28
|
+
# resr_creat_by_user_uid: str
|
|
29
|
+
# resr_creat_date: datetime
|
|
30
|
+
# class_version:float = CLASS_VERSION
|
|
31
|
+
# resr_columns_count: int
|
|
32
|
+
# resr_columns: Optional[Dict[Any, Any]] = None #OPTIONAL
|
|
33
|
+
# resr_structure_version: Optional[str]=None # OPTIONAL
|
|
34
|
+
# resr_structure_updt_date: Optional[str]=None #OPTIONAL
|
|
35
|
+
# resr_structure_updt_by_user_uid: Optional[str]=None # OPTIONAL
|
|
36
|
+
# resr_tags: Optional[Dict[Any, Any]] = None #OPTIONAL
|
|
37
|
+
# resr_content_updt_date: Optional[str]=None #OPTIONAL
|
|
38
|
+
# resr_content_updt_by_user_uid: Optional[str]=None # OPTIONAL
|
|
39
|
+
# puid: Optional[str] = None #TO BE SETUP BY Validator
|
|
40
|
+
# metadata_version: Optional[float] = None #TO BE SETUP BY Validator
|
|
41
|
+
# metadata_creat_date: Optional[datetime] = None #TO BE SETUP BY Management Service
|
|
42
|
+
# metadata_creat_by: Optional[str] = None #TO BE SETUP BY Management Service
|
|
43
|
+
# metadata_updt_date: Optional[datetime] = None #TO BE SETUP BY Management Service
|
|
44
|
+
# metadata_updt_by: Optional[str] = None #TO BE SETUP BY Management Service
|
|
45
|
+
|
|
46
|
+
# @validator('puid', pre=True, always=True)
|
|
47
|
+
# def set_puid(cls, puid, values):
|
|
48
|
+
# if puid is None:
|
|
49
|
+
# return f"{datetime.utcnow().strftime('%Y%m%d%H%M')}{uuid.uuid4().hex[:8]}_{MODULE}{CLASS_REF}".lower()
|
|
50
|
+
# return puid
|
|
51
|
+
|
|
52
|
+
# @validator('metadata_version', pre=True, always=True)
|
|
53
|
+
# def set_metadata_version(cls, metadata_version, values):
|
|
54
|
+
# if metadata_version is None:
|
|
55
|
+
# return 1.0
|
|
56
|
+
# else:
|
|
57
|
+
# return metadata_version + 0.1
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# @validator('resr_pulse_module', pre=True, always=True)
|
|
61
|
+
# def validate_resr_pulse_module(cls, resr_pulse_modules):
|
|
62
|
+
# if resr_pulse_modules not in enums.pulse_modules:
|
|
63
|
+
# raise ValueError("Invalid pulse_modules values provided.")
|
|
64
|
+
# return resr_pulse_modules
|
|
65
|
+
|
|
66
|
+
# @validator('resr_type', pre=True, always=True)
|
|
67
|
+
# def validate_resr_type(cls, resr_type):
|
|
68
|
+
# if resr_type not in enums.resource_types:
|
|
69
|
+
# raise ValueError("Invalid resource_types value provided.")
|
|
70
|
+
# return resr_type
|
|
71
|
+
|
|
72
|
+
# @validator('resr_classifications', pre=True, always=True)
|
|
73
|
+
# def validate_resr_classifications(cls, resr_classifications):
|
|
74
|
+
# if not resr_classifications.issubset(enums.resource_classifications):
|
|
75
|
+
# raise ValueError("Invalid resr_classifications values provided.")
|
|
76
|
+
# return resr_classifications
|
|
77
|
+
|
|
78
|
+
# @validator('resr_contents', pre=True, always=True)
|
|
79
|
+
# def validate_resr_contents(cls, resr_contents):
|
|
80
|
+
# if not resr_contents.issubset(enums.resource_contents):
|
|
81
|
+
# raise ValueError("Invalid resr_contents values provided.")
|
|
82
|
+
# return resr_contents
|
|
83
|
+
|
|
84
|
+
# @validator('resr_original_or_processed', pre=True, always=True)
|
|
85
|
+
# def validate_resr_original_or_processed(cls, resr_original_or_processed):
|
|
86
|
+
# if resr_original_or_processed not in enums.resource_original_or_processed:
|
|
87
|
+
# raise ValueError("Invalid resr_original_or_processed value provided.")
|
|
88
|
+
# return resr_original_or_processed
|
|
89
|
+
|
|
90
|
+
# @validator('resr_origin', pre=True, always=True)
|
|
91
|
+
# def validate_resr_origin(cls, resr_origin):
|
|
92
|
+
# if resr_origin not in enums.resource_origins:
|
|
93
|
+
# raise ValueError("Invalid resource_origins value provided.")
|
|
94
|
+
# return resr_origin
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# @validator('metadata_creat_date', 'metadata_updt_date', pre=True)
|
|
98
|
+
# def parse_date(cls, value):
|
|
99
|
+
# if value is None:
|
|
100
|
+
# return value
|
|
101
|
+
# if isinstance(value, datetime):
|
|
102
|
+
# return value
|
|
103
|
+
# try:
|
|
104
|
+
# # Assuming Firestore returns an ISO 8601 string, adjust if necessary
|
|
105
|
+
# return dateutil.parser.isoparse(value)
|
|
106
|
+
# except (TypeError, ValueError):
|
|
107
|
+
# raise ValidationError(f"Invalid datetime format inside Resource Description: {value}")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# @validator('metadata_updt_date', 'metadata_updt_date', pre=True, always=True)
|
|
112
|
+
# def set_default_updt_date(cls, date, values):
|
|
113
|
+
# if date is None:
|
|
114
|
+
# return datetime.utcnow().isoformat()
|
|
115
|
+
# return date
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# pylint: disable=missing-module-docstring
|
|
2
|
+
# pylint: disable=missing-function-docstring
|
|
3
|
+
# pylint: disable=logging-fstring-interpolation
|
|
4
|
+
# pylint: disable=line-too-long
|
|
5
|
+
# pylint: disable=missing-class-docstring
|
|
6
|
+
# pylint: disable=broad-exception-caught
|
|
7
|
+
|
|
8
|
+
def list_as_strings(*enums):
|
|
9
|
+
"""Converts a list of Enum members to their string values."""
|
|
10
|
+
return [str(enum) for enum in enums]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: ipulse_shared_core_ftredge
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.1.1
|
|
4
4
|
Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
|
|
5
5
|
Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
|
|
6
6
|
Author: Russlan Ramdowar
|
|
@@ -8,6 +8,4 @@ License-File: LICENCE
|
|
|
8
8
|
Requires-Dist: pydantic[email]~=2.5
|
|
9
9
|
Requires-Dist: python-dateutil~=2.8
|
|
10
10
|
Requires-Dist: pytest~=7.1
|
|
11
|
-
Requires-Dist:
|
|
12
|
-
Requires-Dist: google-cloud-error-reporting~=1.11.0
|
|
13
|
-
Requires-Dist: google-cloud-bigquery~=3.24.0
|
|
11
|
+
Requires-Dist: ipulse_shared_base_ftredge>=2.2.2
|
|
@@ -3,28 +3,18 @@ README.md
|
|
|
3
3
|
pyproject.toml
|
|
4
4
|
setup.py
|
|
5
5
|
src/ipulse_shared_core_ftredge/__init__.py
|
|
6
|
-
src/ipulse_shared_core_ftredge/utils_custom_logs.py
|
|
7
|
-
src/ipulse_shared_core_ftredge/utils_gcp.py
|
|
8
|
-
src/ipulse_shared_core_ftredge/utils_gcp_for_pipelines.py
|
|
9
|
-
src/ipulse_shared_core_ftredge/utils_pipelinemon.py
|
|
10
|
-
src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py
|
|
11
6
|
src/ipulse_shared_core_ftredge.egg-info/PKG-INFO
|
|
12
7
|
src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt
|
|
13
8
|
src/ipulse_shared_core_ftredge.egg-info/dependency_links.txt
|
|
14
9
|
src/ipulse_shared_core_ftredge.egg-info/requires.txt
|
|
15
10
|
src/ipulse_shared_core_ftredge.egg-info/top_level.txt
|
|
16
|
-
src/ipulse_shared_core_ftredge/enums/__init__.py
|
|
17
|
-
src/ipulse_shared_core_ftredge/enums/enums_common_utils.py
|
|
18
|
-
src/ipulse_shared_core_ftredge/enums/enums_data_eng.py
|
|
19
|
-
src/ipulse_shared_core_ftredge/enums/enums_module_fincore.py
|
|
20
|
-
src/ipulse_shared_core_ftredge/enums/enums_modules.py
|
|
21
11
|
src/ipulse_shared_core_ftredge/models/__init__.py
|
|
22
|
-
src/ipulse_shared_core_ftredge/models/audit_log_firestore.py
|
|
23
12
|
src/ipulse_shared_core_ftredge/models/organisation.py
|
|
24
|
-
src/ipulse_shared_core_ftredge/models/pulse_enums.py
|
|
25
13
|
src/ipulse_shared_core_ftredge/models/resource_catalog_item.py
|
|
26
14
|
src/ipulse_shared_core_ftredge/models/user_auth.py
|
|
27
15
|
src/ipulse_shared_core_ftredge/models/user_profile.py
|
|
28
16
|
src/ipulse_shared_core_ftredge/models/user_profile_update.py
|
|
29
17
|
src/ipulse_shared_core_ftredge/models/user_status.py
|
|
18
|
+
src/ipulse_shared_core_ftredge/utils/__init__.py
|
|
19
|
+
src/ipulse_shared_core_ftredge/utils/utils_common.py
|
|
30
20
|
tests/test_utils_gcp.py
|
{ipulse_shared_core_ftredge-2.57 → ipulse_shared_core_ftredge-3.1.1}/tests/test_utils_gcp.py
RENAMED
|
@@ -8,7 +8,7 @@ import os
|
|
|
8
8
|
import json
|
|
9
9
|
from unittest.mock import MagicMock, patch
|
|
10
10
|
import pytest
|
|
11
|
-
from
|
|
11
|
+
from ipulse_shared_core_ftredge.utils_cloud_gcp import write_json_to_gcs
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
# Mocking Google Cloud Storage components for testing using pytest-mock
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
from .models import (Organisation, UserAuth, UserProfile,
|
|
2
|
-
UserStatus, UserProfileUpdate, pulse_enums)
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
from .utils_gcp import (setup_gcp_logger_and_error_report,
|
|
6
|
-
read_csv_from_gcs, read_json_from_gcs,
|
|
7
|
-
write_csv_to_gcs,write_json_to_gcs)
|
|
8
|
-
from .utils_custom_logs import (ContextLog)
|
|
9
|
-
from .utils_pipelinemon import ( Pipelinemon)
|
|
10
|
-
from .utils_gcp_for_pipelines import (write_json_to_gcs_in_pipeline )
|
|
11
|
-
|
|
12
|
-
from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
|
|
13
|
-
check_format_against_schema_template)
|
|
14
|
-
|
|
15
|
-
from .enums import (TargetLogs, LogStatus, LogLevel, Unit, Frequency,
|
|
16
|
-
Module, SubModule, BaseDataCategory,
|
|
17
|
-
FinCoreCategory, FincCoreSubCategory,
|
|
18
|
-
FinCoreRecordsCategory, ExchangeOrPublisher,
|
|
19
|
-
SourcingPipelineType, SourcingTriggerType,
|
|
20
|
-
DWEvent, DWEventTriggerType)
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
# pylint: disable=missing-module-docstring
|
|
3
|
-
# pylint: disable=missing-function-docstring
|
|
4
|
-
# pylint: disable=missing-class-docstring
|
|
5
|
-
|
|
6
|
-
from .enums_common_utils import (LogLevel,
|
|
7
|
-
LogStatus,
|
|
8
|
-
SystemsImpacted,
|
|
9
|
-
TargetLogs,
|
|
10
|
-
Unit,
|
|
11
|
-
Frequency)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
from .enums_modules import(Module,
|
|
15
|
-
SubModule,
|
|
16
|
-
BaseDataCategory)
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
from .enums_module_fincore import (FinCoreCategory,
|
|
20
|
-
FincCoreSubCategory,
|
|
21
|
-
FinCoreRecordsCategory,
|
|
22
|
-
ExchangeOrPublisher)
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
from .enums_data_eng import (SourcingTriggerType,
|
|
27
|
-
SourcingPipelineType,
|
|
28
|
-
DWEvent,
|
|
29
|
-
DWEventTriggerType)
|
|
@@ -1,177 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
# pylint: disable=missing-module-docstring
|
|
3
|
-
# pylint: disable=missing-function-docstring
|
|
4
|
-
# pylint: disable=missing-class-docstring
|
|
5
|
-
# pylint: disable=line-too-long
|
|
6
|
-
|
|
7
|
-
from enum import Enum
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class SystemsImpacted(Enum):
|
|
11
|
-
NO = "__no"
|
|
12
|
-
YES = "__yes"
|
|
13
|
-
INVESTIGATE = "__investigate"
|
|
14
|
-
MULTIPLE = "__multiple"
|
|
15
|
-
DB = "db"
|
|
16
|
-
BQ_TABLE= "bq_table"
|
|
17
|
-
BQ_TABLES = "bq_tables"
|
|
18
|
-
GCS_BUCKET = "gcs_bucket"
|
|
19
|
-
GCS_BUCKETS = "gcs_buckets"
|
|
20
|
-
GCS_BUCKET_FILE = "gcs_bucket_file"
|
|
21
|
-
GCS_BUCKET_FILES = "gcs_bucket_files"
|
|
22
|
-
API = "api"
|
|
23
|
-
APIS = "apis"
|
|
24
|
-
LOCAL_FILE = "local_file"
|
|
25
|
-
LOCAL_FILES = "local_files"
|
|
26
|
-
|
|
27
|
-
class TargetLogs(Enum):
|
|
28
|
-
MIXED="mixed_logs"
|
|
29
|
-
SUCCESSES = "success_logs"
|
|
30
|
-
NOTICES = "notice_logs"
|
|
31
|
-
SUCCESSES_AND_NOTICES = "succs_n_notc_logs"
|
|
32
|
-
WARNINGS = "warning_logs"
|
|
33
|
-
WARNINGS_AND_ERRORS = "warn_n_err_logs"
|
|
34
|
-
ERRORS = "error_logs"
|
|
35
|
-
|
|
36
|
-
class LogLevel(Enum):
|
|
37
|
-
"""
|
|
38
|
-
Standardized notice levels for data engineering pipelines,
|
|
39
|
-
designed for easy analysis and identification of manual
|
|
40
|
-
intervention needs.
|
|
41
|
-
"""
|
|
42
|
-
DEBUG = 10 # Detailed debug information (for development/troubleshooting)
|
|
43
|
-
|
|
44
|
-
INFO = 100
|
|
45
|
-
INFO_REMOTE_PERSISTNACE_COMPLETE= 101
|
|
46
|
-
INFO_REMOTE_UPDATE_COMPLETE = 102
|
|
47
|
-
INFO_REMOTE_DELETE_COMPLETE = 103
|
|
48
|
-
|
|
49
|
-
INFO_REMOTE_BULK_PERSISTNACE_COMPLETE= 111
|
|
50
|
-
INFO_REMOTE_BULK_UPDATE_COMPLETE = 112
|
|
51
|
-
INFO_REMOTE_BULK_DELETE_COMPLETE = 113
|
|
52
|
-
|
|
53
|
-
INFO_LOCAL_PERSISTNACE_COMPLETE = 121
|
|
54
|
-
|
|
55
|
-
SUCCESS = 201
|
|
56
|
-
SUCCESS_WITH_NOTICES = 211
|
|
57
|
-
SUCCESS_WITH_WARNINGS = 212
|
|
58
|
-
|
|
59
|
-
NOTICE = 300 # Maybe same file or data already fully or partially exists
|
|
60
|
-
NOTICE_ALREADY_EXISTS = 301 # Data already exists, no action required
|
|
61
|
-
NOTICE_PARTIAL_EXISTS = 302 # Partial data exists, no action required
|
|
62
|
-
NOTICE_ACTION_CANCELLED = 303 # Data processing cancelled, no action required
|
|
63
|
-
|
|
64
|
-
# Warnings indicate potential issues that might require attention:
|
|
65
|
-
WARNING = 400 # General warning, no immediate action required
|
|
66
|
-
# WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
|
|
67
|
-
WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
|
|
68
|
-
WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
|
|
69
|
-
WARNING_FIX_REQUIRED = 404 # Action required, pipeline can likely continue
|
|
70
|
-
|
|
71
|
-
ERROR = 500 # General error, no immediate action required
|
|
72
|
-
|
|
73
|
-
ERROR_EXCEPTION = 501
|
|
74
|
-
ERROR_CUSTOM = 502 # Temporary error, automatic retry likely to succeed
|
|
75
|
-
ERROR_OPERATION_PARTIALLY_FAILED = 511 # Partial or full failure, manual intervention required
|
|
76
|
-
ERROR_OPERATION_FAILED = 512 # Operation failed, manual intervention required
|
|
77
|
-
ERORR_OPERATION_WITH_WARNINGS = 513 # Partial or full failure, manual intervention required
|
|
78
|
-
ERORR_OPERATION_WITH_ERRORS = 514 # Partial or full failure, manual intervention required
|
|
79
|
-
ERORR_OPERATION_WITH_WARNINGS_OR_ERRORS = 515 # Partial or full failure, manual intervention required
|
|
80
|
-
|
|
81
|
-
ERROR_THRESHOLD_REACHED = 551
|
|
82
|
-
ERROR_PIPELINE_THRESHOLD_REACHED = 552 # Error due to threshold reached, no immediate action required
|
|
83
|
-
ERROR_SUBTHRESHOLD_REACHED = 553 # Error due to threshold reached, no immediate action required
|
|
84
|
-
ERROR_DATA_QUALITY_THRESHOLD_REACHED = 554 # Error due to threshold reached, no immediate action required
|
|
85
|
-
# Critical errors indicate severe failures requiring immediate attention:
|
|
86
|
-
CRITICAL=600 # General critical error, requires immediate action
|
|
87
|
-
CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure, stackoverflow ), requires immediate action
|
|
88
|
-
|
|
89
|
-
UNKNOWN=1001 # Unknown error, should not be used in normal operation
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
class LogStatus(Enum):
|
|
93
|
-
OPEN = "open"
|
|
94
|
-
ACKNOWLEDGED = "acknowledged"
|
|
95
|
-
IN_PROGRESS = "in_progress"
|
|
96
|
-
RESOLVED = "resolved"
|
|
97
|
-
IGNORED = "ignored"
|
|
98
|
-
CANCELLED = "cancelled"
|
|
99
|
-
|
|
100
|
-
### Exception during full exection, partially saved
|
|
101
|
-
# Exception during ensemble pipeline; modifications collected in local object , nothing persisted
|
|
102
|
-
# Exception during ensemble pipeline; modifications persisted , metadata failed
|
|
103
|
-
# Exception during ensemble pipeline; modifications persisted , metadata persisted
|
|
104
|
-
# Exception during ensemble pipeline; modifications persisted , metadata persisted
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
class Unit(Enum):
|
|
108
|
-
MIX="MIX"
|
|
109
|
-
# Currency and Financial Values
|
|
110
|
-
USD = "USD" # United States Dollar
|
|
111
|
-
EUR = "EUR" # Euro
|
|
112
|
-
JPY = "JPY" # Japanese Yen
|
|
113
|
-
GBP = "GBP" # British Pound Sterling
|
|
114
|
-
AUD = "AUD" # Australian Dollar
|
|
115
|
-
CAD = "CAD" # Canadian Dollar
|
|
116
|
-
CHF = "CHF" # Swiss Franc
|
|
117
|
-
CNY = "CNY" # Chinese Yuan Renminbi
|
|
118
|
-
SEK = "SEK" # Swedish Krona
|
|
119
|
-
NZD = "NZD" # New Zealand Dollar
|
|
120
|
-
MXN = "MXN" # Mexican Peso
|
|
121
|
-
SGD = "SGD" # Singapore Dollar
|
|
122
|
-
HKD = "HKD" # Hong Kong Dollar
|
|
123
|
-
NOK = "NOK" # Norwegian Krone
|
|
124
|
-
KRW = "KRW" # South Korean Won
|
|
125
|
-
RUB = "RUB" # Russian Ruble
|
|
126
|
-
INR = "INR" # Indian Rupee
|
|
127
|
-
BRL = "BRL" # Brazilian Real
|
|
128
|
-
ZAR = "ZAR" # South African Rand
|
|
129
|
-
CURRENCY = "currency" # General currency, when specific currency is not needed
|
|
130
|
-
|
|
131
|
-
# Stock Market and Investments
|
|
132
|
-
SHARES = "shares" # Number of shares
|
|
133
|
-
PERCENT = "prcnt" # Percentage, used for rates and ratios
|
|
134
|
-
BPS = "bps" # Basis points, often used for interest rates and financial ratios
|
|
135
|
-
|
|
136
|
-
# Volume and Quantitative Measurements
|
|
137
|
-
VOLUME = "volume" # Trading volume in units
|
|
138
|
-
MILLIONS = "mills" # Millions, used for large quantities or sums
|
|
139
|
-
BILLIONS = "bills" # Billions, used for very large quantities or sums
|
|
140
|
-
|
|
141
|
-
# Commodity Specific Units
|
|
142
|
-
BARRELS = "barrels" # Barrels, specifically for oil and similar liquids
|
|
143
|
-
TONNES = "tonnes" # Tonnes, for bulk materials like metals or grains
|
|
144
|
-
TROY_OUNCES = "troy_oz" # Troy ounces, specifically for precious metals
|
|
145
|
-
|
|
146
|
-
# Real Estate and Physical Properties
|
|
147
|
-
SQUARE_FEET = "sq_ft" # Square feet, for area measurement in real estate
|
|
148
|
-
METER_SQUARE = "m2" # Square meters, for area measurement in real estate
|
|
149
|
-
ACRES = "acres" # Acres, used for measuring large plots of land
|
|
150
|
-
|
|
151
|
-
# Miscellaneous and Other Measures
|
|
152
|
-
UNITS = "units" # Generic units, applicable when other specific units are not suitable
|
|
153
|
-
COUNT = "count" # Count, used for tallying items or events
|
|
154
|
-
INDEX_POINTS = "index_pnts" # Index points, used in measuring indices like stock market indices
|
|
155
|
-
RATIO = "ratio" # Ratio, for various financial ratios
|
|
156
|
-
|
|
157
|
-
class Frequency(Enum):
|
|
158
|
-
ONE_MIN = "1min"
|
|
159
|
-
FIVE_MIN="5min"
|
|
160
|
-
FIFTEEN_MIN="15min"
|
|
161
|
-
THIRTY_MIN = "30min"
|
|
162
|
-
ONE_H = "1h"
|
|
163
|
-
TWO_H = "2h"
|
|
164
|
-
SIX_H = "6h"
|
|
165
|
-
TWELVE_H = "12h"
|
|
166
|
-
FOUR_H = "4h"
|
|
167
|
-
EOD="eod"
|
|
168
|
-
ONE_D = "1d"
|
|
169
|
-
TWO_D = "2d"
|
|
170
|
-
THREE_D = "3d"
|
|
171
|
-
ONE_W = "1w"
|
|
172
|
-
ONE_M = "1m"
|
|
173
|
-
TWO_M="2m"
|
|
174
|
-
THREE_M="3m"
|
|
175
|
-
SIX_M="6m"
|
|
176
|
-
ONE_Y="1y"
|
|
177
|
-
THREE_Y="3y"
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
# pylint: disable=missing-module-docstring
|
|
2
|
-
# pylint: disable=missing-function-docstring
|
|
3
|
-
# pylint: disable=missing-class-docstring
|
|
4
|
-
from enum import Enum
|
|
5
|
-
|
|
6
|
-
class SourcingTriggerType(Enum):
|
|
7
|
-
BULK_MANUAL = "bulk_manual"
|
|
8
|
-
BULK_SCHEDULED = "bulk_scheduled" # almost always historic bulk is manual
|
|
9
|
-
RECENT_SCHEDULED = "recent_scheduled"
|
|
10
|
-
RECENT_MANUAL = "recent_manual"
|
|
11
|
-
ADHOC_MANUAL = "adhoc_manual"
|
|
12
|
-
ADHOC_SCHEDULED = "adhoc_scheduled"
|
|
13
|
-
|
|
14
|
-
class SourcingPipelineType(Enum):
|
|
15
|
-
LOCAL_GET_API_TO_GCS = "local_get_api_to_gcs"
|
|
16
|
-
LOCAL_GET_API_INMEMORY = "local_get_api_inmemory"
|
|
17
|
-
LOCAL_GET_API_TO_LOCAL_FILE = "local_get_api_to_local_file"
|
|
18
|
-
LOCAL_DOWNLOAD_WEB_FILE_TO_LOCAL = "local_download_web_file_to_local"
|
|
19
|
-
LOCAL_DOWNLOAD_WEB_FILE_TO_GCS = "local_download_web_file_to_gcs"
|
|
20
|
-
CLOUD_GET_API_TO_GCS = "cloud_get_api_to_gcs"
|
|
21
|
-
CLOUD_GET_API_INMEMORY = "cloud_get_api_inmemory"
|
|
22
|
-
|
|
23
|
-
class DWEventTriggerType(Enum):
|
|
24
|
-
GCS_UPLOAD_TRIGGER_CF = "gcs_upload_trigger_cf"
|
|
25
|
-
HTTP_TRIGGER_CF_FOR_GCS_FILE = "http_trigger_cf_for_gcs_file"
|
|
26
|
-
PUBSUB_TRIGGER_CF_FOR_GCS_FILE = "pubsub_trigger_cf_for_gcs_file"
|
|
27
|
-
LOCAL_SCRIPT_FOR_GCS_FILE = "local_script_for_gcs_file"
|
|
28
|
-
INSIDE_SOURCING_FUNCTION = "inside_sourcing_function"
|
|
29
|
-
|
|
30
|
-
class DWEvent(Enum):
|
|
31
|
-
INSERT_NOREPLACE_1O_NT = "insert_noreplace_1o_nt"
|
|
32
|
-
MERGE_NOREPLACE_NO_1T = "merge_noreplace_no_1t"
|
|
33
|
-
MERGE_NOREPLACE_NO_NT = "merge_noreplace_no_nt"
|
|
34
|
-
INSERT_NOREPLACE_1O_1T = "insert_noreplace_1o_1t"
|
|
35
|
-
MERGE_NOREPLACE_1O_NT = "merge_noreplace_1o_nt"
|
|
36
|
-
INSERT_REPLACE_1O_1T = "insert_replace_1o_1t"
|
|
37
|
-
INSERT_REPLACE_1O_NT = "insert_replace_1o_nt"
|
|
38
|
-
MERGE_REPLACE_NO_NT = "merge_replace_no_nt"
|
|
39
|
-
MERGE_REPLACE_1O_NT = "merge_replace_1o_nt"
|
|
40
|
-
MERGE_REPLACE_NO_1T = "merge_replace_no_1t"
|
|
41
|
-
DELETE_1O_1T = "delete_1o_1t"
|
|
42
|
-
DELETE_1O_NT = "delete_1o_nt"
|
|
43
|
-
DELETE_NO_1T = "delete_no_1t"
|
|
44
|
-
DELETE_NO_NT = "delete_no_nt"
|
ipulse_shared_core_ftredge-2.57/src/ipulse_shared_core_ftredge/enums/enums_module_fincore.py
DELETED
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
# pylint: disable=missing-module-docstring
|
|
2
|
-
# pylint: disable=missing-function-docstring
|
|
3
|
-
# pylint: disable=missing-class-docstring
|
|
4
|
-
from enum import Enum
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class FinCoreCategory(Enum):
|
|
8
|
-
MARKET="market" # Market prices data
|
|
9
|
-
CORPORATE="corp" # Corporate data such as financial statements and earnings, similar to fundamental data
|
|
10
|
-
FUNDAMENTAL="fundam"
|
|
11
|
-
ECONOMY="economy"
|
|
12
|
-
NEWS="news"
|
|
13
|
-
SENTIMENT="sntmnt"
|
|
14
|
-
SOCIAL="social"
|
|
15
|
-
POLITICS="poltcs"
|
|
16
|
-
OTHER="other"
|
|
17
|
-
|
|
18
|
-
class FincCoreSubCategory(Enum):
|
|
19
|
-
STOCKS = "stocks"
|
|
20
|
-
BONDS = "bonds"
|
|
21
|
-
COMMODITIES = "cmmdt"
|
|
22
|
-
CURRENCIES = "crrncy"
|
|
23
|
-
CRYPTOCURRENCIES = "crypto"
|
|
24
|
-
REAL_ESTATE = "realest"
|
|
25
|
-
EQUITY_INDICES = "eqindx"
|
|
26
|
-
OPTIONS = "options"
|
|
27
|
-
FUTURES = "futures"
|
|
28
|
-
ETF = "etf"
|
|
29
|
-
ECONOMIC_INDICATORS = "ecoind"
|
|
30
|
-
FUNDAMENTALS = "fundam"
|
|
31
|
-
OTHER = "othr"
|
|
32
|
-
|
|
33
|
-
class FinCoreRecordsCategory(Enum):
|
|
34
|
-
PRICE="pric"
|
|
35
|
-
SPOT= "spot"
|
|
36
|
-
OHLCVA="ohlcva"
|
|
37
|
-
OHLCV="ohlcv"
|
|
38
|
-
OPEN="open"
|
|
39
|
-
HIGH="high"
|
|
40
|
-
LOW="low"
|
|
41
|
-
CLOSE="close"
|
|
42
|
-
VOLUME="volume"
|
|
43
|
-
ADJC="adjc"
|
|
44
|
-
FUNDAMENTAL="fundam" # treat this differently
|
|
45
|
-
EARNINGS="earnings"
|
|
46
|
-
CASH_FLOW="cashflw"
|
|
47
|
-
BALANCE_SHEET="blnce_sht"
|
|
48
|
-
INTERNAL_TRANSACTIONS="internaltrans"
|
|
49
|
-
INDICATORS="indic"
|
|
50
|
-
ARTICLE="article"
|
|
51
|
-
INSTA_POST="isntapost"
|
|
52
|
-
TWEET="tweet"
|
|
53
|
-
OTHER="othr"
|
|
54
|
-
|
|
55
|
-
class ExchangeOrPublisher(Enum):
|
|
56
|
-
CC="cc"
|
|
57
|
-
US="us"
|
|
58
|
-
NASDAQ="nasdaq"
|