ethyca-fides 2.63.1rc0__py2.py3-none-any.whl → 2.63.2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/METADATA +1 -1
- {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/RECORD +110 -101
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +160 -0
- fides/api/db/base.py +2 -0
- fides/api/main.py +1 -0
- fides/api/models/detection_discovery/monitor_task.py +1 -0
- fides/api/models/manual_tasks/__init__.py +8 -0
- fides/api/models/manual_tasks/manual_task.py +110 -0
- fides/api/models/manual_tasks/manual_task_log.py +100 -0
- fides/api/schemas/manual_tasks/__init__.py +0 -0
- fides/api/schemas/manual_tasks/manual_task_schemas.py +79 -0
- fides/api/schemas/manual_tasks/manual_task_status.py +151 -0
- fides/api/util/cache.py +77 -1
- fides/config/redis_settings.py +99 -8
- fides/service/manual_tasks/__init__.py +0 -0
- fides/service/manual_tasks/manual_task_service.py +150 -0
- fides/service/messaging/aws_ses_service.py +5 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +1 -1
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{PEElhfUdgE5bJjiyu5QCD → IrQqz_6ngcumU4YlWY9nL}/_buildManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/{PEElhfUdgE5bJjiyu5QCD → IrQqz_6ngcumU4YlWY9nL}/_ssgManifest.js +0 -0
@@ -0,0 +1,110 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
3
|
+
from sqlalchemy import Column, DateTime, ForeignKey, String
|
4
|
+
from sqlalchemy.ext.declarative import declared_attr
|
5
|
+
from sqlalchemy.orm import Session, relationship
|
6
|
+
|
7
|
+
from fides.api.db.base_class import Base
|
8
|
+
from fides.api.db.util import EnumColumn
|
9
|
+
from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
|
10
|
+
from fides.api.schemas.manual_tasks.manual_task_schemas import (
|
11
|
+
ManualTaskLogStatus,
|
12
|
+
ManualTaskParentEntityType,
|
13
|
+
ManualTaskReferenceType,
|
14
|
+
ManualTaskType,
|
15
|
+
)
|
16
|
+
|
17
|
+
|
18
|
+
class ManualTask(Base):
|
19
|
+
"""Model for storing manual tasks.
|
20
|
+
|
21
|
+
This model can be used for both privacy request tasks and general tasks.
|
22
|
+
For privacy requests, it replaces the functionality of manual webhooks.
|
23
|
+
For other use cases, it provides a flexible task management system.
|
24
|
+
|
25
|
+
There can only be one ManualTask per parent entity.
|
26
|
+
You can create multiple Configs for the same ManualTask.
|
27
|
+
"""
|
28
|
+
|
29
|
+
@declared_attr
|
30
|
+
def __tablename__(cls) -> str:
|
31
|
+
"""Overriding base class method to set the table name."""
|
32
|
+
return "manual_task"
|
33
|
+
|
34
|
+
# Database columns
|
35
|
+
task_type = Column(
|
36
|
+
EnumColumn(ManualTaskType),
|
37
|
+
nullable=False,
|
38
|
+
default=ManualTaskType.privacy_request,
|
39
|
+
)
|
40
|
+
parent_entity_id = Column(String, nullable=False)
|
41
|
+
parent_entity_type = Column(
|
42
|
+
EnumColumn(ManualTaskParentEntityType),
|
43
|
+
nullable=False,
|
44
|
+
default=ManualTaskParentEntityType.connection_config,
|
45
|
+
)
|
46
|
+
due_date = Column(DateTime, nullable=True)
|
47
|
+
|
48
|
+
# Relationships
|
49
|
+
references = relationship(
|
50
|
+
"ManualTaskReference",
|
51
|
+
back_populates="task",
|
52
|
+
uselist=True,
|
53
|
+
cascade="all, delete-orphan",
|
54
|
+
)
|
55
|
+
logs = relationship(
|
56
|
+
"ManualTaskLog",
|
57
|
+
back_populates="task",
|
58
|
+
primaryjoin="and_(ManualTask.id == ManualTaskLog.task_id)",
|
59
|
+
viewonly=True,
|
60
|
+
order_by="ManualTaskLog.created_at",
|
61
|
+
)
|
62
|
+
|
63
|
+
# Properties
|
64
|
+
@property
|
65
|
+
def assigned_users(self) -> list[str]:
|
66
|
+
"""Get all users assigned to this task."""
|
67
|
+
if not self.references:
|
68
|
+
return []
|
69
|
+
return [
|
70
|
+
ref.reference_id
|
71
|
+
for ref in self.references
|
72
|
+
if ref.reference_type == ManualTaskReferenceType.assigned_user
|
73
|
+
]
|
74
|
+
|
75
|
+
# CRUD Operations
|
76
|
+
@classmethod
|
77
|
+
def create(
|
78
|
+
cls, db: Session, *, data: dict[str, Any], check_name: bool = True
|
79
|
+
) -> "ManualTask":
|
80
|
+
"""Create a new manual task."""
|
81
|
+
task = super().create(db=db, data=data, check_name=check_name)
|
82
|
+
ManualTaskLog.create_log(
|
83
|
+
db=db,
|
84
|
+
task_id=task.id,
|
85
|
+
status=ManualTaskLogStatus.created,
|
86
|
+
message=f"Created manual task for {data['task_type']}",
|
87
|
+
)
|
88
|
+
return task
|
89
|
+
|
90
|
+
|
91
|
+
class ManualTaskReference(Base):
|
92
|
+
"""Join table to associate manual tasks with multiple references.
|
93
|
+
|
94
|
+
A single task may have many references including privacy requests, configurations, and assigned users.
|
95
|
+
"""
|
96
|
+
|
97
|
+
@declared_attr
|
98
|
+
def __tablename__(cls) -> str:
|
99
|
+
"""Overriding base class method to set the table name."""
|
100
|
+
return "manual_task_reference"
|
101
|
+
|
102
|
+
# Database columns
|
103
|
+
task_id = Column(
|
104
|
+
String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
|
105
|
+
)
|
106
|
+
reference_id = Column(String, nullable=False)
|
107
|
+
reference_type = Column(EnumColumn(ManualTaskReferenceType), nullable=False)
|
108
|
+
|
109
|
+
# Relationships
|
110
|
+
task = relationship("ManualTask", back_populates="references")
|
@@ -0,0 +1,100 @@
|
|
1
|
+
from typing import TYPE_CHECKING, Any, Optional
|
2
|
+
|
3
|
+
from sqlalchemy import Column, ForeignKey, String
|
4
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
5
|
+
from sqlalchemy.ext.declarative import declared_attr
|
6
|
+
from sqlalchemy.orm import Session, relationship
|
7
|
+
|
8
|
+
from fides.api.db.base_class import Base
|
9
|
+
from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskLogStatus
|
10
|
+
|
11
|
+
if TYPE_CHECKING:
|
12
|
+
from fides.api.models.manual_tasks.manual_task import ManualTask
|
13
|
+
|
14
|
+
|
15
|
+
class ManualTaskLog(Base):
|
16
|
+
"""Model for storing manual task execution logs."""
|
17
|
+
|
18
|
+
@declared_attr
|
19
|
+
def __tablename__(cls) -> str:
|
20
|
+
"""Overriding base class method to set the table name."""
|
21
|
+
return "manual_task_log"
|
22
|
+
|
23
|
+
task_id = Column(
|
24
|
+
String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
|
25
|
+
)
|
26
|
+
# TODO: Add foreign key constraints when config and instance are implemented
|
27
|
+
config_id = Column(String, nullable=True)
|
28
|
+
instance_id = Column(String, nullable=True)
|
29
|
+
status = Column(String, nullable=False)
|
30
|
+
message = Column(String, nullable=True)
|
31
|
+
details = Column(JSONB, nullable=True)
|
32
|
+
|
33
|
+
# Relationships - using string references to avoid circular imports
|
34
|
+
task = relationship("ManualTask", back_populates="logs", foreign_keys=[task_id])
|
35
|
+
# TODO: Add config and instance relationships when they are implemented
|
36
|
+
# config = relationship("ManualTaskConfig", back_populates="logs")
|
37
|
+
# instance = relationship("ManualTaskInstance", back_populates="logs")
|
38
|
+
|
39
|
+
@classmethod
|
40
|
+
def create_log(
|
41
|
+
cls,
|
42
|
+
db: Session,
|
43
|
+
status: ManualTaskLogStatus,
|
44
|
+
task_id: str,
|
45
|
+
config_id: Optional[str] = None,
|
46
|
+
instance_id: Optional[str] = None,
|
47
|
+
message: Optional[str] = None,
|
48
|
+
details: Optional[dict[str, Any]] = None,
|
49
|
+
) -> "ManualTaskLog":
|
50
|
+
"""Create a new task log entry.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
db: Database session
|
54
|
+
task_id: ID of the task
|
55
|
+
status: Status of the log entry
|
56
|
+
message: Optional message describing the event
|
57
|
+
details: Optional additional details about the event
|
58
|
+
"""
|
59
|
+
data = {
|
60
|
+
"task_id": task_id,
|
61
|
+
"config_id": config_id,
|
62
|
+
"instance_id": instance_id,
|
63
|
+
"status": status,
|
64
|
+
"message": message,
|
65
|
+
"details": details,
|
66
|
+
}
|
67
|
+
return cls.create(db=db, data=data)
|
68
|
+
|
69
|
+
@classmethod
|
70
|
+
def create_error_log(
|
71
|
+
cls,
|
72
|
+
db: Session,
|
73
|
+
task_id: str,
|
74
|
+
message: str,
|
75
|
+
config_id: Optional[str] = None,
|
76
|
+
instance_id: Optional[str] = None,
|
77
|
+
details: Optional[dict[str, Any]] = None,
|
78
|
+
) -> "ManualTaskLog":
|
79
|
+
"""Create a new error log entry.
|
80
|
+
|
81
|
+
Args:
|
82
|
+
db: Database session
|
83
|
+
task_id: ID of the task
|
84
|
+
message: Error message describing what went wrong
|
85
|
+
config_id: Optional ID of the configuration
|
86
|
+
instance_id: Optional ID of the instance
|
87
|
+
details: Optional additional details about the error
|
88
|
+
|
89
|
+
Returns:
|
90
|
+
The created error log entry
|
91
|
+
"""
|
92
|
+
return cls.create_log(
|
93
|
+
db=db,
|
94
|
+
status=ManualTaskLogStatus.error,
|
95
|
+
task_id=task_id,
|
96
|
+
config_id=config_id,
|
97
|
+
instance_id=instance_id,
|
98
|
+
message=message,
|
99
|
+
details=details,
|
100
|
+
)
|
File without changes
|
@@ -0,0 +1,79 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from enum import Enum
|
3
|
+
from typing import Annotated, Any, Optional
|
4
|
+
|
5
|
+
from pydantic import ConfigDict, Field
|
6
|
+
|
7
|
+
from fides.api.schemas.base_class import FidesSchema
|
8
|
+
|
9
|
+
|
10
|
+
class ManualTaskType(str, Enum):
|
11
|
+
"""Enum for manual task types."""
|
12
|
+
|
13
|
+
privacy_request = "privacy_request"
|
14
|
+
# Add more task types as needed
|
15
|
+
|
16
|
+
|
17
|
+
class ManualTaskParentEntityType(str, Enum):
|
18
|
+
"""Enum for manual task parent entity types."""
|
19
|
+
|
20
|
+
connection_config = (
|
21
|
+
"connection_config" # used for access and erasure privacy requests
|
22
|
+
)
|
23
|
+
# Add more parent entity types as needed
|
24
|
+
|
25
|
+
|
26
|
+
class ManualTaskReferenceType(str, Enum):
|
27
|
+
"""Enum for manual task reference types."""
|
28
|
+
|
29
|
+
privacy_request = "privacy_request"
|
30
|
+
connection_config = "connection_config"
|
31
|
+
manual_task_config = "manual_task_config"
|
32
|
+
assigned_user = "assigned_user" # Reference to the user assigned to the task
|
33
|
+
# Add more reference types as needed
|
34
|
+
|
35
|
+
|
36
|
+
class ManualTaskLogStatus(str, Enum):
|
37
|
+
"""Enum for manual task log status."""
|
38
|
+
|
39
|
+
created = "created"
|
40
|
+
updated = "updated"
|
41
|
+
in_processing = "in_processing"
|
42
|
+
complete = "complete"
|
43
|
+
error = "error"
|
44
|
+
retrying = "retrying"
|
45
|
+
paused = "paused"
|
46
|
+
awaiting_input = "awaiting_input"
|
47
|
+
|
48
|
+
|
49
|
+
class ManualTaskLogCreate(FidesSchema):
|
50
|
+
"""Schema for creating a manual task log entry."""
|
51
|
+
|
52
|
+
model_config = ConfigDict(extra="forbid")
|
53
|
+
|
54
|
+
task_id: Annotated[str, Field(..., description="ID of the task")]
|
55
|
+
status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
|
56
|
+
message: Annotated[Optional[str], Field(None, description="Log message")]
|
57
|
+
details: Annotated[
|
58
|
+
Optional[dict[str, Any]], Field(None, description="Additional details")
|
59
|
+
]
|
60
|
+
config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
|
61
|
+
instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
|
62
|
+
|
63
|
+
|
64
|
+
class ManualTaskLogResponse(FidesSchema):
|
65
|
+
"""Schema for manual task log response."""
|
66
|
+
|
67
|
+
model_config = ConfigDict(extra="forbid")
|
68
|
+
|
69
|
+
id: Annotated[str, Field(..., description="Log ID")]
|
70
|
+
task_id: Annotated[str, Field(..., description="Task ID")]
|
71
|
+
status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
|
72
|
+
message: Annotated[Optional[str], Field(None, description="Log message")]
|
73
|
+
details: Annotated[
|
74
|
+
Optional[dict[str, Any]], Field(None, description="Additional details")
|
75
|
+
]
|
76
|
+
config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
|
77
|
+
instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
|
78
|
+
created_at: Annotated[datetime, Field(..., description="Creation timestamp")]
|
79
|
+
updated_at: Annotated[datetime, Field(..., description="Last update timestamp")]
|
@@ -0,0 +1,151 @@
|
|
1
|
+
from datetime import datetime, timezone
|
2
|
+
from enum import Enum as EnumType
|
3
|
+
from typing import Optional
|
4
|
+
|
5
|
+
from sqlalchemy.orm import Session
|
6
|
+
|
7
|
+
|
8
|
+
class StatusTransitionNotAllowed(Exception):
|
9
|
+
"""Exception raised when a status transition is not allowed."""
|
10
|
+
|
11
|
+
def __init__(self, message: str):
|
12
|
+
self.message = message
|
13
|
+
super().__init__(self.message)
|
14
|
+
|
15
|
+
|
16
|
+
class StatusType(str, EnumType):
|
17
|
+
"""Enum for manual task status."""
|
18
|
+
|
19
|
+
pending = "pending"
|
20
|
+
in_progress = "in_progress"
|
21
|
+
completed = "completed"
|
22
|
+
failed = "failed"
|
23
|
+
|
24
|
+
@classmethod
|
25
|
+
def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
|
26
|
+
"""Get valid transitions from the current status.
|
27
|
+
|
28
|
+
Args:
|
29
|
+
current_status: The current status
|
30
|
+
|
31
|
+
Returns:
|
32
|
+
list[StatusType]: List of valid transitions
|
33
|
+
"""
|
34
|
+
if current_status == cls.pending:
|
35
|
+
return [cls.in_progress, cls.failed]
|
36
|
+
if current_status == cls.in_progress:
|
37
|
+
return [cls.completed, cls.failed]
|
38
|
+
if current_status == cls.completed:
|
39
|
+
return []
|
40
|
+
if current_status == cls.failed:
|
41
|
+
return [cls.pending, cls.in_progress]
|
42
|
+
return []
|
43
|
+
|
44
|
+
|
45
|
+
class StatusTransitionMixin:
|
46
|
+
"""Mixin for handling status transitions.
|
47
|
+
|
48
|
+
This mixin provides methods for managing status transitions and completion tracking.
|
49
|
+
It can be used by any model that needs status management.
|
50
|
+
"""
|
51
|
+
|
52
|
+
# These should be overridden by the implementing class
|
53
|
+
status: StatusType
|
54
|
+
completed_at: Optional[datetime]
|
55
|
+
completed_by_id: Optional[str]
|
56
|
+
|
57
|
+
def _get_valid_transitions(self) -> list[StatusType]:
|
58
|
+
"""Get valid transitions from the current status.
|
59
|
+
|
60
|
+
Returns:
|
61
|
+
list[StatusType]: List of valid transitions
|
62
|
+
"""
|
63
|
+
return StatusType.get_valid_transitions(self.status)
|
64
|
+
|
65
|
+
def _validate_status_transition(self, new_status: StatusType) -> None:
|
66
|
+
"""Validate that a status transition is allowed.
|
67
|
+
|
68
|
+
Args:
|
69
|
+
new_status: The new status to transition to
|
70
|
+
|
71
|
+
Raises:
|
72
|
+
StatusTransitionNotAllowed: If the transition is not allowed
|
73
|
+
"""
|
74
|
+
# Don't allow transitions to the same status
|
75
|
+
if new_status == self.status:
|
76
|
+
raise StatusTransitionNotAllowed(
|
77
|
+
f"Invalid status transition: already in status {new_status}"
|
78
|
+
)
|
79
|
+
|
80
|
+
# Get valid transitions for current status
|
81
|
+
valid_transitions = self._get_valid_transitions()
|
82
|
+
if new_status not in valid_transitions:
|
83
|
+
raise StatusTransitionNotAllowed(
|
84
|
+
f"Invalid status transition from {self.status} to {new_status}. "
|
85
|
+
f"Valid transitions are: {valid_transitions}"
|
86
|
+
)
|
87
|
+
|
88
|
+
def update_status(
|
89
|
+
self, db: Session, new_status: StatusType, user_id: Optional[str] = None
|
90
|
+
) -> None:
|
91
|
+
"""Update the status with validation and completion handling.
|
92
|
+
|
93
|
+
Args:
|
94
|
+
db: Database session
|
95
|
+
new_status: New status to set
|
96
|
+
user_id: Optional user ID who is making the change
|
97
|
+
"""
|
98
|
+
self._validate_status_transition(new_status)
|
99
|
+
|
100
|
+
if new_status == StatusType.completed:
|
101
|
+
self.completed_at = datetime.now(timezone.utc)
|
102
|
+
self.completed_by_id = user_id
|
103
|
+
elif new_status == StatusType.pending:
|
104
|
+
# Reset completion fields if going back to pending
|
105
|
+
self.completed_at = None
|
106
|
+
self.completed_by_id = None
|
107
|
+
|
108
|
+
self.status = new_status
|
109
|
+
db.add(self)
|
110
|
+
db.commit()
|
111
|
+
|
112
|
+
def mark_completed(self, db: Session, user_id: str) -> None:
|
113
|
+
"""Mark as completed.
|
114
|
+
|
115
|
+
Args:
|
116
|
+
db: Database session
|
117
|
+
user_id: user ID who completed the task
|
118
|
+
"""
|
119
|
+
self.update_status(db, StatusType.completed, user_id)
|
120
|
+
|
121
|
+
def mark_failed(self, db: Session) -> None:
|
122
|
+
"""Mark as failed."""
|
123
|
+
self.update_status(db, StatusType.failed)
|
124
|
+
|
125
|
+
def start_progress(self, db: Session) -> None:
|
126
|
+
"""Mark as in progress."""
|
127
|
+
self.update_status(db, StatusType.in_progress)
|
128
|
+
|
129
|
+
def reset_to_pending(self, db: Session) -> None:
|
130
|
+
"""Reset to pending status."""
|
131
|
+
self.update_status(db, StatusType.pending)
|
132
|
+
|
133
|
+
@property
|
134
|
+
def is_completed(self) -> bool:
|
135
|
+
"""Check if completed."""
|
136
|
+
return self.status == StatusType.completed
|
137
|
+
|
138
|
+
@property
|
139
|
+
def is_failed(self) -> bool:
|
140
|
+
"""Check if failed."""
|
141
|
+
return self.status == StatusType.failed
|
142
|
+
|
143
|
+
@property
|
144
|
+
def is_in_progress(self) -> bool:
|
145
|
+
"""Check if in progress."""
|
146
|
+
return self.status == StatusType.in_progress
|
147
|
+
|
148
|
+
@property
|
149
|
+
def is_pending(self) -> bool:
|
150
|
+
"""Check if pending."""
|
151
|
+
return self.status == StatusType.pending
|
fides/api/util/cache.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
import json
|
2
|
+
import os
|
2
3
|
from typing import Any, Dict, List, Optional, Union
|
3
4
|
from urllib.parse import unquote_to_bytes
|
4
5
|
|
@@ -27,6 +28,7 @@ from fides.config import CONFIG
|
|
27
28
|
RedisValue = Union[bytes, float, int, str]
|
28
29
|
|
29
30
|
_connection = None
|
31
|
+
_read_only_connection = None
|
30
32
|
|
31
33
|
|
32
34
|
class FidesopsRedis(Redis):
|
@@ -157,6 +159,36 @@ class FidesopsRedis(Redis):
|
|
157
159
|
return list_length
|
158
160
|
|
159
161
|
|
162
|
+
# FIXME: Ideally we don't want our code to be aware of the way tests are run,
|
163
|
+
# e.g that we run them in parallel with pytest-xdist. We need to find a way
|
164
|
+
# to change the pytest_configure_node hook to set the correct environment variable
|
165
|
+
# like we do for the readonly database. It wasn't working so we're using this workaround for now.
|
166
|
+
def _determine_redis_db_index(
|
167
|
+
read_only: Optional[bool] = False,
|
168
|
+
) -> int: # pragma: no cover
|
169
|
+
"""Return the Redis DB index that should be used for the current process.
|
170
|
+
|
171
|
+
Behavior:
|
172
|
+
1. Test mode:
|
173
|
+
- If running under xdist, map `gwN` → DB `N + 1` (reserve DB 0).
|
174
|
+
- If *not* running under xdist, always use DB 1.
|
175
|
+
|
176
|
+
2. Non-test mode: return the value already present in `CONFIG.redis.db_index`
|
177
|
+
"""
|
178
|
+
|
179
|
+
# 1. Test mode logic
|
180
|
+
if CONFIG.test_mode:
|
181
|
+
worker_id = os.getenv("PYTEST_XDIST_WORKER")
|
182
|
+
if worker_id and worker_id.startswith("gw"):
|
183
|
+
suffix = worker_id[2:]
|
184
|
+
if suffix.isdigit():
|
185
|
+
return int(suffix) + 1 # gw0 -> 1, gw1 -> 2, etc.
|
186
|
+
return CONFIG.redis.test_db_index
|
187
|
+
|
188
|
+
# 2. Non-test mode
|
189
|
+
return CONFIG.redis.read_only_db_index if read_only else CONFIG.redis.db_index
|
190
|
+
|
191
|
+
|
160
192
|
def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
|
161
193
|
"""Return a singleton connection to our Redis cache"""
|
162
194
|
|
@@ -173,7 +205,7 @@ def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
|
|
173
205
|
decode_responses=CONFIG.redis.decode_responses,
|
174
206
|
host=CONFIG.redis.host,
|
175
207
|
port=CONFIG.redis.port,
|
176
|
-
db=
|
208
|
+
db=_determine_redis_db_index(),
|
177
209
|
username=CONFIG.redis.user,
|
178
210
|
password=CONFIG.redis.password,
|
179
211
|
ssl=CONFIG.redis.ssl,
|
@@ -202,6 +234,50 @@ def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
|
|
202
234
|
return _connection
|
203
235
|
|
204
236
|
|
237
|
+
def get_read_only_cache() -> FidesopsRedis:
|
238
|
+
"""
|
239
|
+
Return a singleton connection to the read-only Redis cache.
|
240
|
+
If read-only is not enabled, return the regular cache.
|
241
|
+
"""
|
242
|
+
# If read-only is not enabled, return the regular cache
|
243
|
+
if not CONFIG.redis.read_only_enabled:
|
244
|
+
logger.debug(
|
245
|
+
"Read-only Redis is not enabled. Returning writeable cache connection instead."
|
246
|
+
)
|
247
|
+
return get_cache()
|
248
|
+
|
249
|
+
global _read_only_connection # pylint: disable=W0603
|
250
|
+
if _read_only_connection is None:
|
251
|
+
logger.debug("Creating new read-only Redis connection...")
|
252
|
+
_read_only_connection = FidesopsRedis( # type: ignore[call-overload]
|
253
|
+
charset=CONFIG.redis.charset,
|
254
|
+
decode_responses=CONFIG.redis.decode_responses,
|
255
|
+
host=CONFIG.redis.read_only_host,
|
256
|
+
port=CONFIG.redis.read_only_port,
|
257
|
+
db=_determine_redis_db_index(read_only=True),
|
258
|
+
username=CONFIG.redis.read_only_user,
|
259
|
+
password=CONFIG.redis.read_only_password,
|
260
|
+
ssl=CONFIG.redis.read_only_ssl,
|
261
|
+
ssl_ca_certs=CONFIG.redis.read_only_ssl_ca_certs,
|
262
|
+
ssl_cert_reqs=CONFIG.redis.read_only_ssl_cert_reqs,
|
263
|
+
)
|
264
|
+
logger.debug("New read-only Redis connection created.")
|
265
|
+
|
266
|
+
try:
|
267
|
+
connected = _read_only_connection.ping()
|
268
|
+
logger.debug("Read-only Redis connection succeeded.")
|
269
|
+
except ConnectionErrorFromRedis:
|
270
|
+
connected = False
|
271
|
+
|
272
|
+
if not connected:
|
273
|
+
logger.error(
|
274
|
+
"Unable to establish read-only Redis connection. Returning writeable cache connection instead."
|
275
|
+
)
|
276
|
+
return get_cache()
|
277
|
+
|
278
|
+
return _read_only_connection
|
279
|
+
|
280
|
+
|
205
281
|
def get_identity_cache_key(privacy_request_id: str, identity_attribute: str) -> str:
|
206
282
|
"""Return the key at which to save this PrivacyRequest's identity for the passed in attribute"""
|
207
283
|
# TODO: Remove this prefix
|