ethyca-fides 2.63.1rc0__py2.py3-none-any.whl → 2.63.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/RECORD +110 -101
  3. fides/_version.py +3 -3
  4. fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py +160 -0
  5. fides/api/db/base.py +2 -0
  6. fides/api/main.py +1 -0
  7. fides/api/models/detection_discovery/monitor_task.py +1 -0
  8. fides/api/models/manual_tasks/__init__.py +8 -0
  9. fides/api/models/manual_tasks/manual_task.py +110 -0
  10. fides/api/models/manual_tasks/manual_task_log.py +100 -0
  11. fides/api/schemas/manual_tasks/__init__.py +0 -0
  12. fides/api/schemas/manual_tasks/manual_task_schemas.py +79 -0
  13. fides/api/schemas/manual_tasks/manual_task_status.py +151 -0
  14. fides/api/util/cache.py +77 -1
  15. fides/config/redis_settings.py +99 -8
  16. fides/service/manual_tasks/__init__.py +0 -0
  17. fides/service/manual_tasks/manual_task_service.py +150 -0
  18. fides/service/messaging/aws_ses_service.py +5 -1
  19. fides/ui-build/static/admin/404.html +1 -1
  20. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  21. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  22. fides/ui-build/static/admin/add-systems.html +1 -1
  23. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  24. fides/ui-build/static/admin/consent/configure.html +1 -1
  25. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  26. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  27. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  28. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  29. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  30. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  31. fides/ui-build/static/admin/consent/properties.html +1 -1
  32. fides/ui-build/static/admin/consent/reporting.html +1 -1
  33. fides/ui-build/static/admin/consent.html +1 -1
  34. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  35. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  36. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  37. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  38. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  39. fides/ui-build/static/admin/data-catalog.html +1 -1
  40. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  41. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  42. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  43. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  44. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  45. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  46. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  47. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  48. fides/ui-build/static/admin/datamap.html +1 -1
  49. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  50. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  51. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  52. fides/ui-build/static/admin/dataset/new.html +1 -1
  53. fides/ui-build/static/admin/dataset.html +1 -1
  54. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  55. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  56. fides/ui-build/static/admin/datastore-connection.html +1 -1
  57. fides/ui-build/static/admin/index.html +1 -1
  58. fides/ui-build/static/admin/integrations/[id].html +1 -1
  59. fides/ui-build/static/admin/integrations.html +1 -1
  60. fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
  61. fides/ui-build/static/admin/lib/fides-headless.js +1 -1
  62. fides/ui-build/static/admin/lib/fides-preview.js +1 -1
  63. fides/ui-build/static/admin/lib/fides-tcf.js +1 -1
  64. fides/ui-build/static/admin/lib/fides.js +2 -2
  65. fides/ui-build/static/admin/login/[provider].html +1 -1
  66. fides/ui-build/static/admin/login.html +1 -1
  67. fides/ui-build/static/admin/messaging/[id].html +1 -1
  68. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  69. fides/ui-build/static/admin/messaging.html +1 -1
  70. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  71. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  72. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  73. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  74. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  75. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  76. fides/ui-build/static/admin/poc/forms.html +1 -1
  77. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  78. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  79. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  80. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  81. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  82. fides/ui-build/static/admin/privacy-requests.html +1 -1
  83. fides/ui-build/static/admin/properties/[id].html +1 -1
  84. fides/ui-build/static/admin/properties/add-property.html +1 -1
  85. fides/ui-build/static/admin/properties.html +1 -1
  86. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  87. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  88. fides/ui-build/static/admin/settings/about.html +1 -1
  89. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  90. fides/ui-build/static/admin/settings/consent.html +1 -1
  91. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  92. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  93. fides/ui-build/static/admin/settings/domains.html +1 -1
  94. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  95. fides/ui-build/static/admin/settings/locations.html +1 -1
  96. fides/ui-build/static/admin/settings/organization.html +1 -1
  97. fides/ui-build/static/admin/settings/regulations.html +1 -1
  98. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  99. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  100. fides/ui-build/static/admin/systems.html +1 -1
  101. fides/ui-build/static/admin/taxonomy.html +1 -1
  102. fides/ui-build/static/admin/user-management/new.html +1 -1
  103. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  104. fides/ui-build/static/admin/user-management.html +1 -1
  105. {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/WHEEL +0 -0
  106. {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/entry_points.txt +0 -0
  107. {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/licenses/LICENSE +0 -0
  108. {ethyca_fides-2.63.1rc0.dist-info → ethyca_fides-2.63.2.dist-info}/top_level.txt +0 -0
  109. /fides/ui-build/static/admin/_next/static/{PEElhfUdgE5bJjiyu5QCD → IrQqz_6ngcumU4YlWY9nL}/_buildManifest.js +0 -0
  110. /fides/ui-build/static/admin/_next/static/{PEElhfUdgE5bJjiyu5QCD → IrQqz_6ngcumU4YlWY9nL}/_ssgManifest.js +0 -0
@@ -0,0 +1,110 @@
1
+ from typing import Any
2
+
3
+ from sqlalchemy import Column, DateTime, ForeignKey, String
4
+ from sqlalchemy.ext.declarative import declared_attr
5
+ from sqlalchemy.orm import Session, relationship
6
+
7
+ from fides.api.db.base_class import Base
8
+ from fides.api.db.util import EnumColumn
9
+ from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
10
+ from fides.api.schemas.manual_tasks.manual_task_schemas import (
11
+ ManualTaskLogStatus,
12
+ ManualTaskParentEntityType,
13
+ ManualTaskReferenceType,
14
+ ManualTaskType,
15
+ )
16
+
17
+
18
+ class ManualTask(Base):
19
+ """Model for storing manual tasks.
20
+
21
+ This model can be used for both privacy request tasks and general tasks.
22
+ For privacy requests, it replaces the functionality of manual webhooks.
23
+ For other use cases, it provides a flexible task management system.
24
+
25
+ There can only be one ManualTask per parent entity.
26
+ You can create multiple Configs for the same ManualTask.
27
+ """
28
+
29
+ @declared_attr
30
+ def __tablename__(cls) -> str:
31
+ """Overriding base class method to set the table name."""
32
+ return "manual_task"
33
+
34
+ # Database columns
35
+ task_type = Column(
36
+ EnumColumn(ManualTaskType),
37
+ nullable=False,
38
+ default=ManualTaskType.privacy_request,
39
+ )
40
+ parent_entity_id = Column(String, nullable=False)
41
+ parent_entity_type = Column(
42
+ EnumColumn(ManualTaskParentEntityType),
43
+ nullable=False,
44
+ default=ManualTaskParentEntityType.connection_config,
45
+ )
46
+ due_date = Column(DateTime, nullable=True)
47
+
48
+ # Relationships
49
+ references = relationship(
50
+ "ManualTaskReference",
51
+ back_populates="task",
52
+ uselist=True,
53
+ cascade="all, delete-orphan",
54
+ )
55
+ logs = relationship(
56
+ "ManualTaskLog",
57
+ back_populates="task",
58
+ primaryjoin="and_(ManualTask.id == ManualTaskLog.task_id)",
59
+ viewonly=True,
60
+ order_by="ManualTaskLog.created_at",
61
+ )
62
+
63
+ # Properties
64
+ @property
65
+ def assigned_users(self) -> list[str]:
66
+ """Get all users assigned to this task."""
67
+ if not self.references:
68
+ return []
69
+ return [
70
+ ref.reference_id
71
+ for ref in self.references
72
+ if ref.reference_type == ManualTaskReferenceType.assigned_user
73
+ ]
74
+
75
+ # CRUD Operations
76
+ @classmethod
77
+ def create(
78
+ cls, db: Session, *, data: dict[str, Any], check_name: bool = True
79
+ ) -> "ManualTask":
80
+ """Create a new manual task."""
81
+ task = super().create(db=db, data=data, check_name=check_name)
82
+ ManualTaskLog.create_log(
83
+ db=db,
84
+ task_id=task.id,
85
+ status=ManualTaskLogStatus.created,
86
+ message=f"Created manual task for {data['task_type']}",
87
+ )
88
+ return task
89
+
90
+
91
+ class ManualTaskReference(Base):
92
+ """Join table to associate manual tasks with multiple references.
93
+
94
+ A single task may have many references including privacy requests, configurations, and assigned users.
95
+ """
96
+
97
+ @declared_attr
98
+ def __tablename__(cls) -> str:
99
+ """Overriding base class method to set the table name."""
100
+ return "manual_task_reference"
101
+
102
+ # Database columns
103
+ task_id = Column(
104
+ String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
105
+ )
106
+ reference_id = Column(String, nullable=False)
107
+ reference_type = Column(EnumColumn(ManualTaskReferenceType), nullable=False)
108
+
109
+ # Relationships
110
+ task = relationship("ManualTask", back_populates="references")
@@ -0,0 +1,100 @@
1
+ from typing import TYPE_CHECKING, Any, Optional
2
+
3
+ from sqlalchemy import Column, ForeignKey, String
4
+ from sqlalchemy.dialects.postgresql import JSONB
5
+ from sqlalchemy.ext.declarative import declared_attr
6
+ from sqlalchemy.orm import Session, relationship
7
+
8
+ from fides.api.db.base_class import Base
9
+ from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskLogStatus
10
+
11
+ if TYPE_CHECKING:
12
+ from fides.api.models.manual_tasks.manual_task import ManualTask
13
+
14
+
15
+ class ManualTaskLog(Base):
16
+ """Model for storing manual task execution logs."""
17
+
18
+ @declared_attr
19
+ def __tablename__(cls) -> str:
20
+ """Overriding base class method to set the table name."""
21
+ return "manual_task_log"
22
+
23
+ task_id = Column(
24
+ String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
25
+ )
26
+ # TODO: Add foreign key constraints when config and instance are implemented
27
+ config_id = Column(String, nullable=True)
28
+ instance_id = Column(String, nullable=True)
29
+ status = Column(String, nullable=False)
30
+ message = Column(String, nullable=True)
31
+ details = Column(JSONB, nullable=True)
32
+
33
+ # Relationships - using string references to avoid circular imports
34
+ task = relationship("ManualTask", back_populates="logs", foreign_keys=[task_id])
35
+ # TODO: Add config and instance relationships when they are implemented
36
+ # config = relationship("ManualTaskConfig", back_populates="logs")
37
+ # instance = relationship("ManualTaskInstance", back_populates="logs")
38
+
39
+ @classmethod
40
+ def create_log(
41
+ cls,
42
+ db: Session,
43
+ status: ManualTaskLogStatus,
44
+ task_id: str,
45
+ config_id: Optional[str] = None,
46
+ instance_id: Optional[str] = None,
47
+ message: Optional[str] = None,
48
+ details: Optional[dict[str, Any]] = None,
49
+ ) -> "ManualTaskLog":
50
+ """Create a new task log entry.
51
+
52
+ Args:
53
+ db: Database session
54
+ task_id: ID of the task
55
+ status: Status of the log entry
56
+ message: Optional message describing the event
57
+ details: Optional additional details about the event
58
+ """
59
+ data = {
60
+ "task_id": task_id,
61
+ "config_id": config_id,
62
+ "instance_id": instance_id,
63
+ "status": status,
64
+ "message": message,
65
+ "details": details,
66
+ }
67
+ return cls.create(db=db, data=data)
68
+
69
+ @classmethod
70
+ def create_error_log(
71
+ cls,
72
+ db: Session,
73
+ task_id: str,
74
+ message: str,
75
+ config_id: Optional[str] = None,
76
+ instance_id: Optional[str] = None,
77
+ details: Optional[dict[str, Any]] = None,
78
+ ) -> "ManualTaskLog":
79
+ """Create a new error log entry.
80
+
81
+ Args:
82
+ db: Database session
83
+ task_id: ID of the task
84
+ message: Error message describing what went wrong
85
+ config_id: Optional ID of the configuration
86
+ instance_id: Optional ID of the instance
87
+ details: Optional additional details about the error
88
+
89
+ Returns:
90
+ The created error log entry
91
+ """
92
+ return cls.create_log(
93
+ db=db,
94
+ status=ManualTaskLogStatus.error,
95
+ task_id=task_id,
96
+ config_id=config_id,
97
+ instance_id=instance_id,
98
+ message=message,
99
+ details=details,
100
+ )
File without changes
@@ -0,0 +1,79 @@
1
+ from datetime import datetime
2
+ from enum import Enum
3
+ from typing import Annotated, Any, Optional
4
+
5
+ from pydantic import ConfigDict, Field
6
+
7
+ from fides.api.schemas.base_class import FidesSchema
8
+
9
+
10
+ class ManualTaskType(str, Enum):
11
+ """Enum for manual task types."""
12
+
13
+ privacy_request = "privacy_request"
14
+ # Add more task types as needed
15
+
16
+
17
+ class ManualTaskParentEntityType(str, Enum):
18
+ """Enum for manual task parent entity types."""
19
+
20
+ connection_config = (
21
+ "connection_config" # used for access and erasure privacy requests
22
+ )
23
+ # Add more parent entity types as needed
24
+
25
+
26
+ class ManualTaskReferenceType(str, Enum):
27
+ """Enum for manual task reference types."""
28
+
29
+ privacy_request = "privacy_request"
30
+ connection_config = "connection_config"
31
+ manual_task_config = "manual_task_config"
32
+ assigned_user = "assigned_user" # Reference to the user assigned to the task
33
+ # Add more reference types as needed
34
+
35
+
36
+ class ManualTaskLogStatus(str, Enum):
37
+ """Enum for manual task log status."""
38
+
39
+ created = "created"
40
+ updated = "updated"
41
+ in_processing = "in_processing"
42
+ complete = "complete"
43
+ error = "error"
44
+ retrying = "retrying"
45
+ paused = "paused"
46
+ awaiting_input = "awaiting_input"
47
+
48
+
49
+ class ManualTaskLogCreate(FidesSchema):
50
+ """Schema for creating a manual task log entry."""
51
+
52
+ model_config = ConfigDict(extra="forbid")
53
+
54
+ task_id: Annotated[str, Field(..., description="ID of the task")]
55
+ status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
56
+ message: Annotated[Optional[str], Field(None, description="Log message")]
57
+ details: Annotated[
58
+ Optional[dict[str, Any]], Field(None, description="Additional details")
59
+ ]
60
+ config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
61
+ instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
62
+
63
+
64
+ class ManualTaskLogResponse(FidesSchema):
65
+ """Schema for manual task log response."""
66
+
67
+ model_config = ConfigDict(extra="forbid")
68
+
69
+ id: Annotated[str, Field(..., description="Log ID")]
70
+ task_id: Annotated[str, Field(..., description="Task ID")]
71
+ status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
72
+ message: Annotated[Optional[str], Field(None, description="Log message")]
73
+ details: Annotated[
74
+ Optional[dict[str, Any]], Field(None, description="Additional details")
75
+ ]
76
+ config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
77
+ instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
78
+ created_at: Annotated[datetime, Field(..., description="Creation timestamp")]
79
+ updated_at: Annotated[datetime, Field(..., description="Last update timestamp")]
@@ -0,0 +1,151 @@
1
+ from datetime import datetime, timezone
2
+ from enum import Enum as EnumType
3
+ from typing import Optional
4
+
5
+ from sqlalchemy.orm import Session
6
+
7
+
8
+ class StatusTransitionNotAllowed(Exception):
9
+ """Exception raised when a status transition is not allowed."""
10
+
11
+ def __init__(self, message: str):
12
+ self.message = message
13
+ super().__init__(self.message)
14
+
15
+
16
+ class StatusType(str, EnumType):
17
+ """Enum for manual task status."""
18
+
19
+ pending = "pending"
20
+ in_progress = "in_progress"
21
+ completed = "completed"
22
+ failed = "failed"
23
+
24
+ @classmethod
25
+ def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
26
+ """Get valid transitions from the current status.
27
+
28
+ Args:
29
+ current_status: The current status
30
+
31
+ Returns:
32
+ list[StatusType]: List of valid transitions
33
+ """
34
+ if current_status == cls.pending:
35
+ return [cls.in_progress, cls.failed]
36
+ if current_status == cls.in_progress:
37
+ return [cls.completed, cls.failed]
38
+ if current_status == cls.completed:
39
+ return []
40
+ if current_status == cls.failed:
41
+ return [cls.pending, cls.in_progress]
42
+ return []
43
+
44
+
45
+ class StatusTransitionMixin:
46
+ """Mixin for handling status transitions.
47
+
48
+ This mixin provides methods for managing status transitions and completion tracking.
49
+ It can be used by any model that needs status management.
50
+ """
51
+
52
+ # These should be overridden by the implementing class
53
+ status: StatusType
54
+ completed_at: Optional[datetime]
55
+ completed_by_id: Optional[str]
56
+
57
+ def _get_valid_transitions(self) -> list[StatusType]:
58
+ """Get valid transitions from the current status.
59
+
60
+ Returns:
61
+ list[StatusType]: List of valid transitions
62
+ """
63
+ return StatusType.get_valid_transitions(self.status)
64
+
65
+ def _validate_status_transition(self, new_status: StatusType) -> None:
66
+ """Validate that a status transition is allowed.
67
+
68
+ Args:
69
+ new_status: The new status to transition to
70
+
71
+ Raises:
72
+ StatusTransitionNotAllowed: If the transition is not allowed
73
+ """
74
+ # Don't allow transitions to the same status
75
+ if new_status == self.status:
76
+ raise StatusTransitionNotAllowed(
77
+ f"Invalid status transition: already in status {new_status}"
78
+ )
79
+
80
+ # Get valid transitions for current status
81
+ valid_transitions = self._get_valid_transitions()
82
+ if new_status not in valid_transitions:
83
+ raise StatusTransitionNotAllowed(
84
+ f"Invalid status transition from {self.status} to {new_status}. "
85
+ f"Valid transitions are: {valid_transitions}"
86
+ )
87
+
88
+ def update_status(
89
+ self, db: Session, new_status: StatusType, user_id: Optional[str] = None
90
+ ) -> None:
91
+ """Update the status with validation and completion handling.
92
+
93
+ Args:
94
+ db: Database session
95
+ new_status: New status to set
96
+ user_id: Optional user ID who is making the change
97
+ """
98
+ self._validate_status_transition(new_status)
99
+
100
+ if new_status == StatusType.completed:
101
+ self.completed_at = datetime.now(timezone.utc)
102
+ self.completed_by_id = user_id
103
+ elif new_status == StatusType.pending:
104
+ # Reset completion fields if going back to pending
105
+ self.completed_at = None
106
+ self.completed_by_id = None
107
+
108
+ self.status = new_status
109
+ db.add(self)
110
+ db.commit()
111
+
112
+ def mark_completed(self, db: Session, user_id: str) -> None:
113
+ """Mark as completed.
114
+
115
+ Args:
116
+ db: Database session
117
+ user_id: user ID who completed the task
118
+ """
119
+ self.update_status(db, StatusType.completed, user_id)
120
+
121
+ def mark_failed(self, db: Session) -> None:
122
+ """Mark as failed."""
123
+ self.update_status(db, StatusType.failed)
124
+
125
+ def start_progress(self, db: Session) -> None:
126
+ """Mark as in progress."""
127
+ self.update_status(db, StatusType.in_progress)
128
+
129
+ def reset_to_pending(self, db: Session) -> None:
130
+ """Reset to pending status."""
131
+ self.update_status(db, StatusType.pending)
132
+
133
+ @property
134
+ def is_completed(self) -> bool:
135
+ """Check if completed."""
136
+ return self.status == StatusType.completed
137
+
138
+ @property
139
+ def is_failed(self) -> bool:
140
+ """Check if failed."""
141
+ return self.status == StatusType.failed
142
+
143
+ @property
144
+ def is_in_progress(self) -> bool:
145
+ """Check if in progress."""
146
+ return self.status == StatusType.in_progress
147
+
148
+ @property
149
+ def is_pending(self) -> bool:
150
+ """Check if pending."""
151
+ return self.status == StatusType.pending
fides/api/util/cache.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import json
2
+ import os
2
3
  from typing import Any, Dict, List, Optional, Union
3
4
  from urllib.parse import unquote_to_bytes
4
5
 
@@ -27,6 +28,7 @@ from fides.config import CONFIG
27
28
  RedisValue = Union[bytes, float, int, str]
28
29
 
29
30
  _connection = None
31
+ _read_only_connection = None
30
32
 
31
33
 
32
34
  class FidesopsRedis(Redis):
@@ -157,6 +159,36 @@ class FidesopsRedis(Redis):
157
159
  return list_length
158
160
 
159
161
 
162
+ # FIXME: Ideally we don't want our code to be aware of the way tests are run,
163
+ # e.g that we run them in parallel with pytest-xdist. We need to find a way
164
+ # to change the pytest_configure_node hook to set the correct environment variable
165
+ # like we do for the readonly database. It wasn't working so we're using this workaround for now.
166
+ def _determine_redis_db_index(
167
+ read_only: Optional[bool] = False,
168
+ ) -> int: # pragma: no cover
169
+ """Return the Redis DB index that should be used for the current process.
170
+
171
+ Behavior:
172
+ 1. Test mode:
173
+ - If running under xdist, map `gwN` → DB `N + 1` (reserve DB 0).
174
+ - If *not* running under xdist, always use DB 1.
175
+
176
+ 2. Non-test mode: return the value already present in `CONFIG.redis.db_index`
177
+ """
178
+
179
+ # 1. Test mode logic
180
+ if CONFIG.test_mode:
181
+ worker_id = os.getenv("PYTEST_XDIST_WORKER")
182
+ if worker_id and worker_id.startswith("gw"):
183
+ suffix = worker_id[2:]
184
+ if suffix.isdigit():
185
+ return int(suffix) + 1 # gw0 -> 1, gw1 -> 2, etc.
186
+ return CONFIG.redis.test_db_index
187
+
188
+ # 2. Non-test mode
189
+ return CONFIG.redis.read_only_db_index if read_only else CONFIG.redis.db_index
190
+
191
+
160
192
  def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
161
193
  """Return a singleton connection to our Redis cache"""
162
194
 
@@ -173,7 +205,7 @@ def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
173
205
  decode_responses=CONFIG.redis.decode_responses,
174
206
  host=CONFIG.redis.host,
175
207
  port=CONFIG.redis.port,
176
- db=CONFIG.redis.db_index,
208
+ db=_determine_redis_db_index(),
177
209
  username=CONFIG.redis.user,
178
210
  password=CONFIG.redis.password,
179
211
  ssl=CONFIG.redis.ssl,
@@ -202,6 +234,50 @@ def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
202
234
  return _connection
203
235
 
204
236
 
237
+ def get_read_only_cache() -> FidesopsRedis:
238
+ """
239
+ Return a singleton connection to the read-only Redis cache.
240
+ If read-only is not enabled, return the regular cache.
241
+ """
242
+ # If read-only is not enabled, return the regular cache
243
+ if not CONFIG.redis.read_only_enabled:
244
+ logger.debug(
245
+ "Read-only Redis is not enabled. Returning writeable cache connection instead."
246
+ )
247
+ return get_cache()
248
+
249
+ global _read_only_connection # pylint: disable=W0603
250
+ if _read_only_connection is None:
251
+ logger.debug("Creating new read-only Redis connection...")
252
+ _read_only_connection = FidesopsRedis( # type: ignore[call-overload]
253
+ charset=CONFIG.redis.charset,
254
+ decode_responses=CONFIG.redis.decode_responses,
255
+ host=CONFIG.redis.read_only_host,
256
+ port=CONFIG.redis.read_only_port,
257
+ db=_determine_redis_db_index(read_only=True),
258
+ username=CONFIG.redis.read_only_user,
259
+ password=CONFIG.redis.read_only_password,
260
+ ssl=CONFIG.redis.read_only_ssl,
261
+ ssl_ca_certs=CONFIG.redis.read_only_ssl_ca_certs,
262
+ ssl_cert_reqs=CONFIG.redis.read_only_ssl_cert_reqs,
263
+ )
264
+ logger.debug("New read-only Redis connection created.")
265
+
266
+ try:
267
+ connected = _read_only_connection.ping()
268
+ logger.debug("Read-only Redis connection succeeded.")
269
+ except ConnectionErrorFromRedis:
270
+ connected = False
271
+
272
+ if not connected:
273
+ logger.error(
274
+ "Unable to establish read-only Redis connection. Returning writeable cache connection instead."
275
+ )
276
+ return get_cache()
277
+
278
+ return _read_only_connection
279
+
280
+
205
281
  def get_identity_cache_key(privacy_request_id: str, identity_attribute: str) -> str:
206
282
  """Return the key at which to save this PrivacyRequest's identity for the passed in attribute"""
207
283
  # TODO: Remove this prefix