pacnpal/thrillwiki_django_no_react
1
1
Fork 0
mirror of https://github.com/pacnpal/thrillwiki_django_no_react.git synced 2026-02-05 05:45:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

Based on the git diff provided, here's a concise and descriptive commit message:

Browse Source 
feat: add passkey authentication and enhance user preferences

- Add passkey login security event type with fingerprint icon
- Include request and site context in email confirmation for backend
- Add user_id exact match filter to prevent incorrect user lookups
- Enable PATCH method for updating user preferences via API
- Add moderation_preferences support to user settings
- Optimize ticket queries with select_related and prefetch_related

This commit introduces passkey authentication tracking, improves user
profile filtering accuracy, and extends the preferences API to support
updates. Query optimizations reduce database hits for ticket listings.
This commit is contained in:
pacnpal
2026-01-12 19:13:05 -05:00
parent 2b66814d82
commit d631f3183c
56 changed files with 5860 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
backend/apps/core/api/observability_serializers.py Normal file
View File

@@ -0,0 +1,176 @@
"""
Serializers for observability API endpoints.
Provides serializers for PipelineError, Anomaly, AlertCorrelationRule,
CleanupJobLog, and DataRetentionStats.
"""
from rest_framework import serializers
from apps.core.choices.serializers import RichChoiceSerializerField
from apps.core.models import (
AlertCorrelationRule,
Anomaly,
CleanupJobLog,
PipelineError,
)
class PipelineErrorSerializer(serializers.ModelSerializer):
"""Serializer for pipeline errors."""
severity = RichChoiceSerializerField(
choice_group="pipeline_error_severities",
domain="core",
)
resolved_by_username = serializers.CharField(
source="resolved_by.username",
read_only=True,
allow_null=True,
)
class Meta:
model = PipelineError
fields = [
"id",
"function_name",
"error_message",
"error_code",
"error_context",
"stack_trace",
"severity",
"submission_id",
"item_id",
"request_id",
"trace_id",
"resolved",
"resolved_by",
"resolved_by_username",
"resolved_at",
"resolution_notes",
"occurred_at",
]
read_only_fields = ["id", "occurred_at", "resolved_by_username"]
class PipelineErrorResolveSerializer(serializers.Serializer):
"""Serializer for resolving pipeline errors."""
resolution_notes = serializers.CharField(required=False, allow_blank=True)
class AnomalySerializer(serializers.ModelSerializer):
"""Serializer for detected anomalies."""
anomaly_type = RichChoiceSerializerField(
choice_group="anomaly_types",
domain="core",
)
severity = RichChoiceSerializerField(
choice_group="severity_levels",
domain="core",
)
alert_message = serializers.CharField(
source="alert.message",
read_only=True,
allow_null=True,
)
alert_resolved_at = serializers.DateTimeField(
source="alert.resolved_at",
read_only=True,
allow_null=True,
)
alert_id = serializers.UUIDField(
source="alert.id",
read_only=True,
allow_null=True,
)
class Meta:
model = Anomaly
fields = [
"id",
"metric_name",
"metric_category",
"anomaly_type",
"severity",
"anomaly_value",
"baseline_value",
"deviation_score",
"confidence_score",
"detection_algorithm",
"time_window_start",
"time_window_end",
"alert_created",
"alert_id",
"alert_message",
"alert_resolved_at",
"detected_at",
]
read_only_fields = [
"id",
"detected_at",
"alert_id",
"alert_message",
"alert_resolved_at",
]
class AlertCorrelationRuleSerializer(serializers.ModelSerializer):
"""Serializer for alert correlation rules."""
incident_severity = RichChoiceSerializerField(
choice_group="severity_levels",
domain="core",
)
class Meta:
model = AlertCorrelationRule
fields = [
"id",
"rule_name",
"rule_description",
"min_alerts_required",
"time_window_minutes",
"incident_severity",
"incident_title_template",
"is_active",
"created_at",
"updated_at",
]
read_only_fields = ["id", "created_at", "updated_at"]
class CleanupJobLogSerializer(serializers.ModelSerializer):
"""Serializer for cleanup job logs."""
status = RichChoiceSerializerField(
choice_group="cleanup_job_statuses",
domain="core",
)
class Meta:
model = CleanupJobLog
fields = [
"id",
"job_name",
"status",
"records_processed",
"records_deleted",
"error_message",
"duration_ms",
"executed_at",
]
read_only_fields = ["id", "executed_at"]
class DataRetentionStatsSerializer(serializers.Serializer):
"""Serializer for data retention statistics view."""
table_name = serializers.CharField()
total_records = serializers.IntegerField()
last_7_days = serializers.IntegerField()
last_30_days = serializers.IntegerField()
oldest_record = serializers.DateTimeField(allow_null=True)
newest_record = serializers.DateTimeField(allow_null=True)
table_size = serializers.CharField()

351
backend/apps/core/api/observability_views.py Normal file
View File

@@ -0,0 +1,351 @@
"""
ViewSets and Views for observability API endpoints.
Provides CRUD operations for PipelineError, read-only access for
Anomaly, AlertCorrelationRule, CleanupJobLog, and aggregated views
for DataRetentionStats.
"""
from django.db import connection
from django.db.models import Count, Max, Min
from django.db.models.functions import Coalesce
from django.utils import timezone
from django_filters.rest_framework import DjangoFilterBackend
from drf_spectacular.utils import extend_schema, extend_schema_view
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.filters import OrderingFilter, SearchFilter
from rest_framework.permissions import IsAdminUser
from rest_framework.response import Response
from rest_framework.views import APIView
from apps.core.models import (
AlertCorrelationRule,
Anomaly,
CleanupJobLog,
PipelineError,
)
from .observability_serializers import (
AlertCorrelationRuleSerializer,
AnomalySerializer,
CleanupJobLogSerializer,
DataRetentionStatsSerializer,
PipelineErrorResolveSerializer,
PipelineErrorSerializer,
)
@extend_schema_view(
list=extend_schema(
summary="List pipeline errors",
description="Get all pipeline errors, optionally filtered by severity or resolved status.",
tags=["Admin - Observability"],
),
retrieve=extend_schema(
summary="Get pipeline error",
description="Get details of a specific pipeline error.",
tags=["Admin - Observability"],
),
create=extend_schema(
summary="Create pipeline error",
description="Create a new pipeline error.",
tags=["Admin - Observability"],
),
update=extend_schema(
summary="Update pipeline error",
description="Update an existing pipeline error.",
tags=["Admin - Observability"],
),
partial_update=extend_schema(
summary="Partial update pipeline error",
description="Partially update an existing pipeline error.",
tags=["Admin - Observability"],
),
destroy=extend_schema(
summary="Delete pipeline error",
description="Delete a pipeline error.",
tags=["Admin - Observability"],
),
)
class PipelineErrorViewSet(viewsets.ModelViewSet):
"""
ViewSet for managing pipeline errors.
Provides CRUD operations plus a resolve action for marking errors as resolved.
"""
queryset = PipelineError.objects.select_related("resolved_by").all()
serializer_class = PipelineErrorSerializer
permission_classes = [IsAdminUser]
filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter]
filterset_fields = ["severity", "function_name", "resolved", "error_code"]
search_fields = ["error_message", "function_name", "error_code"]
ordering_fields = ["occurred_at", "severity"]
ordering = ["-occurred_at"]
def get_queryset(self):
queryset = super().get_queryset()
# Date range filtering
start_date = self.request.query_params.get("start_date")
end_date = self.request.query_params.get("end_date")
if start_date:
queryset = queryset.filter(occurred_at__gte=start_date)
if end_date:
queryset = queryset.filter(occurred_at__lte=end_date)
return queryset
@extend_schema(
summary="Resolve pipeline error",
description="Mark a pipeline error as resolved.",
request=PipelineErrorResolveSerializer,
responses={200: PipelineErrorSerializer},
tags=["Admin - Observability"],
)
@action(detail=True, methods=["post"])
def resolve(self, request, pk=None):
"""Mark a pipeline error as resolved."""
error = self.get_object()
if error.resolved:
return Response(
{"detail": "Error is already resolved"},
status=status.HTTP_400_BAD_REQUEST,
)
serializer = PipelineErrorResolveSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
error.resolved = True
error.resolved_at = timezone.now()
error.resolved_by = request.user
error.resolution_notes = serializer.validated_data.get("resolution_notes", "")
error.save()
return Response(PipelineErrorSerializer(error).data)
@extend_schema_view(
list=extend_schema(
summary="List recent anomalies",
description="Get recent anomalies with optional filtering by severity or type.",
tags=["Admin - Observability"],
),
retrieve=extend_schema(
summary="Get anomaly details",
description="Get details of a specific anomaly.",
tags=["Admin - Observability"],
),
)
class AnomalyViewSet(viewsets.ReadOnlyModelViewSet):
"""
ViewSet for viewing detected anomalies.
Provides read-only access to anomaly data with filtering options.
This serves as the recent_anomalies_view endpoint.
"""
queryset = Anomaly.objects.select_related("alert").all()
serializer_class = AnomalySerializer
permission_classes = [IsAdminUser]
filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter]
filterset_fields = ["severity", "anomaly_type", "metric_category", "alert_created"]
search_fields = ["metric_name", "metric_category"]
ordering_fields = ["detected_at", "severity", "deviation_score"]
ordering = ["-detected_at"]
def get_queryset(self):
queryset = super().get_queryset()
# Date range filtering
start_date = self.request.query_params.get("start_date")
end_date = self.request.query_params.get("end_date")
if start_date:
queryset = queryset.filter(detected_at__gte=start_date)
if end_date:
queryset = queryset.filter(detected_at__lte=end_date)
return queryset
@extend_schema_view(
list=extend_schema(
summary="List alert correlations",
description="Get all alert correlation rules with optional filtering.",
tags=["Admin - Observability"],
),
retrieve=extend_schema(
summary="Get alert correlation rule",
description="Get details of a specific alert correlation rule.",
tags=["Admin - Observability"],
),
create=extend_schema(
summary="Create alert correlation rule",
description="Create a new alert correlation rule.",
tags=["Admin - Observability"],
),
update=extend_schema(
summary="Update alert correlation rule",
description="Update an existing alert correlation rule.",
tags=["Admin - Observability"],
),
partial_update=extend_schema(
summary="Partial update alert correlation rule",
description="Partially update an existing alert correlation rule.",
tags=["Admin - Observability"],
),
destroy=extend_schema(
summary="Delete alert correlation rule",
description="Delete an alert correlation rule.",
tags=["Admin - Observability"],
),
)
class AlertCorrelationViewSet(viewsets.ModelViewSet):
"""
ViewSet for managing alert correlation rules.
Provides CRUD operations for configuring how alerts are correlated.
This serves as the alert_correlations_view endpoint.
"""
queryset = AlertCorrelationRule.objects.all()
serializer_class = AlertCorrelationRuleSerializer
permission_classes = [IsAdminUser]
filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter]
filterset_fields = ["is_active", "incident_severity"]
search_fields = ["rule_name", "rule_description"]
ordering_fields = ["rule_name", "created_at"]
ordering = ["rule_name"]
@extend_schema_view(
list=extend_schema(
summary="List cleanup job logs",
description="Get all cleanup job logs with optional filtering by status.",
tags=["Admin - Observability"],
),
retrieve=extend_schema(
summary="Get cleanup job log",
description="Get details of a specific cleanup job log entry.",
tags=["Admin - Observability"],
),
)
class CleanupJobLogViewSet(viewsets.ReadOnlyModelViewSet):
"""
ViewSet for viewing cleanup job logs.
Provides read-only access to cleanup job execution history.
"""
queryset = CleanupJobLog.objects.all()
serializer_class = CleanupJobLogSerializer
permission_classes = [IsAdminUser]
filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter]
filterset_fields = ["status", "job_name"]
search_fields = ["job_name", "error_message"]
ordering_fields = ["executed_at", "duration_ms"]
ordering = ["-executed_at"]
def get_queryset(self):
queryset = super().get_queryset()
# Date range filtering
start_date = self.request.query_params.get("start_date")
end_date = self.request.query_params.get("end_date")
if start_date:
queryset = queryset.filter(executed_at__gte=start_date)
if end_date:
queryset = queryset.filter(executed_at__lte=end_date)
return queryset
@extend_schema(
summary="Get data retention stats",
description="Get aggregated data retention statistics for monitoring database growth.",
tags=["Admin - Observability"],
responses={200: DataRetentionStatsSerializer(many=True)},
)
class DataRetentionStatsView(APIView):
"""
API view for data retention statistics.
Returns aggregated statistics about table sizes, record counts,
and data age for monitoring data retention and growth.
"""
permission_classes = [IsAdminUser]
def get(self, request):
"""Get data retention statistics for key tables."""
from datetime import timedelta
from django.apps import apps
now = timezone.now()
seven_days_ago = now - timedelta(days=7)
thirty_days_ago = now - timedelta(days=30)
# Tables to report on
tables_to_check = [
("core", "pipelineerror", "occurred_at"),
("core", "applicationerror", "created_at"),
("core", "systemalert", "created_at"),
("core", "requestmetadata", "created_at"),
("core", "anomaly", "detected_at"),
("core", "cleanupjoblog", "executed_at"),
("moderation", "editsubmission", "created_at"),
("moderation", "moderationauditlog", "created_at"),
("notifications", "notificationlog", "created_at"),
]
stats = []
for app_label, model_name, date_field in tables_to_check:
try:
model = apps.get_model(app_label, model_name)
filter_kwargs_7d = {f"{date_field}__gte": seven_days_ago}
filter_kwargs_30d = {f"{date_field}__gte": thirty_days_ago}
# Get record counts and date ranges
qs = model.objects.aggregate(
total=Coalesce(Count("id"), 0),
last_7_days=Coalesce(Count("id", filter=model.objects.filter(**filter_kwargs_7d).query.where), 0),
last_30_days=Coalesce(Count("id", filter=model.objects.filter(**filter_kwargs_30d).query.where), 0),
oldest_record=Min(date_field),
newest_record=Max(date_field),
)
# Get table size from database
table_name = model._meta.db_table
with connection.cursor() as cursor:
cursor.execute(
"""
SELECT pg_size_pretty(pg_total_relation_size(%s))
""",
[table_name],
)
result = cursor.fetchone()
table_size = result[0] if result else "Unknown"
stats.append(
{
"table_name": table_name,
"total_records": model.objects.count(),
"last_7_days": model.objects.filter(**filter_kwargs_7d).count(),
"last_30_days": model.objects.filter(**filter_kwargs_30d).count(),
"oldest_record": qs.get("oldest_record"),
"newest_record": qs.get("newest_record"),
"table_size": table_size,
}
)
except Exception:
# Skip tables that don't exist or have errors
continue
serializer = DataRetentionStatsSerializer(stats, many=True)
return Response(serializer.data)

3
backend/apps/core/choices/__init__.py
View File

@@ -15,7 +15,7 @@ Key Components:
from .base import ChoiceCategory, ChoiceGroup, RichChoice
from .fields import RichChoiceField
from .registry import ChoiceRegistry, register_choices
from .serializers import RichChoiceOptionSerializer, RichChoiceSerializer
from .serializers import RichChoiceOptionSerializer, RichChoiceSerializer, RichChoiceSerializerField
from .utils import get_choice_display, validate_choice_value
__all__ = [
@@ -26,6 +26,7 @@ __all__ = [
"register_choices",
"RichChoiceField",
"RichChoiceSerializer",
"RichChoiceSerializerField",
"RichChoiceOptionSerializer",
"validate_choice_value",
"get_choice_display",

670
backend/apps/core/choices/core_choices.py
View File

@@ -2,7 +2,8 @@
Core System Rich Choice Objects
This module defines all choice objects for core system functionality,
including health checks, API statuses, and other system-level choices.
including health checks, API statuses, severity levels, alert types,
and other system-level choices.
"""
from .base import ChoiceCategory, RichChoice
@@ -124,6 +125,584 @@ ENTITY_TYPES = [
),
]
# ============================================================================
# Severity Levels (used by ApplicationError, SystemAlert, Incident, RequestMetadata)
# ============================================================================
SEVERITY_LEVELS = [
RichChoice(
value="critical",
label="Critical",
description="Critical issue requiring immediate attention",
metadata={
"color": "red",
"icon": "alert-octagon",
"css_class": "bg-red-100 text-red-800 border-red-300",
"sort_order": 1,
"priority": 1,
},
category=ChoiceCategory.PRIORITY,
),
RichChoice(
value="high",
label="High",
description="High priority issue",
metadata={
"color": "orange",
"icon": "alert-triangle",
"css_class": "bg-orange-100 text-orange-800 border-orange-300",
"sort_order": 2,
"priority": 2,
},
category=ChoiceCategory.PRIORITY,
),
RichChoice(
value="medium",
label="Medium",
description="Medium priority issue",
metadata={
"color": "yellow",
"icon": "info",
"css_class": "bg-yellow-100 text-yellow-800 border-yellow-300",
"sort_order": 3,
"priority": 3,
},
category=ChoiceCategory.PRIORITY,
),
RichChoice(
value="low",
label="Low",
description="Low priority issue",
metadata={
"color": "blue",
"icon": "info",
"css_class": "bg-blue-100 text-blue-800 border-blue-300",
"sort_order": 4,
"priority": 4,
},
category=ChoiceCategory.PRIORITY,
),
]
# Extended severity levels including debug/info/warning/error for RequestMetadata
REQUEST_SEVERITY_LEVELS = [
RichChoice(
value="debug",
label="Debug",
description="Debug-level information",
metadata={
"color": "gray",
"icon": "bug",
"css_class": "bg-gray-100 text-gray-800",
"sort_order": 1,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="info",
label="Info",
description="Informational message",
metadata={
"color": "blue",
"icon": "info",
"css_class": "bg-blue-100 text-blue-800",
"sort_order": 2,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="warning",
label="Warning",
description="Warning condition",
metadata={
"color": "yellow",
"icon": "alert-triangle",
"css_class": "bg-yellow-100 text-yellow-800",
"sort_order": 3,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="error",
label="Error",
description="Error condition",
metadata={
"color": "red",
"icon": "x-circle",
"css_class": "bg-red-100 text-red-800",
"sort_order": 4,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="critical",
label="Critical",
description="Critical error requiring immediate attention",
metadata={
"color": "red",
"icon": "alert-octagon",
"css_class": "bg-red-200 text-red-900 font-bold",
"sort_order": 5,
},
category=ChoiceCategory.TECHNICAL,
),
]
# ============================================================================
# Error/Request Sources
# ============================================================================
ERROR_SOURCES = [
RichChoice(
value="frontend",
label="Frontend",
description="Error originated from frontend application",
metadata={
"color": "purple",
"icon": "monitor",
"css_class": "bg-purple-100 text-purple-800",
"sort_order": 1,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="backend",
label="Backend",
description="Error originated from backend server",
metadata={
"color": "blue",
"icon": "server",
"css_class": "bg-blue-100 text-blue-800",
"sort_order": 2,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="api",
label="API",
description="Error originated from API layer",
metadata={
"color": "green",
"icon": "code",
"css_class": "bg-green-100 text-green-800",
"sort_order": 3,
},
category=ChoiceCategory.TECHNICAL,
),
]
# ============================================================================
# System Alert Types
# ============================================================================
SYSTEM_ALERT_TYPES = [
RichChoice(
value="orphaned_images",
label="Orphaned Images",
description="Images not associated with any entity",
metadata={"color": "orange", "icon": "image", "sort_order": 1},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="stale_submissions",
label="Stale Submissions",
description="Submissions pending for too long",
metadata={"color": "yellow", "icon": "clock", "sort_order": 2},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="circular_dependency",
label="Circular Dependency",
description="Detected circular reference in data",
metadata={"color": "red", "icon": "refresh-cw", "sort_order": 3},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="validation_error",
label="Validation Error",
description="Data validation failure",
metadata={"color": "red", "icon": "alert-circle", "sort_order": 4},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="ban_attempt",
label="Ban Attempt",
description="User ban action was triggered",
metadata={"color": "red", "icon": "shield-off", "sort_order": 5},
category=ChoiceCategory.SECURITY,
),
RichChoice(
value="upload_timeout",
label="Upload Timeout",
description="File upload exceeded time limit",
metadata={"color": "orange", "icon": "upload-cloud", "sort_order": 6},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="high_error_rate",
label="High Error Rate",
description="Elevated error rate detected",
metadata={"color": "red", "icon": "trending-up", "sort_order": 7},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="database_connection",
label="Database Connection",
description="Database connectivity issue",
metadata={"color": "red", "icon": "database", "sort_order": 8},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="memory_usage",
label="Memory Usage",
description="High memory consumption detected",
metadata={"color": "orange", "icon": "cpu", "sort_order": 9},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="queue_backup",
label="Queue Backup",
description="Task queue is backing up",
metadata={"color": "yellow", "icon": "layers", "sort_order": 10},
category=ChoiceCategory.TECHNICAL,
),
]
# ============================================================================
# Metric Types for Rate Limiting
# ============================================================================
METRIC_TYPES = [
RichChoice(
value="block_rate",
label="Block Rate",
description="Percentage of requests being blocked",
metadata={"color": "red", "icon": "shield", "sort_order": 1},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="total_requests",
label="Total Requests",
description="Total number of requests",
metadata={"color": "blue", "icon": "activity", "sort_order": 2},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="unique_ips",
label="Unique IPs",
description="Number of unique IP addresses",
metadata={"color": "purple", "icon": "globe", "sort_order": 3},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="function_specific",
label="Function Specific",
description="Metrics for a specific function",
metadata={"color": "green", "icon": "code", "sort_order": 4},
category=ChoiceCategory.TECHNICAL,
),
]
# ============================================================================
# Incident Statuses
# ============================================================================
INCIDENT_STATUSES = [
RichChoice(
value="open",
label="Open",
description="Incident is open and awaiting investigation",
metadata={
"color": "red",
"icon": "alert-circle",
"css_class": "bg-red-100 text-red-800",
"sort_order": 1,
"is_active": True,
},
category=ChoiceCategory.STATUS,
),
RichChoice(
value="investigating",
label="Investigating",
description="Incident is being actively investigated",
metadata={
"color": "yellow",
"icon": "search",
"css_class": "bg-yellow-100 text-yellow-800",
"sort_order": 2,
"is_active": True,
},
category=ChoiceCategory.STATUS,
),
RichChoice(
value="resolved",
label="Resolved",
description="Incident has been resolved",
metadata={
"color": "green",
"icon": "check-circle",
"css_class": "bg-green-100 text-green-800",
"sort_order": 3,
"is_active": False,
},
category=ChoiceCategory.STATUS,
),
RichChoice(
value="closed",
label="Closed",
description="Incident is closed",
metadata={
"color": "gray",
"icon": "x-circle",
"css_class": "bg-gray-100 text-gray-800",
"sort_order": 4,
"is_active": False,
},
category=ChoiceCategory.STATUS,
),
]
# ============================================================================
# Alert Sources
# ============================================================================
ALERT_SOURCES = [
RichChoice(
value="system",
label="System Alert",
description="Alert from system monitoring",
metadata={"color": "blue", "icon": "server", "sort_order": 1},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="rate_limit",
label="Rate Limit Alert",
description="Alert from rate limiting system",
metadata={"color": "orange", "icon": "shield", "sort_order": 2},
category=ChoiceCategory.TECHNICAL,
),
]
# ============================================================================
# Pipeline Error Severities
# ============================================================================
PIPELINE_ERROR_SEVERITIES = [
RichChoice(
value="critical",
label="Critical",
description="Critical pipeline failure requiring immediate attention",
metadata={
"color": "red",
"icon": "alert-octagon",
"css_class": "bg-red-100 text-red-800 border-red-300",
"sort_order": 1,
"priority": 1,
},
category=ChoiceCategory.PRIORITY,
),
RichChoice(
value="error",
label="Error",
description="Pipeline error that needs investigation",
metadata={
"color": "orange",
"icon": "alert-triangle",
"css_class": "bg-orange-100 text-orange-800 border-orange-300",
"sort_order": 2,
"priority": 2,
},
category=ChoiceCategory.PRIORITY,
),
RichChoice(
value="warning",
label="Warning",
description="Pipeline warning that may need attention",
metadata={
"color": "yellow",
"icon": "alert-circle",
"css_class": "bg-yellow-100 text-yellow-800 border-yellow-300",
"sort_order": 3,
"priority": 3,
},
category=ChoiceCategory.PRIORITY,
),
RichChoice(
value="info",
label="Info",
description="Informational pipeline event",
metadata={
"color": "blue",
"icon": "info",
"css_class": "bg-blue-100 text-blue-800 border-blue-300",
"sort_order": 4,
"priority": 4,
},
category=ChoiceCategory.PRIORITY,
),
]
# ============================================================================
# Anomaly Types
# ============================================================================
ANOMALY_TYPES = [
RichChoice(
value="spike",
label="Spike",
description="Sudden increase in metric value",
metadata={
"color": "red",
"icon": "trending-up",
"css_class": "bg-red-100 text-red-800",
"sort_order": 1,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="drop",
label="Drop",
description="Sudden decrease in metric value",
metadata={
"color": "blue",
"icon": "trending-down",
"css_class": "bg-blue-100 text-blue-800",
"sort_order": 2,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="trend_change",
label="Trend Change",
description="Change in the overall trend direction",
metadata={
"color": "yellow",
"icon": "activity",
"css_class": "bg-yellow-100 text-yellow-800",
"sort_order": 3,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="outlier",
label="Outlier",
description="Value outside normal distribution",
metadata={
"color": "purple",
"icon": "git-branch",
"css_class": "bg-purple-100 text-purple-800",
"sort_order": 4,
},
category=ChoiceCategory.TECHNICAL,
),
RichChoice(
value="threshold_breach",
label="Threshold Breach",
description="Value exceeded configured threshold",
metadata={
"color": "orange",
"icon": "alert-triangle",
"css_class": "bg-orange-100 text-orange-800",
"sort_order": 5,
},
category=ChoiceCategory.TECHNICAL,
),
]
# ============================================================================
# Cleanup Job Statuses
# ============================================================================
CLEANUP_JOB_STATUSES = [
RichChoice(
value="success",
label="Success",
description="Cleanup job completed successfully",
metadata={
"color": "green",
"icon": "check-circle",
"css_class": "bg-green-100 text-green-800",
"sort_order": 1,
},
category=ChoiceCategory.STATUS,
),
RichChoice(
value="failed",
label="Failed",
description="Cleanup job failed with errors",
metadata={
"color": "red",
"icon": "x-circle",
"css_class": "bg-red-100 text-red-800",
"sort_order": 2,
},
category=ChoiceCategory.STATUS,
),
RichChoice(
value="partial",
label="Partial",
description="Cleanup job completed with some failures",
metadata={
"color": "yellow",
"icon": "alert-circle",
"css_class": "bg-yellow-100 text-yellow-800",
"sort_order": 3,
},
category=ChoiceCategory.STATUS,
),
RichChoice(
value="skipped",
label="Skipped",
description="Cleanup job was skipped",
metadata={
"color": "gray",
"icon": "skip-forward",
"css_class": "bg-gray-100 text-gray-800",
"sort_order": 4,
},
category=ChoiceCategory.STATUS,
),
]
# ============================================================================
# Date Precision (shared across multiple domains)
# ============================================================================
DATE_PRECISION = [
RichChoice(
value="exact",
label="Exact Date",
description="Date is known exactly",
metadata={"color": "green", "icon": "calendar", "sort_order": 1, "format": "YYYY-MM-DD"},
category=ChoiceCategory.CLASSIFICATION,
),
RichChoice(
value="month",
label="Month and Year",
description="Only month and year are known",
metadata={"color": "blue", "icon": "calendar", "sort_order": 2, "format": "YYYY-MM"},
category=ChoiceCategory.CLASSIFICATION,
),
RichChoice(
value="year",
label="Year Only",
description="Only the year is known",
metadata={"color": "yellow", "icon": "calendar", "sort_order": 3, "format": "YYYY"},
category=ChoiceCategory.CLASSIFICATION,
),
RichChoice(
value="decade",
label="Decade",
description="Only the decade is known",
metadata={"color": "orange", "icon": "calendar", "sort_order": 4, "format": "YYYYs"},
category=ChoiceCategory.CLASSIFICATION,
),
RichChoice(
value="century",
label="Century",
description="Only the century is known",
metadata={"color": "gray", "icon": "calendar", "sort_order": 5, "format": "YYc"},
category=ChoiceCategory.CLASSIFICATION,
),
RichChoice(
value="approximate",
label="Approximate",
description="Date is approximate/estimated",
metadata={"color": "gray", "icon": "help-circle", "sort_order": 6, "format": "~YYYY"},
category=ChoiceCategory.CLASSIFICATION,
),
]
def register_core_choices():
"""Register all core system choices with the global registry"""
@@ -152,6 +731,95 @@ def register_core_choices():
metadata={"domain": "core", "type": "entity_type"},
)
register_choices(
name="severity_levels",
choices=SEVERITY_LEVELS,
domain="core",
description="Severity levels for errors and alerts",
metadata={"domain": "core", "type": "severity"},
)
register_choices(
name="request_severity_levels",
choices=REQUEST_SEVERITY_LEVELS,
domain="core",
description="Extended severity levels for request metadata",
metadata={"domain": "core", "type": "request_severity"},
)
register_choices(
name="error_sources",
choices=ERROR_SOURCES,
domain="core",
description="Sources of application errors",
metadata={"domain": "core", "type": "error_source"},
)
register_choices(
name="system_alert_types",
choices=SYSTEM_ALERT_TYPES,
domain="core",
description="Types of system alerts",
metadata={"domain": "core", "type": "alert_type"},
)
register_choices(
name="metric_types",
choices=METRIC_TYPES,
domain="core",
description="Types of rate limit metrics",
metadata={"domain": "core", "type": "metric_type"},
)
register_choices(
name="incident_statuses",
choices=INCIDENT_STATUSES,
domain="core",
description="Incident status options",
metadata={"domain": "core", "type": "incident_status"},
)
register_choices(
name="alert_sources",
choices=ALERT_SOURCES,
domain="core",
description="Sources of alerts",
metadata={"domain": "core", "type": "alert_source"},
)
register_choices(
name="pipeline_error_severities",
choices=PIPELINE_ERROR_SEVERITIES,
domain="core",
description="Severity levels for pipeline errors",
metadata={"domain": "core", "type": "pipeline_error_severity"},
)
register_choices(
name="anomaly_types",
choices=ANOMALY_TYPES,
domain="core",
description="Types of detected anomalies",
metadata={"domain": "core", "type": "anomaly_type"},
)
register_choices(
name="cleanup_job_statuses",
choices=CLEANUP_JOB_STATUSES,
domain="core",
description="Status options for cleanup jobs",
metadata={"domain": "core", "type": "cleanup_job_status"},
)
register_choices(
name="date_precision",
choices=DATE_PRECISION,
domain="core",
description="Date precision options",
metadata={"domain": "core", "type": "date_precision"},
)
# Auto-register choices when module is imported
register_core_choices()

133
backend/apps/core/choices/filters.py Normal file
View File

@@ -0,0 +1,133 @@
"""
Django-filter Integration for Rich Choices
This module provides django-filter compatible filter classes that integrate
with the RichChoice registry system.
"""
from typing import Any
from django_filters import ChoiceFilter, MultipleChoiceFilter
from .registry import registry
class RichChoiceFilter(ChoiceFilter):
"""
Django-filter ChoiceFilter that uses the RichChoice registry.
This is the REQUIRED replacement for ChoiceFilter with inline choices.
Usage:
class MyFilterSet(django_filters.FilterSet):
status = RichChoiceFilter(
choice_group="ticket_statuses",
domain="support",
)
"""
def __init__(
self,
choice_group: str,
domain: str = "core",
allow_deprecated: bool = False,
**kwargs
):
"""
Initialize the filter.
Args:
choice_group: Name of the choice group in the registry
domain: Domain namespace for the choice group
allow_deprecated: Whether to include deprecated choices
**kwargs: Additional arguments passed to ChoiceFilter
"""
self.choice_group = choice_group
self.domain = domain
self.allow_deprecated = allow_deprecated
# Get choices from registry
if allow_deprecated:
choices_list = registry.get_choices(choice_group, domain)
else:
choices_list = registry.get_active_choices(choice_group, domain)
choices = [(c.value, c.label) for c in choices_list]
super().__init__(choices=choices, **kwargs)
class RichMultipleChoiceFilter(MultipleChoiceFilter):
"""
Django-filter MultipleChoiceFilter that uses the RichChoice registry.
This is the REQUIRED replacement for MultipleChoiceFilter with inline choices.
Usage:
class MyFilterSet(django_filters.FilterSet):
statuses = RichMultipleChoiceFilter(
choice_group="ticket_statuses",
domain="support",
field_name="status",
)
"""
def __init__(
self,
choice_group: str,
domain: str = "core",
allow_deprecated: bool = False,
**kwargs
):
"""
Initialize the filter.
Args:
choice_group: Name of the choice group in the registry
domain: Domain namespace for the choice group
allow_deprecated: Whether to include deprecated choices
**kwargs: Additional arguments passed to MultipleChoiceFilter
"""
self.choice_group = choice_group
self.domain = domain
self.allow_deprecated = allow_deprecated
# Get choices from registry
if allow_deprecated:
choices_list = registry.get_choices(choice_group, domain)
else:
choices_list = registry.get_active_choices(choice_group, domain)
choices = [(c.value, c.label) for c in choices_list]
super().__init__(choices=choices, **kwargs)
def get_choice_filter_class(
choice_group: str,
domain: str = "core",
allow_deprecated: bool = False,
**extra_kwargs: Any
) -> type[RichChoiceFilter]:
"""
Factory function to create a RichChoiceFilter class with preset choices.
Useful when you need to define the filter class dynamically or
when the choice_group/domain aren't available at class definition time.
Usage:
StatusFilter = get_choice_filter_class("ticket_statuses", "support")
class MyFilterSet(django_filters.FilterSet):
status = StatusFilter()
"""
class DynamicRichChoiceFilter(RichChoiceFilter):
def __init__(self, **kwargs):
kwargs.setdefault("choice_group", choice_group)
kwargs.setdefault("domain", domain)
kwargs.setdefault("allow_deprecated", allow_deprecated)
for key, value in extra_kwargs.items():
kwargs.setdefault(key, value)
super().__init__(**kwargs)
return DynamicRichChoiceFilter

95
backend/apps/core/choices/serializers.py
View File

@@ -265,3 +265,98 @@ def serialize_choice_value(value: str, choice_group: str, domain: str = "core",
}
else:
return value
class RichChoiceSerializerField(serializers.ChoiceField):
"""
DRF serializer field for RichChoice values.
This field validates input against the RichChoice registry and provides
type-safe choice handling with proper error messages. It is the REQUIRED
replacement for serializers.ChoiceField with inline choices.
Usage:
class MySerializer(serializers.Serializer):
status = RichChoiceSerializerField(
choice_group="ticket_statuses",
domain="support",
)
# With rich metadata in output
severity = RichChoiceSerializerField(
choice_group="severity_levels",
domain="core",
include_metadata=True,
)
"""
def __init__(
self,
choice_group: str,
domain: str = "core",
include_metadata: bool = False,
allow_deprecated: bool = False,
**kwargs
):
"""
Initialize the serializer field.
Args:
choice_group: Name of the choice group in the registry
domain: Domain namespace for the choice group
include_metadata: Whether to include rich choice metadata in output
allow_deprecated: Whether to allow deprecated choices
**kwargs: Additional arguments passed to ChoiceField
"""
self.choice_group = choice_group
self.domain = domain
self.include_metadata = include_metadata
self.allow_deprecated = allow_deprecated
# Get choices from registry for validation
if allow_deprecated:
choices_list = registry.get_choices(choice_group, domain)
else:
choices_list = registry.get_active_choices(choice_group, domain)
# Build choices tuple for DRF ChoiceField
choices = [(c.value, c.label) for c in choices_list]
# Store valid values for error messages
self._valid_values = [c.value for c in choices_list]
super().__init__(choices=choices, **kwargs)
def to_representation(self, value: str) -> Any:
"""Convert choice value to representation."""
if not value:
return value
if self.include_metadata:
return serialize_choice_value(
value,
self.choice_group,
self.domain,
include_metadata=True
)
return value
def to_internal_value(self, data: Any) -> str:
"""Convert input data to choice value."""
# Handle rich choice object input (value dict)
if isinstance(data, dict) and "value" in data:
data = data["value"]
# Validate and return
return super().to_internal_value(data)
def fail(self, key: str, **kwargs: Any) -> None:
"""Provide better error messages with valid choices listed."""
if key == "invalid_choice":
valid_choices = ", ".join(self._valid_values)
raise serializers.ValidationError(
f"'{kwargs.get('input', '')}' is not a valid choice for {self.choice_group}. "
f"Valid choices are: {valid_choices}"
)
super().fail(key, **kwargs)

69
backend/apps/core/migrations/0011_alter_applicationerror_severity_and_more.py Normal file
View File

@@ -0,0 +1,69 @@
# Generated by Django 5.2.10 on 2026-01-11 00:48
import apps.core.choices.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('core', '0010_add_milestone_model'),
]
operations = [
migrations.AlterField(
model_name='applicationerror',
name='severity',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='severity_levels', choices=[('critical', 'Critical'), ('high', 'High'), ('medium', 'Medium'), ('low', 'Low')], db_index=True, default='medium', domain='core', help_text='Error severity level', max_length=20),
),
migrations.AlterField(
model_name='applicationerror',
name='source',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='error_sources', choices=[('frontend', 'Frontend'), ('backend', 'Backend'), ('api', 'API')], db_index=True, domain='core', help_text='Where the error originated', max_length=20),
),
migrations.AlterField(
model_name='incident',
name='severity',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='severity_levels', choices=[('critical', 'Critical'), ('high', 'High'), ('medium', 'Medium'), ('low', 'Low')], db_index=True, domain='core', help_text='Incident severity level', max_length=20),
),
migrations.AlterField(
model_name='incident',
name='status',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='incident_statuses', choices=[('open', 'Open'), ('investigating', 'Investigating'), ('resolved', 'Resolved'), ('closed', 'Closed')], db_index=True, default='open', domain='core', help_text='Current incident status', max_length=20),
),
migrations.AlterField(
model_name='incidentalert',
name='alert_source',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='alert_sources', choices=[('system', 'System Alert'), ('rate_limit', 'Rate Limit Alert')], domain='core', help_text='Source type of the alert', max_length=20),
),
migrations.AlterField(
model_name='milestone',
name='event_date_precision',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='date_precision', choices=[('exact', 'Exact Date'), ('month', 'Month and Year'), ('year', 'Year Only'), ('decade', 'Decade'), ('century', 'Century'), ('approximate', 'Approximate')], default='exact', domain='core', help_text='Precision of the event date', max_length=20),
),
migrations.AlterField(
model_name='milestoneevent',
name='event_date_precision',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='date_precision', choices=[('exact', 'Exact Date'), ('month', 'Month and Year'), ('year', 'Year Only'), ('decade', 'Decade'), ('century', 'Century'), ('approximate', 'Approximate')], default='exact', domain='core', help_text='Precision of the event date', max_length=20),
),
migrations.AlterField(
model_name='ratelimitalertconfig',
name='metric_type',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='metric_types', choices=[('block_rate', 'Block Rate'), ('total_requests', 'Total Requests'), ('unique_ips', 'Unique IPs'), ('function_specific', 'Function Specific')], db_index=True, domain='core', help_text='Type of metric to monitor', max_length=50),
),
migrations.AlterField(
model_name='requestmetadata',
name='severity',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='request_severity_levels', choices=[('debug', 'Debug'), ('info', 'Info'), ('warning', 'Warning'), ('error', 'Error'), ('critical', 'Critical')], db_index=True, default='info', domain='core', help_text='Error severity level', max_length=20),
),
migrations.AlterField(
model_name='systemalert',
name='alert_type',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='system_alert_types', choices=[('orphaned_images', 'Orphaned Images'), ('stale_submissions', 'Stale Submissions'), ('circular_dependency', 'Circular Dependency'), ('validation_error', 'Validation Error'), ('ban_attempt', 'Ban Attempt'), ('upload_timeout', 'Upload Timeout'), ('high_error_rate', 'High Error Rate'), ('database_connection', 'Database Connection'), ('memory_usage', 'Memory Usage'), ('queue_backup', 'Queue Backup')], db_index=True, domain='core', help_text='Type of system alert', max_length=50),
),
migrations.AlterField(
model_name='systemalert',
name='severity',
field=apps.core.choices.fields.RichChoiceField(allow_deprecated=False, choice_group='severity_levels', choices=[('critical', 'Critical'), ('high', 'High'), ('medium', 'Medium'), ('low', 'Low')], db_index=True, domain='core', help_text='Alert severity level', max_length=20),
),
]

320
backend/apps/core/migrations/0012_alertcorrelationrule_cleanupjoblog_anomaly_and_more.py Normal file
View File

@@ -0,0 +1,320 @@
# Generated by Django 5.2.10 on 2026-01-11 18:06
import apps.core.choices.fields
import django.db.models.deletion
import uuid
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("core", "0011_alter_applicationerror_severity_and_more"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name="AlertCorrelationRule",
fields=[
("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
(
"rule_name",
models.CharField(
db_index=True, help_text="Unique name for this correlation rule", max_length=255, unique=True
),
),
(
"rule_description",
models.TextField(blank=True, help_text="Description of what this rule correlates"),
),
(
"min_alerts_required",
models.PositiveIntegerField(
default=3, help_text="Minimum number of alerts needed to trigger correlation"
),
),
(
"time_window_minutes",
models.PositiveIntegerField(default=30, help_text="Time window in minutes for alert correlation"),
),
(
"incident_severity",
apps.core.choices.fields.RichChoiceField(
allow_deprecated=False,
choice_group="severity_levels",
choices=[("critical", "Critical"), ("high", "High"), ("medium", "Medium"), ("low", "Low")],
default="medium",
domain="core",
help_text="Severity to assign to correlated incidents",
max_length=20,
),
),
(
"incident_title_template",
models.CharField(
help_text="Template for incident title (supports {count}, {rule_name})", max_length=255
),
),
(
"is_active",
models.BooleanField(db_index=True, default=True, help_text="Whether this rule is currently active"),
),
("created_at", models.DateTimeField(auto_now_add=True, help_text="When this rule was created")),
("updated_at", models.DateTimeField(auto_now=True, help_text="When this rule was last updated")),
],
options={
"verbose_name": "Alert Correlation Rule",
"verbose_name_plural": "Alert Correlation Rules",
"ordering": ["rule_name"],
},
),
migrations.CreateModel(
name="CleanupJobLog",
fields=[
("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
("job_name", models.CharField(db_index=True, help_text="Name of the cleanup job", max_length=255)),
(
"status",
apps.core.choices.fields.RichChoiceField(
allow_deprecated=False,
choice_group="cleanup_job_statuses",
choices=[
("success", "Success"),
("failed", "Failed"),
("partial", "Partial"),
("skipped", "Skipped"),
],
db_index=True,
default="success",
domain="core",
help_text="Execution status",
max_length=20,
),
),
("records_processed", models.PositiveIntegerField(default=0, help_text="Number of records processed")),
("records_deleted", models.PositiveIntegerField(default=0, help_text="Number of records deleted")),
("error_message", models.TextField(blank=True, help_text="Error message if job failed", null=True)),
(
"duration_ms",
models.PositiveIntegerField(blank=True, help_text="Execution duration in milliseconds", null=True),
),
(
"executed_at",
models.DateTimeField(auto_now_add=True, db_index=True, help_text="When this job was executed"),
),
],
options={
"verbose_name": "Cleanup Job Log",
"verbose_name_plural": "Cleanup Job Logs",
"ordering": ["-executed_at"],
"indexes": [
models.Index(fields=["job_name", "executed_at"], name="core_cleanu_job_nam_4530fd_idx"),
models.Index(fields=["status", "executed_at"], name="core_cleanu_status_fa6360_idx"),
],
},
),
migrations.CreateModel(
name="Anomaly",
fields=[
("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
(
"metric_name",
models.CharField(
db_index=True, help_text="Name of the metric that exhibited anomalous behavior", max_length=255
),
),
(
"metric_category",
models.CharField(
db_index=True,
help_text="Category of the metric (e.g., 'performance', 'error_rate', 'traffic')",
max_length=100,
),
),
(
"anomaly_type",
apps.core.choices.fields.RichChoiceField(
allow_deprecated=False,
choice_group="anomaly_types",
choices=[
("spike", "Spike"),
("drop", "Drop"),
("trend_change", "Trend Change"),
("outlier", "Outlier"),
("threshold_breach", "Threshold Breach"),
],
db_index=True,
domain="core",
help_text="Type of anomaly detected",
max_length=30,
),
),
(
"severity",
apps.core.choices.fields.RichChoiceField(
allow_deprecated=False,
choice_group="severity_levels",
choices=[("critical", "Critical"), ("high", "High"), ("medium", "Medium"), ("low", "Low")],
db_index=True,
domain="core",
help_text="Severity of the anomaly",
max_length=20,
),
),
(
"anomaly_value",
models.DecimalField(decimal_places=6, help_text="The anomalous value detected", max_digits=20),
),
(
"baseline_value",
models.DecimalField(decimal_places=6, help_text="The expected baseline value", max_digits=20),
),
(
"deviation_score",
models.DecimalField(decimal_places=4, help_text="Standard deviations from normal", max_digits=10),
),
(
"confidence_score",
models.DecimalField(
decimal_places=4, help_text="Confidence score of the detection (0-1)", max_digits=5
),
),
("detection_algorithm", models.CharField(help_text="Algorithm used for detection", max_length=100)),
("time_window_start", models.DateTimeField(help_text="Start of the detection time window")),
("time_window_end", models.DateTimeField(help_text="End of the detection time window")),
(
"alert_created",
models.BooleanField(
db_index=True, default=False, help_text="Whether an alert was created for this anomaly"
),
),
(
"detected_at",
models.DateTimeField(auto_now_add=True, db_index=True, help_text="When this anomaly was detected"),
),
(
"alert",
models.ForeignKey(
blank=True,
help_text="Linked system alert if created",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="anomalies",
to="core.systemalert",
),
),
],
options={
"verbose_name": "Anomaly",
"verbose_name_plural": "Anomalies",
"ordering": ["-detected_at"],
"indexes": [
models.Index(fields=["metric_name", "detected_at"], name="core_anomal_metric__06c3c9_idx"),
models.Index(fields=["severity", "detected_at"], name="core_anomal_severit_ea7a17_idx"),
models.Index(fields=["anomaly_type", "detected_at"], name="core_anomal_anomaly_eb45f7_idx"),
models.Index(fields=["alert_created", "detected_at"], name="core_anomal_alert_c_5a0c1a_idx"),
],
},
),
migrations.CreateModel(
name="PipelineError",
fields=[
("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
(
"function_name",
models.CharField(
db_index=True, help_text="Name of the function/pipeline that failed", max_length=255
),
),
("error_message", models.TextField(help_text="Error message describing the failure")),
(
"error_code",
models.CharField(
blank=True, db_index=True, help_text="Error code for categorization", max_length=100, null=True
),
),
("error_context", models.JSONField(blank=True, help_text="Additional context data as JSON", null=True)),
("stack_trace", models.TextField(blank=True, help_text="Full stack trace for debugging", null=True)),
(
"severity",
apps.core.choices.fields.RichChoiceField(
allow_deprecated=False,
choice_group="pipeline_error_severities",
choices=[
("critical", "Critical"),
("error", "Error"),
("warning", "Warning"),
("info", "Info"),
],
db_index=True,
default="error",
domain="core",
help_text="Severity level of the error",
max_length=20,
),
),
(
"submission_id",
models.UUIDField(
blank=True, db_index=True, help_text="ID of related content submission if applicable", null=True
),
),
(
"item_id",
models.CharField(
blank=True,
db_index=True,
help_text="Generic reference to related item",
max_length=255,
null=True,
),
),
(
"request_id",
models.UUIDField(blank=True, db_index=True, help_text="Request ID for correlation", null=True),
),
("trace_id", models.UUIDField(blank=True, db_index=True, help_text="Distributed trace ID", null=True)),
(
"resolved",
models.BooleanField(db_index=True, default=False, help_text="Whether this error has been resolved"),
),
(
"resolved_at",
models.DateTimeField(
blank=True, db_index=True, help_text="When this error was resolved", null=True
),
),
(
"resolution_notes",
models.TextField(blank=True, help_text="Notes about how the error was resolved", null=True),
),
(
"occurred_at",
models.DateTimeField(auto_now_add=True, db_index=True, help_text="When this error occurred"),
),
(
"resolved_by",
models.ForeignKey(
blank=True,
help_text="User who resolved this error",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="resolved_pipeline_errors",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"verbose_name": "Pipeline Error",
"verbose_name_plural": "Pipeline Errors",
"ordering": ["-occurred_at"],
"indexes": [
models.Index(fields=["severity", "occurred_at"], name="core_pipeli_severit_9c8037_idx"),
models.Index(fields=["function_name", "occurred_at"], name="core_pipeli_functio_efb015_idx"),
models.Index(fields=["resolved", "occurred_at"], name="core_pipeli_resolve_cd60c5_idx"),
],
},
),
]

479
backend/apps/core/models.py
View File

@@ -8,8 +8,12 @@ from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.text import slugify
from apps.core.choices.fields import RichChoiceField
from apps.core.history import TrackedModel
# Import choices module to ensure registration on app load
from apps.core.choices import core_choices # noqa: F401
@pghistory.track()
class SlugHistory(models.Model):
@@ -136,17 +140,6 @@ class ApplicationError(models.Model):
reported via API (frontend) and displayed in the admin dashboard.
"""
class Severity(models.TextChoices):
CRITICAL = "critical", "Critical"
HIGH = "high", "High"
MEDIUM = "medium", "Medium"
LOW = "low", "Low"
class Source(models.TextChoices):
FRONTEND = "frontend", "Frontend"
BACKEND = "backend", "Backend"
API = "api", "API"
# Identity
error_id = models.UUIDField(
unique=True,
@@ -180,16 +173,18 @@ class ApplicationError(models.Model):
db_index=True,
help_text="Application-specific error code",
)
severity = models.CharField(
severity = RichChoiceField(
choice_group="severity_levels",
domain="core",
max_length=20,
choices=Severity.choices,
default=Severity.MEDIUM,
default="medium",
db_index=True,
help_text="Error severity level",
)
source = models.CharField(
source = RichChoiceField(
choice_group="error_sources",
domain="core",
max_length=20,
choices=Source.choices,
db_index=True,
help_text="Where the error originated",
)
@@ -308,34 +303,18 @@ class SystemAlert(models.Model):
validation errors, ban attempts, upload timeouts, and high error rates.
"""
class AlertType(models.TextChoices):
ORPHANED_IMAGES = "orphaned_images", "Orphaned Images"
STALE_SUBMISSIONS = "stale_submissions", "Stale Submissions"
CIRCULAR_DEPENDENCY = "circular_dependency", "Circular Dependency"
VALIDATION_ERROR = "validation_error", "Validation Error"
BAN_ATTEMPT = "ban_attempt", "Ban Attempt"
UPLOAD_TIMEOUT = "upload_timeout", "Upload Timeout"
HIGH_ERROR_RATE = "high_error_rate", "High Error Rate"
DATABASE_CONNECTION = "database_connection", "Database Connection"
MEMORY_USAGE = "memory_usage", "Memory Usage"
QUEUE_BACKUP = "queue_backup", "Queue Backup"
class Severity(models.TextChoices):
LOW = "low", "Low"
MEDIUM = "medium", "Medium"
HIGH = "high", "High"
CRITICAL = "critical", "Critical"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
alert_type = models.CharField(
alert_type = RichChoiceField(
choice_group="system_alert_types",
domain="core",
max_length=50,
choices=AlertType.choices,
db_index=True,
help_text="Type of system alert",
)
severity = models.CharField(
severity = RichChoiceField(
choice_group="severity_levels",
domain="core",
max_length=20,
choices=Severity.choices,
db_index=True,
help_text="Alert severity level",
)
@@ -386,16 +365,11 @@ class RateLimitAlertConfig(models.Model):
Defines thresholds that trigger alerts when exceeded.
"""
class MetricType(models.TextChoices):
BLOCK_RATE = "block_rate", "Block Rate"
TOTAL_REQUESTS = "total_requests", "Total Requests"
UNIQUE_IPS = "unique_ips", "Unique IPs"
FUNCTION_SPECIFIC = "function_specific", "Function Specific"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
metric_type = models.CharField(
metric_type = RichChoiceField(
choice_group="metric_types",
domain="core",
max_length=50,
choices=MetricType.choices,
db_index=True,
help_text="Type of metric to monitor",
)
@@ -484,18 +458,6 @@ class Incident(models.Model):
allowing teams to track and resolve related alerts together.
"""
class Status(models.TextChoices):
OPEN = "open", "Open"
INVESTIGATING = "investigating", "Investigating"
RESOLVED = "resolved", "Resolved"
CLOSED = "closed", "Closed"
class Severity(models.TextChoices):
LOW = "low", "Low"
MEDIUM = "medium", "Medium"
HIGH = "high", "High"
CRITICAL = "critical", "Critical"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
incident_number = models.CharField(
max_length=20,
@@ -505,16 +467,18 @@ class Incident(models.Model):
)
title = models.CharField(max_length=255, help_text="Brief description of the incident")
description = models.TextField(null=True, blank=True, help_text="Detailed description")
severity = models.CharField(
severity = RichChoiceField(
choice_group="severity_levels",
domain="core",
max_length=20,
choices=Severity.choices,
db_index=True,
help_text="Incident severity level",
)
status = models.CharField(
status = RichChoiceField(
choice_group="incident_statuses",
domain="core",
max_length=20,
choices=Status.choices,
default=Status.OPEN,
default="open",
db_index=True,
help_text="Current incident status",
)
@@ -582,10 +546,6 @@ class IncidentAlert(models.Model):
Supports linking both system alerts and rate limit alerts.
"""
class AlertSource(models.TextChoices):
SYSTEM = "system", "System Alert"
RATE_LIMIT = "rate_limit", "Rate Limit Alert"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
incident = models.ForeignKey(
Incident,
@@ -593,9 +553,10 @@ class IncidentAlert(models.Model):
related_name="linked_alerts",
help_text="The incident this alert is linked to",
)
alert_source = models.CharField(
alert_source = RichChoiceField(
choice_group="alert_sources",
domain="core",
max_length=20,
choices=AlertSource.choices,
help_text="Source type of the alert",
)
alert_id = models.UUIDField(help_text="ID of the linked alert")
@@ -633,13 +594,6 @@ class RequestMetadata(models.Model):
dashboard for error monitoring and analytics.
"""
class Severity(models.TextChoices):
DEBUG = "debug", "Debug"
INFO = "info", "Info"
WARNING = "warning", "Warning"
ERROR = "error", "Error"
CRITICAL = "critical", "Critical"
# Identity & Correlation
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
request_id = models.CharField(
@@ -789,10 +743,11 @@ class RequestMetadata(models.Model):
null=True,
help_text="React component stack trace",
)
severity = models.CharField(
severity = RichChoiceField(
choice_group="request_severity_levels",
domain="core",
max_length=20,
choices=Severity.choices,
default=Severity.INFO,
default="info",
db_index=True,
help_text="Error severity level",
)
@@ -1062,14 +1017,6 @@ class Milestone(TrackedModel):
Maps to frontend milestoneValidationSchema in entityValidationSchemas.ts
"""
class DatePrecision(models.TextChoices):
EXACT = "exact", "Exact Date"
MONTH = "month", "Month and Year"
YEAR = "year", "Year Only"
DECADE = "decade", "Decade"
CENTURY = "century", "Century"
APPROXIMATE = "approximate", "Approximate"
# Core event information
title = models.CharField(
max_length=200,
@@ -1088,10 +1035,11 @@ class Milestone(TrackedModel):
db_index=True,
help_text="Date when the event occurred or will occur",
)
event_date_precision = models.CharField(
event_date_precision = RichChoiceField(
choice_group="date_precision",
domain="core",
max_length=20,
choices=DatePrecision.choices,
default=DatePrecision.EXACT,
default="exact",
help_text="Precision of the event date",
)
@@ -1161,3 +1109,354 @@ class Milestone(TrackedModel):
def __str__(self) -> str:
return f"{self.title} ({self.event_date})"
class PipelineError(models.Model):
"""
Tracks pipeline/processing errors for debugging and monitoring.
Records errors that occur during data processing pipelines,
approval workflows, and other automated operations. Supports
resolution tracking and filtering by severity, function, and date.
"""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
function_name = models.CharField(
max_length=255,
db_index=True,
help_text="Name of the function/pipeline that failed",
)
error_message = models.TextField(
help_text="Error message describing the failure",
)
error_code = models.CharField(
max_length=100,
blank=True,
null=True,
db_index=True,
help_text="Error code for categorization",
)
error_context = models.JSONField(
blank=True,
null=True,
help_text="Additional context data as JSON",
)
stack_trace = models.TextField(
blank=True,
null=True,
help_text="Full stack trace for debugging",
)
severity = RichChoiceField(
choice_group="pipeline_error_severities",
domain="core",
max_length=20,
default="error",
db_index=True,
help_text="Severity level of the error",
)
# References
submission_id = models.UUIDField(
blank=True,
null=True,
db_index=True,
help_text="ID of related content submission if applicable",
)
item_id = models.CharField(
max_length=255,
blank=True,
null=True,
db_index=True,
help_text="Generic reference to related item",
)
request_id = models.UUIDField(
blank=True,
null=True,
db_index=True,
help_text="Request ID for correlation",
)
trace_id = models.UUIDField(
blank=True,
null=True,
db_index=True,
help_text="Distributed trace ID",
)
# Resolution
resolved = models.BooleanField(
default=False,
db_index=True,
help_text="Whether this error has been resolved",
)
resolved_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
blank=True,
null=True,
on_delete=models.SET_NULL,
related_name="resolved_pipeline_errors",
help_text="User who resolved this error",
)
resolved_at = models.DateTimeField(
blank=True,
null=True,
db_index=True,
help_text="When this error was resolved",
)
resolution_notes = models.TextField(
blank=True,
null=True,
help_text="Notes about how the error was resolved",
)
# Timestamps
occurred_at = models.DateTimeField(
auto_now_add=True,
db_index=True,
help_text="When this error occurred",
)
class Meta:
ordering = ["-occurred_at"]
verbose_name = "Pipeline Error"
verbose_name_plural = "Pipeline Errors"
indexes = [
models.Index(fields=["severity", "occurred_at"]),
models.Index(fields=["function_name", "occurred_at"]),
models.Index(fields=["resolved", "occurred_at"]),
]
def __str__(self) -> str:
return f"[{self.get_severity_display()}] {self.function_name}: {self.error_message[:50]}"
class Anomaly(models.Model):
"""
Records detected anomalies in system metrics.
Anomalies are identified by detection algorithms and stored
for analysis and alerting. Each anomaly includes the metric
details, deviation scores, and optional links to created alerts.
"""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
metric_name = models.CharField(
max_length=255,
db_index=True,
help_text="Name of the metric that exhibited anomalous behavior",
)
metric_category = models.CharField(
max_length=100,
db_index=True,
help_text="Category of the metric (e.g., 'performance', 'error_rate', 'traffic')",
)
anomaly_type = RichChoiceField(
choice_group="anomaly_types",
domain="core",
max_length=30,
db_index=True,
help_text="Type of anomaly detected",
)
severity = RichChoiceField(
choice_group="severity_levels",
domain="core",
max_length=20,
db_index=True,
help_text="Severity of the anomaly",
)
# Metric values
anomaly_value = models.DecimalField(
max_digits=20,
decimal_places=6,
help_text="The anomalous value detected",
)
baseline_value = models.DecimalField(
max_digits=20,
decimal_places=6,
help_text="The expected baseline value",
)
deviation_score = models.DecimalField(
max_digits=10,
decimal_places=4,
help_text="Standard deviations from normal",
)
confidence_score = models.DecimalField(
max_digits=5,
decimal_places=4,
help_text="Confidence score of the detection (0-1)",
)
# Detection context
detection_algorithm = models.CharField(
max_length=100,
help_text="Algorithm used for detection",
)
time_window_start = models.DateTimeField(
help_text="Start of the detection time window",
)
time_window_end = models.DateTimeField(
help_text="End of the detection time window",
)
# Alert linkage
alert_created = models.BooleanField(
default=False,
db_index=True,
help_text="Whether an alert was created for this anomaly",
)
alert = models.ForeignKey(
SystemAlert,
blank=True,
null=True,
on_delete=models.SET_NULL,
related_name="anomalies",
help_text="Linked system alert if created",
)
# Timestamps
detected_at = models.DateTimeField(
auto_now_add=True,
db_index=True,
help_text="When this anomaly was detected",
)
class Meta:
ordering = ["-detected_at"]
verbose_name = "Anomaly"
verbose_name_plural = "Anomalies"
indexes = [
models.Index(fields=["metric_name", "detected_at"]),
models.Index(fields=["severity", "detected_at"]),
models.Index(fields=["anomaly_type", "detected_at"]),
models.Index(fields=["alert_created", "detected_at"]),
]
def __str__(self) -> str:
return f"[{self.get_severity_display()}] {self.metric_name}: {self.get_anomaly_type_display()}"
class AlertCorrelationRule(models.Model):
"""
Defines rules for correlating multiple alerts.
When multiple alerts match a correlation rule's pattern within
the time window, they can be grouped into an incident. This
helps reduce alert fatigue and identify related issues.
"""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
rule_name = models.CharField(
max_length=255,
unique=True,
db_index=True,
help_text="Unique name for this correlation rule",
)
rule_description = models.TextField(
blank=True,
help_text="Description of what this rule correlates",
)
min_alerts_required = models.PositiveIntegerField(
default=3,
help_text="Minimum number of alerts needed to trigger correlation",
)
time_window_minutes = models.PositiveIntegerField(
default=30,
help_text="Time window in minutes for alert correlation",
)
incident_severity = RichChoiceField(
choice_group="severity_levels",
domain="core",
max_length=20,
default="medium",
help_text="Severity to assign to correlated incidents",
)
incident_title_template = models.CharField(
max_length=255,
help_text="Template for incident title (supports {count}, {rule_name})",
)
is_active = models.BooleanField(
default=True,
db_index=True,
help_text="Whether this rule is currently active",
)
# Timestamps
created_at = models.DateTimeField(
auto_now_add=True,
help_text="When this rule was created",
)
updated_at = models.DateTimeField(
auto_now=True,
help_text="When this rule was last updated",
)
class Meta:
ordering = ["rule_name"]
verbose_name = "Alert Correlation Rule"
verbose_name_plural = "Alert Correlation Rules"
def __str__(self) -> str:
status = "active" if self.is_active else "inactive"
return f"{self.rule_name} ({status})"
class CleanupJobLog(models.Model):
"""
Audit log for cleanup and maintenance jobs.
Records the execution of background cleanup tasks,
including success/failure status, records processed,
and execution time for monitoring and debugging.
"""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
job_name = models.CharField(
max_length=255,
db_index=True,
help_text="Name of the cleanup job",
)
status = RichChoiceField(
choice_group="cleanup_job_statuses",
domain="core",
max_length=20,
default="success",
db_index=True,
help_text="Execution status",
)
records_processed = models.PositiveIntegerField(
default=0,
help_text="Number of records processed",
)
records_deleted = models.PositiveIntegerField(
default=0,
help_text="Number of records deleted",
)
error_message = models.TextField(
blank=True,
null=True,
help_text="Error message if job failed",
)
duration_ms = models.PositiveIntegerField(
blank=True,
null=True,
help_text="Execution duration in milliseconds",
)
# Timestamps
executed_at = models.DateTimeField(
auto_now_add=True,
db_index=True,
help_text="When this job was executed",
)
class Meta:
ordering = ["-executed_at"]
verbose_name = "Cleanup Job Log"
verbose_name_plural = "Cleanup Job Logs"
indexes = [
models.Index(fields=["job_name", "executed_at"]),
models.Index(fields=["status", "executed_at"]),
]
def __str__(self) -> str:
return f"{self.job_name}: {self.get_status_display()} ({self.records_processed} processed)"