Right to Deletion.

# deletion/service.py

"""
User data deletion service.

Orchestrates deletion across all systems where user data exists.
"""

from dataclasses import dataclass, field
from typing import List, Dict, Optional, Any
from datetime import datetime, timedelta, timezone
from enum import Enum
import uuid
import logging
import asyncio

logger = logging.getLogger(__name__)


class DeletionStatus(Enum):
    """Status of a deletion request."""
    PENDING = "pending"
    IN_PROGRESS = "in_progress"
    AWAITING_VERIFICATION = "awaiting_verification"
    COMPLETED = "completed"
    PARTIALLY_COMPLETED = "partially_completed"
    FAILED = "failed"
    CANCELLED = "cancelled"


class SystemDeletionStatus(Enum):
    """Status of deletion in a specific system."""
    PENDING = "pending"
    IN_PROGRESS = "in_progress"
    COMPLETED = "completed"
    FAILED = "failed"
    SKIPPED = "skipped"
    NOT_APPLICABLE = "not_applicable"


@dataclass
class DeletionTarget:
    """A system that needs deletion."""
    system_name: str
    system_type: str  # database, cache, storage, third_party, etc.
    deletion_method: str  # How to delete in this system
    priority: int  # Order of deletion (lower = first)
    status: SystemDeletionStatus = SystemDeletionStatus.PENDING
    started_at: Optional[datetime] = None
    completed_at: Optional[datetime] = None
    error_message: Optional[str] = None
    records_deleted: int = 0


@dataclass
class DeletionRequest:
    """A request to delete a user's data."""
    id: str
    user_id: str
    tenant_id: str
    requested_at: datetime
    requested_by: str  # user, admin, automated
    reason: str
    status: DeletionStatus
    targets: List[DeletionTarget]
    deadline: datetime  # GDPR 30 days
    completed_at: Optional[datetime] = None
    verified_at: Optional[datetime] = None
    verification_report: Optional[Dict] = None


class DeletionService:
    """
    Orchestrates user data deletion across all systems.
    
    Key responsibilities:
    - Accept deletion requests
    - Coordinate deletion across systems
    - Track progress and handle failures
    - Verify deletion completion
    - Maintain audit trail
    """
    
    # Systems in deletion order (dependencies first)
    DELETION_TARGETS = [
        # First: Search indexes
        DeletionTarget("elasticsearch", "search", "delete_document", priority=2),
        
        # Second: File storage
        DeletionTarget("s3_files", "storage", "delete_objects", priority=3),
        DeletionTarget("s3_uploads", "storage", "delete_objects", priority=3),
        
        # Third: Analytics (before primary DB to capture user_id mapping)
        DeletionTarget("bigquery", "analytics", "delete_rows", priority=4),
        DeletionTarget("mixpanel", "analytics", "delete_user", priority=4),
        
        # Fourth: Third-party services
        DeletionTarget("stripe", "third_party", "delete_customer", priority=5),
        DeletionTarget("zendesk", "third_party", "delete_user", priority=5),
        DeletionTarget("intercom", "third_party", "delete_user", priority=5),
        DeletionTarget("mailchimp", "third_party", "unsubscribe_delete", priority=5),
        
        # Then: Primary database (has foreign keys)
        DeletionTarget("postgresql", "database", "delete_cascade", priority=10),
        
        # Last: Caches, AFTER the primary DB, so a concurrent read can't
        # refill them from rows that haven't been deleted yet
        DeletionTarget("redis_cache", "cache", "delete_keys", priority=11),
        DeletionTarget("cdn_cache", "cache", "purge_urls", priority=11),
    ]
    
    def __init__(
        self,
        db,
        deletion_executors: Dict[str, 'DeletionExecutor'],
        event_publisher,
        notification_service
    ):
        self.db = db
        self.executors = deletion_executors
        self.events = event_publisher
        self.notifications = notification_service
    
    async def create_deletion_request(
        self,
        user_id: str,
        tenant_id: str,
        requested_by: str,
        reason: str
    ) -> DeletionRequest:
        """
        Create a new deletion request.
        
        This starts the deletion process.
        """
        request_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc)
        
        # Determine which targets apply to this user
        targets = await self._determine_targets(user_id, tenant_id)
        
        request = DeletionRequest(
            id=request_id,
            user_id=user_id,
            tenant_id=tenant_id,
            requested_at=now,
            requested_by=requested_by,
            reason=reason,
            status=DeletionStatus.PENDING,
            targets=targets,
            deadline=now + timedelta(days=30)  # GDPR deadline
        )
        
        # Store request
        await self._save_request(request)
        
        # Publish event
        await self.events.publish("deletion", {
            "type": "deletion.requested",
            "request_id": request_id,
            "user_id": user_id,
            "tenant_id": tenant_id
        })
        
        logger.info(
            f"Deletion request created",
            extra={
                "request_id": request_id,
                "user_id": user_id,
                "targets": len(targets)
            }
        )
        
        return request
    
    async def execute_deletion(self, request_id: str) -> DeletionRequest:
        """
        Execute a deletion request.
        
        Coordinates deletion across all target systems.
        """
        request = await self.get_request(request_id)
        
        retryable = [
            DeletionStatus.PENDING,
            DeletionStatus.FAILED,
            DeletionStatus.PARTIALLY_COMPLETED,
        ]
        if request.status not in retryable:
            raise ValueError(f"Cannot execute request in status: {request.status}")
        
        request.status = DeletionStatus.IN_PROGRESS
        await self._save_request(request)
        
        # Group targets by priority (on a retry, skip systems already done)
        priority_groups = {}
        for target in request.targets:
            if target.status == SystemDeletionStatus.COMPLETED:
                continue
            if target.priority not in priority_groups:
                priority_groups[target.priority] = []
            priority_groups[target.priority].append(target)
        
        # Execute in priority order
        all_succeeded = True
        
        for priority in sorted(priority_groups.keys()):
            targets = priority_groups[priority]
            
            # Execute targets at same priority level in parallel
            results = await asyncio.gather(
                *[self._execute_target(request, target) for target in targets],
                return_exceptions=True
            )
            
            # Check for failures
            for target, result in zip(targets, results):
                if isinstance(result, Exception):
                    target.status = SystemDeletionStatus.FAILED
                    target.error_message = str(result)
                    all_succeeded = False
                    logger.error(
                        f"Deletion failed for {target.system_name}: {result}",
                        extra={"request_id": request_id}
                    )
            
            if not all_succeeded:
                # Stop before later groups: PostgreSQL must outlive the copies
                # that still need its user_id mapping. A retry resumes here.
                break
        
        # Update final status
        if all_succeeded:
            request.status = DeletionStatus.AWAITING_VERIFICATION
        else:
            request.status = DeletionStatus.PARTIALLY_COMPLETED
        
        await self._save_request(request)
        
        # Publish completion event
        await self.events.publish("deletion", {
            "type": "deletion.executed",
            "request_id": request_id,
            "status": request.status.value,
            "all_succeeded": all_succeeded
        })
        
        return request
    
    async def _execute_target(
        self,
        request: DeletionRequest,
        target: DeletionTarget
    ):
        """Execute deletion for a single target system."""
        target.status = SystemDeletionStatus.IN_PROGRESS
        target.started_at = datetime.now(timezone.utc)
        
        executor = self.executors.get(target.system_name)
        
        if not executor:
            logger.warning(f"No executor for system: {target.system_name}")
            target.status = SystemDeletionStatus.SKIPPED
            return
        
        try:
            result = await executor.delete_user_data(
                user_id=request.user_id,
                tenant_id=request.tenant_id
            )
            
            target.status = SystemDeletionStatus.COMPLETED
            target.completed_at = datetime.now(timezone.utc)
            target.records_deleted = result.get("records_deleted", 0)
            
            logger.info(
                f"Deletion completed for {target.system_name}",
                extra={
                    "request_id": request.id,
                    "records_deleted": target.records_deleted
                }
            )
            
        except Exception as e:
            target.status = SystemDeletionStatus.FAILED
            target.error_message = str(e)
            target.completed_at = datetime.now(timezone.utc)
            raise
    
    async def verify_deletion(self, request_id: str) -> Dict[str, Any]:
        """
        Verify that deletion was successful.
        
        Checks each system to confirm data is gone.
        """
        request = await self.get_request(request_id)
        
        verification_results = {}
        all_verified = True
        
        for target in request.targets:
            if target.status == SystemDeletionStatus.SKIPPED:
                verification_results[target.system_name] = {
                    "status": "skipped",
                    "verified": True
                }
                continue
            
            executor = self.executors.get(target.system_name)
            
            if not executor:
                verification_results[target.system_name] = {
                    "status": "no_executor",
                    "verified": False
                }
                all_verified = False
                continue
            
            try:
                exists = await executor.check_user_exists(
                    user_id=request.user_id,
                    tenant_id=request.tenant_id
                )
                
                verification_results[target.system_name] = {
                    "status": "verified" if not exists else "data_found",
                    "verified": not exists,
                    "checked_at": datetime.now(timezone.utc).isoformat()
                }
                
                if exists:
                    all_verified = False
                    logger.warning(
                        f"Data still exists in {target.system_name}",
                        extra={"request_id": request_id}
                    )
                    
            except Exception as e:
                verification_results[target.system_name] = {
                    "status": "error",
                    "verified": False,
                    "error": str(e)
                }
                all_verified = False
        
        # Update request
        request.verification_report = verification_results
        request.verified_at = datetime.now(timezone.utc)
        
        if all_verified:
            request.status = DeletionStatus.COMPLETED
            request.completed_at = datetime.now(timezone.utc)
        
        await self._save_request(request)
        
        # Notify user if complete
        if all_verified:
            await self._notify_completion(request)
        
        return {
            "all_verified": all_verified,
            "results": verification_results
        }
    
    async def _determine_targets(
        self,
        user_id: str,
        tenant_id: str
    ) -> List[DeletionTarget]:
        """Determine which systems have data for this user."""
        targets = []
        
        for target_template in self.DELETION_TARGETS:
            # Check if system has data for this user
            executor = self.executors.get(target_template.system_name)
            
            if executor:
                has_data = await executor.check_user_exists(user_id, tenant_id)
                
                if has_data:
                    targets.append(DeletionTarget(
                        system_name=target_template.system_name,
                        system_type=target_template.system_type,
                        deletion_method=target_template.deletion_method,
                        priority=target_template.priority
                    ))
        
        return targets
    
    async def _notify_completion(self, request: DeletionRequest):
        """Notify user that deletion is complete."""
        await self.notifications.send(
            user_id=request.user_id,
            template="deletion_complete",
            context={
                "request_id": request.id,
                "completed_at": request.completed_at.isoformat(),
                "systems_deleted": len([
                    t for t in request.targets 
                    if t.status == SystemDeletionStatus.COMPLETED
                ])
            }
        )
    
    async def get_request(self, request_id: str) -> DeletionRequest:
        """Get a deletion request by ID."""
        row = await self.db.fetchrow(
            "SELECT * FROM deletion_requests WHERE id = $1",
            request_id
        )
        
        if not row:
            raise ValueError(f"Deletion request not found: {request_id}")
        
        return self._row_to_request(row)
    
    async def _save_request(self, request: DeletionRequest):
        """Save deletion request to database."""
        await self.db.execute(
            """
            INSERT INTO deletion_requests 
            (id, user_id, tenant_id, requested_at, requested_by, reason,
             status, targets, deadline, completed_at, verified_at, verification_report)
            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
            ON CONFLICT (id) DO UPDATE SET
                status = $7, targets = $8, completed_at = $10,
                verified_at = $11, verification_report = $12
            """,
            request.id, request.user_id, request.tenant_id,
            request.requested_at, request.requested_by, request.reason,
            request.status.value, 
            [t.__dict__ for t in request.targets],
            request.deadline, request.completed_at, request.verified_at,
            request.verification_report
        )
    
    def _row_to_request(self, row) -> DeletionRequest:
        """Convert database row to DeletionRequest."""
        return DeletionRequest(
            id=row["id"],
            user_id=row["user_id"],
            tenant_id=row["tenant_id"],
            requested_at=row["requested_at"],
            requested_by=row["requested_by"],
            reason=row["reason"],
            status=DeletionStatus(row["status"]),
            targets=[DeletionTarget(**t) for t in row["targets"]],
            deadline=row["deadline"],
            completed_at=row.get("completed_at"),
            verified_at=row.get("verified_at"),
            verification_report=row.get("verification_report")
        )

# deletion/executors.py

"""
System-specific deletion executors.

Each executor knows how to delete user data from a specific system.
"""

from abc import ABC, abstractmethod
from typing import Dict, Any
from elasticsearch import NotFoundError
from google.cloud import bigquery
import logging

logger = logging.getLogger(__name__)


class DeletionExecutor(ABC):
    """Base class for deletion executors."""
    
    @abstractmethod
    async def delete_user_data(
        self,
        user_id: str,
        tenant_id: str
    ) -> Dict[str, Any]:
        """
        Delete all user data from this system.
        
        Returns dict with deletion details.
        """
        pass
    
    @abstractmethod
    async def check_user_exists(
        self,
        user_id: str,
        tenant_id: str
    ) -> bool:
        """Check if user data exists in this system."""
        pass


class PostgreSQLDeletionExecutor(DeletionExecutor):
    """
    Deletes user data from PostgreSQL.
    
    Handles cascading deletions across related tables.
    """
    
    def __init__(self, db_pool):
        self.db = db_pool
    
    async def delete_user_data(
        self,
        user_id: str,
        tenant_id: str
    ) -> Dict[str, Any]:
        """
        Delete user and all related data.
        
        Order matters due to foreign keys:
        1. Delete from leaf tables first
        2. Work up to parent tables
        3. Finally delete user record
        """
        records_deleted = 0
        
        async with self.db.acquire() as conn:
            async with conn.transaction():
                # Delete from leaf tables first (no foreign key dependencies)
                
                # Notifications
                result = await conn.execute(
                    "DELETE FROM notifications WHERE user_id = $1 AND tenant_id = $2",
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
                
                # Activity logs
                result = await conn.execute(
                    "DELETE FROM activity_logs WHERE user_id = $1 AND tenant_id = $2",
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
                
                # Sessions
                result = await conn.execute(
                    "DELETE FROM sessions WHERE user_id = $1 AND tenant_id = $2",
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
                
                # User preferences
                result = await conn.execute(
                    "DELETE FROM user_preferences WHERE user_id = $1 AND tenant_id = $2",
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
                
                # Consent records (keep anonymized version for audit)
                result = await conn.execute(
                    """
                    UPDATE consent_records 
                    SET user_id = 'DELETED', ip_address = 'DELETED'
                    WHERE user_id = $1 AND tenant_id = $2
                    """,
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
                
                # Orders - anonymize rather than delete (financial records)
                result = await conn.execute(
                    """
                    UPDATE orders 
                    SET user_id = NULL, 
                        shipping_address = 'DELETED',
                        billing_address = 'DELETED',
                        customer_email = 'DELETED',
                        customer_phone = 'DELETED'
                    WHERE user_id = $1 AND tenant_id = $2
                    """,
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
                
                # Comments - anonymize to preserve content integrity
                result = await conn.execute(
                    """
                    UPDATE comments 
                    SET user_id = NULL, author_name = 'Deleted User'
                    WHERE user_id = $1 AND tenant_id = $2
                    """,
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
                
                # Finally, delete the user record
                result = await conn.execute(
                    "DELETE FROM users WHERE id = $1 AND tenant_id = $2",
                    user_id, tenant_id
                )
                records_deleted += int(result.split()[-1])
        
        logger.info(
            f"PostgreSQL deletion complete",
            extra={
                "user_id": user_id,
                "records_deleted": records_deleted
            }
        )
        
        return {"records_deleted": records_deleted}
    
    async def check_user_exists(
        self,
        user_id: str,
        tenant_id: str
    ) -> bool:
        """Check if user exists in PostgreSQL."""
        result = await self.db.fetchrow(
            "SELECT 1 FROM users WHERE id = $1 AND tenant_id = $2",
            user_id, tenant_id
        )
        return result is not None


class ElasticsearchDeletionExecutor(DeletionExecutor):
    """Deletes user data from Elasticsearch."""
    
    def __init__(self, es_client):
        self.es = es_client
    
    async def delete_user_data(
        self,
        user_id: str,
        tenant_id: str
    ) -> Dict[str, Any]:
        """Delete user documents from all indices."""
        indices = ["users", "documents", "activities"]
        total_deleted = 0
        
        for index in indices:
            try:
                result = await self.es.delete_by_query(
                    index=f"{tenant_id}_{index}",
                    body={
                        "query": {
                            "term": {"user_id": user_id}
                        }
                    }
                )
                total_deleted += result.get("deleted", 0)
            except NotFoundError:
                continue  # Index doesn't exist for this tenant: nothing to delete
            # Any other error propagates, so the target is marked FAILED
        
        return {"records_deleted": total_deleted}
    
    async def check_user_exists(
        self,
        user_id: str,
        tenant_id: str
    ) -> bool:
        """Check if user exists in any index."""
        indices = ["users", "documents", "activities"]
        
        for index in indices:
            try:
                result = await self.es.count(
                    index=f"{tenant_id}_{index}",
                    body={
                        "query": {
                            "term": {"user_id": user_id}
                        }
                    }
                )
                if result.get("count", 0) > 0:
                    return True
            except NotFoundError:
                continue
            # Other errors propagate: "couldn't check" must never read as "deleted"
        
        return False


class RedisDeletionExecutor(DeletionExecutor):
    """Deletes user data from Redis cache."""
    
    def __init__(self, redis_client):
        self.redis = redis_client
    
    async def delete_user_data(
        self,
        user_id: str,
        tenant_id: str
    ) -> Dict[str, Any]:
        """Delete all user cache keys."""
        patterns = [
            f"tenant:{tenant_id}:user:{user_id}:*",
            f"tenant:{tenant_id}:session:{user_id}:*",
            f"tenant:{tenant_id}:cache:user:{user_id}",
        ]
        
        total_deleted = 0
        
        for pattern in patterns:
            # SCAN, not KEYS: KEYS is O(N) and blocks Redis while it runs
            keys = [k async for k in self.redis.scan_iter(match=pattern)]
            if keys:
                deleted = await self.redis.delete(*keys)
                total_deleted += deleted
        
        return {"records_deleted": total_deleted}
    
    async def check_user_exists(
        self,
        user_id: str,
        tenant_id: str
    ) -> bool:
        """Check if user data exists in cache."""
        patterns = [
            f"tenant:{tenant_id}:user:{user_id}:*",
            f"tenant:{tenant_id}:session:{user_id}:*",
        ]
        
        for pattern in patterns:
            async for _ in self.redis.scan_iter(match=pattern):
                return True
        
        return False


class S3DeletionExecutor(DeletionExecutor):
    """Deletes user files from S3."""
    
    def __init__(self, s3_client, bucket: str):
        self.s3 = s3_client
        self.bucket = bucket
    
    async def delete_user_data(
        self,
        user_id: str,
        tenant_id: str
    ) -> Dict[str, Any]:
        """Delete all user files from S3."""
        prefix = f"tenants/{tenant_id}/users/{user_id}/"
        
        # List all objects with prefix
        objects_to_delete = []
        paginator = self.s3.get_paginator('list_objects_v2')
        
        async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
            for obj in page.get('Contents', []):
                objects_to_delete.append({'Key': obj['Key']})
        
        if not objects_to_delete:
            return {"records_deleted": 0}
        
        # Delete in batches of 1000 (S3 limit). On a versioned bucket this
        # only adds delete markers: old versions must expire via a lifecycle
        # rule (see s3_versioning retention below) or be deleted by VersionId
        total_deleted = 0
        
        for i in range(0, len(objects_to_delete), 1000):
            batch = objects_to_delete[i:i+1000]
            result = await self.s3.delete_objects(
                Bucket=self.bucket,
                Delete={'Objects': batch}
            )
            if result.get('Errors'):
                raise RuntimeError(f"S3 delete errors: {result['Errors'][:3]}")
            total_deleted += len(batch)
        
        return {"records_deleted": total_deleted}
    
    async def check_user_exists(
        self,
        user_id: str,
        tenant_id: str
    ) -> bool:
        """Check if user has files in S3."""
        prefix = f"tenants/{tenant_id}/users/{user_id}/"
        
        result = await self.s3.list_objects_v2(
            Bucket=self.bucket,
            Prefix=prefix,
            MaxKeys=1
        )
        
        return result.get('KeyCount', 0) > 0


class StripeDeletionExecutor(DeletionExecutor):
    """Deletes user data from Stripe."""
    
    def __init__(self, stripe_client):
        self.stripe = stripe_client
    
    async def delete_user_data(
        self,
        user_id: str,
        tenant_id: str
    ) -> Dict[str, Any]:
        """Delete customer from Stripe."""
        # Find Stripe customer by metadata (list() can't filter on metadata;
        # the Search API can). Storing stripe_customer_id locally is better.
        customers = await self.stripe.Customer.search(
            query=f"metadata['user_id']:'{user_id}' AND metadata['tenant_id']:'{tenant_id}'",
            limit=1
        )
        
        if not customers.data:
            return {"records_deleted": 0}
        
        customer = customers.data[0]
        
        # Deleting the customer cancels its subscriptions; Stripe may retain
        # payment records it is legally required to keep
        await self.stripe.Customer.delete(customer.id)
        
        return {"records_deleted": 1, "stripe_customer_id": customer.id}
    
    async def check_user_exists(
        self,
        user_id: str,
        tenant_id: str
    ) -> bool:
        """Check if customer exists in Stripe."""
        customers = await self.stripe.Customer.search(
            query=f"metadata['user_id']:'{user_id}' AND metadata['tenant_id']:'{tenant_id}'",
            limit=1
        )
        
        return len(customers.data) > 0


class BigQueryDeletionExecutor(DeletionExecutor):
    """Deletes user data from BigQuery analytics."""
    
    def __init__(self, bq_client, dataset: str):
        self.bq = bq_client
        self.dataset = dataset
    
    async def delete_user_data(
        self,
        user_id: str,
        tenant_id: str
    ) -> Dict[str, Any]:
        """
        Delete user data from BigQuery tables.
        
        Note: BigQuery DELETE can be slow and expensive.
        Consider partitioning strategy for better deletion.
        """
        tables = ["events", "page_views", "user_properties"]
        total_deleted = 0
        
        for table in tables:
            query = f"""
            DELETE FROM `{self.dataset}.{table}`
            WHERE user_id = @user_id AND tenant_id = @tenant_id
            """
            
            job_config = bigquery.QueryJobConfig(
                query_parameters=[
                    bigquery.ScalarQueryParameter("user_id", "STRING", user_id),
                    bigquery.ScalarQueryParameter("tenant_id", "STRING", tenant_id),
                ]
            )
            
            job = self.bq.query(query, job_config=job_config)
            job.result()  # Wait for the DML job to finish
            total_deleted += job.num_dml_affected_rows or 0
        
        return {"records_deleted": total_deleted}
    
    async def check_user_exists(
        self,
        user_id: str,
        tenant_id: str
    ) -> bool:
        """Check if user exists in BigQuery."""
        query = f"""
        SELECT 1 FROM `{self.dataset}.events`
        WHERE user_id = @user_id AND tenant_id = @tenant_id
        LIMIT 1
        """
        
        job_config = bigquery.QueryJobConfig(
            query_parameters=[
                bigquery.ScalarQueryParameter("user_id", "STRING", user_id),
                bigquery.ScalarQueryParameter("tenant_id", "STRING", tenant_id),
            ]
        )
        
        rows = self.bq.query(query, job_config=job_config).result()
        return rows.total_rows > 0

# deletion/backup_handler.py

"""
Handling user data in backups.

Backups are the hardest deletion challenge because you can't
surgically remove one user from a backup.
"""

from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import List
import logging

logger = logging.getLogger(__name__)


@dataclass
class BackupRetentionPolicy:
    """Backup retention configuration."""
    backup_type: str
    retention_days: int
    deletion_strategy: str  # "expire" or "exclude_restore"


class BackupDeletionHandler:
    """
    Handles deletion requests for data in backups.
    
    Strategy:
    1. Track users pending deletion
    2. When backup expires, deletion is complete for that backup
    3. If backup is restored, apply pending deletions
    """
    
    RETENTION_POLICIES = [
        BackupRetentionPolicy("postgresql_daily", 30, "expire"),
        BackupRetentionPolicy("postgresql_weekly", 90, "expire"),
        BackupRetentionPolicy("mongodb_daily", 30, "expire"),
        BackupRetentionPolicy("s3_versioning", 90, "expire"),
    ]
    
    def __init__(self, db):
        self.db = db
    
    async def register_pending_deletion(
        self,
        deletion_request_id: str,
        user_id: str,
        tenant_id: str
    ):
        """
        Register a user for deletion from backups.
        
        This tracks that when backups expire or are restored,
        this user's data must be deleted.
        """
        # Calculate when backup retention expires for each type
        expiry_dates = {}
        
        for policy in self.RETENTION_POLICIES:
            expiry_date = datetime.now(timezone.utc) + timedelta(days=policy.retention_days)
            expiry_dates[policy.backup_type] = expiry_date
        
        await self.db.execute(
            """
            INSERT INTO backup_pending_deletions
            (deletion_request_id, user_id, tenant_id, created_at, backup_expiry_dates)
            VALUES ($1, $2, $3, $4, $5)
            """,
            deletion_request_id, user_id, tenant_id,
            datetime.now(timezone.utc), expiry_dates
        )
        
        logger.info(
            f"Registered pending backup deletion",
            extra={
                "deletion_request_id": deletion_request_id,
                "user_id": user_id,
                "latest_expiry": max(expiry_dates.values())
            }
        )
    
    async def get_backup_deletion_status(
        self,
        deletion_request_id: str
    ) -> dict:
        """
        Get status of backup deletion for a request.
        """
        record = await self.db.fetchrow(
            """
            SELECT * FROM backup_pending_deletions
            WHERE deletion_request_id = $1
            """,
            deletion_request_id
        )
        
        if not record:
            return {"status": "not_tracked"}
        
        expiry_dates = record["backup_expiry_dates"]
        now = datetime.now(timezone.utc)
        
        status = {}
        all_expired = True
        
        for backup_type, expiry_date in expiry_dates.items():
            if expiry_date <= now:
                status[backup_type] = "expired"
            else:
                status[backup_type] = f"expires_{expiry_date.isoformat()}"
                all_expired = False
        
        return {
            "status": "complete" if all_expired else "pending",
            "backup_status": status,
            "fully_deleted_at": max(expiry_dates.values()) if not all_expired else None
        }
    
    async def on_backup_restore(
        self,
        backup_type: str,
        backup_date: datetime,
        restore_database: str
    ):
        """
        Called when a backup is restored.
        
        Must apply all pending deletions to the restored data.
        """
        # Re-apply every deletion requested AFTER the backup was taken: the
        # backup still contains those users. Deletes are idempotent, so also
        # include requests from the 30 days before it, which may not have
        # finished executing when the snapshot ran.
        pending = await self.db.fetch(
            """
            SELECT * FROM backup_pending_deletions
            WHERE created_at >= $1
            """,
            backup_date - timedelta(days=30)
        )
        
        logger.warning(
            f"Applying {len(pending)} pending deletions to restored backup",
            extra={"backup_type": backup_type, "backup_date": backup_date}
        )
        
        for record in pending:
            # Apply deletion to restored database
            await self._apply_deletion_to_restore(
                record["user_id"],
                record["tenant_id"],
                restore_database
            )
    
    async def _apply_deletion_to_restore(
        self,
        user_id: str,
        tenant_id: str,
        database: str
    ):
        """Apply a pending deletion to a restored database."""
        # This would use the same logic as the main deletion
        # but target the restored database
        logger.info(
            f"Applying deletion to restored database",
            extra={
                "user_id": user_id,
                "database": database
            }
        )

# deletion/anonymization.py

"""
Data anonymization for cases where deletion isn't possible.

Some data can't be deleted (legal retention) but can be anonymized.
"""

from typing import Dict, Any
import hashlib
import secrets
import uuid


class AnonymizationService:
    """
    Anonymizes data instead of deleting it.
    
    Used for:
    - Financial records (legal retention requirements)
    - Aggregated statistics
    - Audit logs
    
    Note: an unsalted hash of an ID is only pseudonymisation (anyone with
    the ID can recompute it), so IDs are replaced randomly and hashes use
    a per-run secret salt that is never stored.
    """
    
    def __init__(self):
        self._salt = secrets.token_bytes(16)  # Discarded with the instance
    
    def anonymize_user_record(self, user: Dict[str, Any]) -> Dict[str, Any]:
        """
        Anonymize a user record.
        
        Removes all PII while preserving structure.
        """
        return {
            "id": self._generate_anonymous_id(user["id"]),
            "email": "deleted@anonymized.local",
            "name": "Deleted User",
            "phone": None,
            "address": None,
            "created_at": user["created_at"],  # Keep for analytics
            "tenant_id": user["tenant_id"],    # Keep for tenant analytics
        }
    
    def anonymize_order(self, order: Dict[str, Any]) -> Dict[str, Any]:
        """
        Anonymize an order record.
        
        Keeps financial data but removes PII.
        """
        return {
            "id": order["id"],
            "user_id": None,  # Remove link to user
            "tenant_id": order["tenant_id"],
            "amount": order["amount"],  # Keep for financial records
            "currency": order["currency"],
            "created_at": order["created_at"],
            "shipping_address": "REDACTED",
            "billing_address": "REDACTED",
            "customer_email": "deleted@anonymized.local",
            "customer_phone": None,
            "items": order["items"],  # Keep order details
        }
    
    def anonymize_event(self, event: Dict[str, Any]) -> Dict[str, Any]:
        """
        Anonymize an analytics event.
        """
        return {
            "event_type": event["event_type"],
            "tenant_id": event["tenant_id"],
            "timestamp": event["timestamp"],
            "user_id": None,  # Remove user link
            "session_id": self._hash_value(event.get("session_id", "")),
            "properties": self._anonymize_properties(event.get("properties", {})),
        }
    
    def _generate_anonymous_id(self, original_id: str) -> str:
        """Generate an anonymous ID with no link to the original."""
        # Random, not hash(original_id): a deterministic hash can be
        # recomputed from the ID and re-linked
        return uuid.uuid4().hex[:16]
    
    def _hash_value(self, value: str) -> str:
        """Hash with the throwaway salt: groups equal values, can't be reversed."""
        if not value:
            return ""
        return hashlib.sha256(self._salt + value.encode()).hexdigest()[:12]
    
    def _anonymize_properties(self, props: Dict) -> Dict:
        """Remove PII from event properties."""
        pii_keys = ["email", "name", "phone", "address", "ip_address", "user_agent"]
        
        return {
            k: "REDACTED" if k in pii_keys else v
            for k, v in props.items()
        }

# deletion/audit.py

"""
Audit logging for deletion requests.

Critical for proving compliance with GDPR.
"""

from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Optional, List
from enum import Enum
import uuid


class AuditEventType(Enum):
    """Types of deletion audit events."""
    REQUEST_CREATED = "request_created"
    REQUEST_APPROVED = "request_approved"
    DELETION_STARTED = "deletion_started"
    SYSTEM_DELETION_STARTED = "system_deletion_started"
    SYSTEM_DELETION_COMPLETED = "system_deletion_completed"
    SYSTEM_DELETION_FAILED = "system_deletion_failed"
    VERIFICATION_STARTED = "verification_started"
    VERIFICATION_COMPLETED = "verification_completed"
    REQUEST_COMPLETED = "request_completed"
    REQUEST_FAILED = "request_failed"


@dataclass
class DeletionAuditEvent:
    """An audit event for a deletion request."""
    id: str
    deletion_request_id: str
    event_type: AuditEventType
    timestamp: datetime
    actor: str  # Who/what triggered this event
    system_name: Optional[str]
    details: dict
    

class DeletionAuditService:
    """
    Records audit trail for all deletion activities.
    
    This audit log is retained even after deletion completes
    to prove compliance.
    """
    
    def __init__(self, db):
        self.db = db
    
    async def log_event(
        self,
        deletion_request_id: str,
        event_type: AuditEventType,
        actor: str,
        system_name: Optional[str] = None,
        details: dict = None
    ) -> DeletionAuditEvent:
        """
        Log an audit event.
        """
        event = DeletionAuditEvent(
            id=str(uuid.uuid4()),
            deletion_request_id=deletion_request_id,
            event_type=event_type,
            timestamp=datetime.now(timezone.utc),
            actor=actor,
            system_name=system_name,
            details=details or {}
        )
        
        await self.db.execute(
            """
            INSERT INTO deletion_audit_log
            (id, deletion_request_id, event_type, timestamp, actor, system_name, details)
            VALUES ($1, $2, $3, $4, $5, $6, $7)
            """,
            event.id, event.deletion_request_id, event.event_type.value,
            event.timestamp, event.actor, event.system_name, event.details
        )
        
        return event
    
    async def get_audit_trail(
        self,
        deletion_request_id: str
    ) -> List[DeletionAuditEvent]:
        """
        Get complete audit trail for a deletion request.
        """
        rows = await self.db.fetch(
            """
            SELECT * FROM deletion_audit_log
            WHERE deletion_request_id = $1
            ORDER BY timestamp ASC
            """,
            deletion_request_id
        )
        
        return [
            DeletionAuditEvent(
                id=row["id"],
                deletion_request_id=row["deletion_request_id"],
                event_type=AuditEventType(row["event_type"]),
                timestamp=row["timestamp"],
                actor=row["actor"],
                system_name=row["system_name"],
                details=row["details"]
            )
            for row in rows
        ]
    
    async def generate_compliance_report(
        self,
        deletion_request_id: str
    ) -> dict:
        """
        Generate a compliance report for a deletion request.
        
        This can be provided to auditors or data protection authorities.
        """
        # Get the request
        request = await self.db.fetchrow(
            "SELECT * FROM deletion_requests WHERE id = $1",
            deletion_request_id
        )
        
        # Get audit trail
        audit_trail = await self.get_audit_trail(deletion_request_id)
        
        # Build report
        report = {
            "report_generated_at": datetime.now(timezone.utc).isoformat(),
            "deletion_request": {
                "id": request["id"],
                "user_id": "REDACTED",  # Don't include actual user ID
                "requested_at": request["requested_at"].isoformat(),
                "requested_by": request["requested_by"],
                "reason": request["reason"],
                "deadline": request["deadline"].isoformat(),
                "status": request["status"],
                "completed_at": request["completed_at"].isoformat() if request["completed_at"] else None,
            },
            "systems_affected": [
                {
                    "system": t["system_name"],
                    "status": t["status"],
                    "records_deleted": t.get("records_deleted", 0),
                    "completed_at": t.get("completed_at")
                }
                for t in request["targets"]
            ],
            "audit_trail": [
                {
                    "timestamp": e.timestamp.isoformat(),
                    "event": e.event_type.value,
                    "system": e.system_name,
                    "actor": e.actor
                }
                for e in audit_trail
            ],
            "verification": request.get("verification_report", {}),
            "compliance_statement": self._generate_compliance_statement(request)
        }
        
        return report
    
    def _generate_compliance_statement(self, request: dict) -> str:
        """Generate a compliance statement for the report."""
        if request["status"] == "completed":
            return (
                f"This deletion request was completed on {request['completed_at']}. "
                f"All personal data for the data subject has been deleted from primary systems. "
                f"Data in backups will be fully purged according to our retention policy."
            )
        elif request["status"] == "partially_completed":
            return (
                "This deletion request has been partially completed. "
                "Some systems reported errors during deletion. "
                "Manual intervention may be required."
            )
        else:
            return f"This deletion request is currently in status: {request['status']}."