Background Jobs Revolution: Mastering Automated Processing in ASP.NET Core (Part

Previous article: Testing Mastery: Catch Bugs Early with Comprehensive Testing in ASP.NET Core (Part- 29 of 40)

The Background Jobs Revolution
Core Background Processing Concepts
Building with IHostedService
Hangfire Mastery
Quartz.NET Enterprise Scheduling
Coravel Simplified Scheduling
Real-World E-Commerce Automation
Advanced Patterns & Scenarios
Monitoring & Diagnostics
Production Ready Deployment

1. The Background Jobs Revolution

Why Background Jobs Matter in Modern Applications

In today's digital landscape, users expect instant responses and seamless experiences. However, many tasks in web applications are time-consuming and would degrade user experience if processed synchronously. This is where background jobs revolutionize application architecture.

Real-World Scenarios:

Sending welcome emails to new users
Generating monthly sales reports
Processing image and video uploads
Data synchronization between systems
Cleaning up temporary files
Sending push notifications

The Evolution of Background Processing

Traditional Approach:

csharp

  
    // Problem: Blocking the main thread
public async Task<IActionResult> ProcessOrder(Order order)
{
    // This blocks the user for several seconds
    await _emailService.SendOrderConfirmation(order);
    await _reportService.GenerateInvoice(order);
    await _analyticsService.TrackOrder(order);
    
    return Ok();
}

Modern Background Jobs Approach

  
    // Solution: Immediate response with background processing
public async Task<IActionResult> ProcessOrder(Order order)
{
    // Immediate response
    _backgroundJobService.Enqueue(() => _emailService.SendOrderConfirmation(order));
    _backgroundJobService.Enqueue(() => _reportService.GenerateInvoice(order));
    _backgroundJobService.Schedule(() => _analyticsService.TrackOrder(order), TimeSpan.FromMinutes(30));
    
    return Ok("Order processing started!");
}

2. Core Background Processing Concepts

2.1 Understanding Different Job Types

  
    public enum JobType
{
    // Fire-and-forget: Execute once, immediately
    FireAndForget,
    
    // Delayed: Execute once, after specified delay
    Delayed,
    
    // Recurring: Execute repeatedly on a schedule
    Recurring,
    
    // Continuation: Execute after another job completes
    Continuation,
    
    // Batch: Execute multiple jobs as a unit
    Batch
}

public class JobCharacteristics
{
    public TimeSpan ExpectedDuration { get; set; }
    public int RetryCount { get; set; }
    public bool IsIdempotent { get; set; }
    public JobPriority Priority { get; set; }
    public DateTime? Timeout { get; set; }
}

public enum JobPriority
{
    Low = 0,
    Normal = 1,
    High = 2,
    Critical = 3
}

2.2 Storage Considerations for Background Jobs

  
    // Job Storage Options Configuration
public class JobStorageConfig
{
    public StorageType Type { get; set; }
    public string ConnectionString { get; set; }
    public TimeSpan JobExpiration { get; set; } = TimeSpan.FromDays(30);
    public int QueuePollInterval { get; set; } = 1000; // ms
}

public enum StorageType
{
    InMemory,       // Development only
    SQLServer,      // Enterprise applications
    PostgreSQL,     // Cross-platform
    Redis,          // High performance
    MongoDB         // Document storage
}

3. Building with IHostedService

3.1 Custom Background Service Implementation

  
    // Basic background service template
public abstract class BackgroundService : IHostedService, IDisposable
{
    private Task _executingTask;
    private readonly CancellationTokenSource _stoppingCts = new();
    private readonly ILogger<BackgroundService> _logger;

    protected BackgroundService(ILogger<BackgroundService> logger)
    {
        _logger = logger;
    }

    protected abstract Task ExecuteAsync(CancellationToken stoppingToken);

    public virtual Task StartAsync(CancellationToken cancellationToken)
    {
        _logger.LogInformation("Background service {ServiceName} is starting", GetType().Name);
        
        _executingTask = ExecuteAsync(_stoppingCts.Token);
        
        if (_executingTask.IsCompleted)
        {
            return _executingTask;
        }
        
        return Task.CompletedTask;
    }

    public virtual async Task StopAsync(CancellationToken cancellationToken)
    {
        _logger.LogInformation("Background service {ServiceName} is stopping", GetType().Name);
        
        if (_executingTask == null)
        {
            return;
        }
        
        try
        {
            _stoppingCts.Cancel();
        }
        finally
        {
            await Task.WhenAny(_executingTask, Task.Delay(Timeout.Infinite, cancellationToken));
        }
    }

    public virtual void Dispose()
    {
        _stoppingCts.Cancel();
        _stoppingCts?.Dispose();
    }
}

3.2 Email Notification Service

  
    // Real-world email background service
public class EmailNotificationService : BackgroundService
{
    private readonly IServiceProvider _serviceProvider;
    private readonly ILogger<EmailNotificationService> _logger;
    private readonly IConfiguration _configuration;
    private readonly TimeSpan _processingInterval = TimeSpan.FromSeconds(30);

    public EmailNotificationService(
        IServiceProvider serviceProvider,
        ILogger<EmailNotificationService> logger,
        IConfiguration configuration)
    {
        _serviceProvider = serviceProvider;
        _logger = logger;
        _configuration = configuration;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("Email Notification Service started");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await ProcessPendingEmails(stoppingToken);
                await Task.Delay(_processingInterval, stoppingToken);
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                _logger.LogError(ex, "Error occurred processing emails");
                await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); // Backoff on error
            }
        }

        _logger.LogInformation("Email Notification Service stopped");
    }

    private async Task ProcessPendingEmails(CancellationToken stoppingToken)
    {
        using var scope = _serviceProvider.CreateScope();
        var emailQueueService = scope.ServiceProvider.GetRequiredService<IEmailQueueService>();
        var emailSender = scope.ServiceProvider.GetRequiredService<IEmailSender>();

        var pendingEmails = await emailQueueService.GetPendingEmailsAsync(50);
        
        _logger.LogInformation("Processing {EmailCount} pending emails", pendingEmails.Count);

        var tasks = pendingEmails.Select(email => ProcessSingleEmail(email, emailSender, emailQueueService, stoppingToken));
        await Task.WhenAll(tasks);
    }

    private async Task ProcessSingleEmail(
        EmailQueueItem email, 
        IEmailSender emailSender,
        IEmailQueueService emailQueueService,
        CancellationToken stoppingToken)
    {
        try
        {
            // Mark as processing
            await emailQueueService.MarkAsProcessingAsync(email.Id);

            // Send email
            var result = await emailSender.SendAsync(
                email.To,
                email.Subject,
                email.Body,
                email.IsHtml);

            if (result.Success)
            {
                await emailQueueService.MarkAsCompletedAsync(email.Id);
                _logger.LogInformation("Successfully sent email to {Recipient}", email.To);
            }
            else
            {
                await emailQueueService.MarkAsFailedAsync(email.Id, result.Error);
                _logger.LogWarning("Failed to send email to {Recipient}: {Error}", email.To, result.Error);
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Error processing email {EmailId} to {Recipient}", email.Id, email.To);
            await emailQueueService.MarkAsFailedAsync(email.Id, ex.Message);
        }
    }
}

// Supporting models and interfaces
public interface IEmailQueueService
{
    Task<List<EmailQueueItem>> GetPendingEmailsAsync(int batchSize);
    Task MarkAsProcessingAsync(Guid emailId);
    Task MarkAsCompletedAsync(Guid emailId);
    Task MarkAsFailedAsync(Guid emailId, string error);
    Task<Guid> QueueEmailAsync(string to, string subject, string body, bool isHtml = true);
}

public class EmailQueueItem
{
    public Guid Id { get; set; }
    public string To { get; set; }
    public string Subject { get; set; }
    public string Body { get; set; }
    public bool IsHtml { get; set; }
    public EmailStatus Status { get; set; }
    public int RetryCount { get; set; }
    public string Error { get; set; }
    public DateTime CreatedAt { get; set; }
    public DateTime? ProcessedAt { get; set; }
}

public enum EmailStatus
{
    Pending = 0,
    Processing = 1,
    Completed = 2,
    Failed = 3
}

public class EmailResult
{
    public bool Success { get; set; }
    public string Error { get; set; }
    public string MessageId { get; set; }
}

3.3 Database Maintenance Service

  
    // Automated database maintenance service
public class DatabaseMaintenanceService : BackgroundService
{
    private readonly IServiceProvider _serviceProvider;
    private readonly ILogger<DatabaseMaintenanceService> _logger;
    private readonly List<MaintenanceTask> _maintenanceTasks;

    public DatabaseMaintenanceService(
        IServiceProvider serviceProvider,
        ILogger<DatabaseMaintenanceService> logger)
    {
        _serviceProvider = serviceProvider;
        _logger = logger;
        
        _maintenanceTasks = new List<MaintenanceTask>
        {
            new MaintenanceTask
            {
                Name = "Clean Old Audit Logs",
                Schedule = TimeSpan.FromHours(24),
                LastRun = DateTime.MinValue,
                Action = CleanOldAuditLogs
            },
            new MaintenanceTask
            {
                Name = "Rebuild Indexes",
                Schedule = TimeSpan.FromDays(7),
                LastRun = DateTime.MinValue,
                Action = RebuildIndexes
            },
            new MaintenanceTask
            {
                Name = "Update Statistics",
                Schedule = TimeSpan.FromDays(1),
                LastRun = DateTime.MinValue,
                Action = UpdateStatistics
            },
            new MaintenanceTask
            {
                Name = "Clean Temporary Files",
                Schedule = TimeSpan.FromHours(6),
                LastRun = DateTime.MinValue,
                Action = CleanTemporaryFiles
            }
        };
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("Database Maintenance Service started");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await ProcessMaintenanceTasks(stoppingToken);
                await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken); // Check every 5 minutes
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                _logger.LogError(ex, "Error in database maintenance service");
                await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken);
            }
        }

        _logger.LogInformation("Database Maintenance Service stopped");
    }

    private async Task ProcessMaintenanceTasks(CancellationToken stoppingToken)
    {
        var now = DateTime.UtcNow;
        var tasksToRun = _maintenanceTasks
            .Where(task => now - task.LastRun >= task.Schedule)
            .ToList();

        if (!tasksToRun.Any())
            return;

        _logger.LogInformation("Running {TaskCount} maintenance tasks", tasksToRun.Count);

        using var scope = _serviceProvider.CreateScope();
        
        foreach (var task in tasksToRun)
        {
            if (stoppingToken.IsCancellationRequested)
                break;

            try
            {
                _logger.LogInformation("Starting maintenance task: {TaskName}", task.Name);
                await task.Action(scope.ServiceProvider, stoppingToken);
                task.LastRun = now;
                _logger.LogInformation("Completed maintenance task: {TaskName}", task.Name);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Error running maintenance task: {TaskName}", task.Name);
            }
        }
    }

    private async Task CleanOldAuditLogs(IServiceProvider services, CancellationToken cancellationToken)
    {
        var dbContext = services.GetRequiredService<ApplicationDbContext>();
        var retentionDays = 90; // Keep logs for 90 days
        
        var cutoffDate = DateTime.UtcNow.AddDays(-retentionDays);
        var oldLogs = await dbContext.AuditLogs
            .Where(log => log.CreatedAt < cutoffDate)
            .ToListAsync(cancellationToken);

        if (oldLogs.Any())
        {
            dbContext.AuditLogs.RemoveRange(oldLogs);
            await dbContext.SaveChangesAsync(cancellationToken);
            _logger.LogInformation("Cleaned {LogCount} old audit logs", oldLogs.Count);
        }
    }

    private async Task RebuildIndexes(IServiceProvider services, CancellationToken cancellationToken)
    {
        var dbContext = services.GetRequiredService<ApplicationDbContext>();
        
        // Rebuild indexes for large tables
        var tables = new[] { "Orders", "Products", "Users", "AuditLogs" };
        
        foreach (var table in tables)
        {
            var sql = $"ALTER INDEX ALL ON [{table}] REBUILD";
            await dbContext.Database.ExecuteSqlRawAsync(sql, cancellationToken);
            _logger.LogInformation("Rebuilt indexes for table: {TableName}", table);
        }
    }

    private async Task UpdateStatistics(IServiceProvider services, CancellationToken cancellationToken)
    {
        var dbContext = services.GetRequiredService<ApplicationDbContext>();
        await dbContext.Database.ExecuteSqlRawAsync("EXEC sp_updatestats", cancellationToken);
        _logger.LogInformation("Updated database statistics");
    }

    private async Task CleanTemporaryFiles(IServiceProvider services, CancellationToken cancellationToken)
    {
        var fileService = services.GetRequiredService<IFileService>();
        var tempPath = Path.GetTempPath();
        var cutoffDate = DateTime.UtcNow.AddDays(-1);
        
        var filesDeleted = await fileService.CleanOldFilesAsync(tempPath, "*.tmp", cutoffDate);
        _logger.LogInformation("Cleaned {FileCount} temporary files", filesDeleted);
    }
}

public class MaintenanceTask
{
    public string Name { get; set; }
    public TimeSpan Schedule { get; set; }
    public DateTime LastRun { get; set; }
    public Func<IServiceProvider, CancellationToken, Task> Action { get; set; }
}

4. Hangfire Mastery

4.1 Hangfire Setup and Configuration

  
    // Program.cs - Hangfire Configuration
using Hangfire;
using Hangfire.SqlServer;

var builder = WebApplication.CreateBuilder(args);

// Add Hangfire services
builder.Services.AddHangfire(configuration => configuration
    .SetDataCompatibilityLevel(CompatibilityLevel.Version_170)
    .UseSimpleAssemblyNameTypeSerializer()
    .UseRecommendedSerializerSettings()
    .UseSqlServerStorage(builder.Configuration.GetConnectionString("HangfireConnection"), new SqlServerStorageOptions
    {
        CommandBatchMaxTimeout = TimeSpan.FromMinutes(5),
        SlidingInvisibilityTimeout = TimeSpan.FromMinutes(5),
        QueuePollInterval = TimeSpan.Zero,
        UseRecommendedIsolationLevel = true,
        DisableGlobalLocks = true,
        SchemaName = "Hangfire"
    }));

// Add the processing server as IHostedService
builder.Services.AddHangfireServer(options =>
{
    options.WorkerCount = Environment.ProcessorCount * 2;
    options.Queues = new[] { "default", "emails", "reports", "critical" };
    options.ServerName = $"Server_{Environment.MachineName}";
});

// Register job-related services
builder.Services.AddScoped<IEmailService, EmailService>();
builder.Services.AddScoped<IReportService, ReportService>();
builder.Services.AddScoped<IDataProcessingService, DataProcessingService>();

var app = builder.Build();

// Configure Hangfire dashboard (only in development or for authorized users)
if (app.Environment.IsDevelopment())
{
    app.UseHangfireDashboard("/hangfire", new DashboardOptions
    {
        Authorization = new[] { new HangfireAuthorizationFilter() },
        DashboardTitle = "ECommerce Background Jobs",
        StatsPollingInterval = 10000
    });
}
else
{
    app.UseHangfireDashboard("/hangfire", new DashboardOptions
    {
        Authorization = new[] { new HangfireAuthorizationFilter() },
        IsReadOnlyFunc = context => false
    });
}

// Configure recurring jobs
ConfigureRecurringJobs();

app.Run();

void ConfigureRecurringJobs()
{
    // Daily cleanup job
    RecurringJob.AddOrUpdate<DatabaseMaintenanceService>(
        "daily-cleanup",
        service => service.CleanOldDataAsync(),
        Cron.Daily(2, 0), // 2 AM daily
        TimeZoneInfo.Local);

    // Hourly statistics update
    RecurringJob.AddOrUpdate<StatisticsService>(
        "update-statistics",
        service => service.UpdateStatisticsAsync(),
        Cron.Hourly,
        TimeZoneInfo.Local);

    // Weekly report generation
    RecurringJob.AddOrUpdate<ReportService>(
        "weekly-reports",
        service => service.GenerateWeeklyReportsAsync(),
        Cron.Weekly(DayOfWeek.Monday, 3, 0), // Monday 3 AM
        TimeZoneInfo.Local);
}

// Hangfire authorization filter
public class HangfireAuthorizationFilter : IDashboardAuthorizationFilter
{
    public bool Authorize(DashboardContext context)
    {
        var httpContext = context.GetHttpContext();
        
        // Allow all in development
        if (httpContext.Request.Host.Host.Contains("localhost"))
            return true;
            
        // Implement your authorization logic here
        return httpContext.User.Identity.IsAuthenticated && 
               httpContext.User.IsInRole("Administrator");
    }
}

4.2 Advanced Job Scenarios with Hangfire

  
    // Advanced job service with retry policies and monitoring
public class AdvancedJobService
{
    private readonly IBackgroundJobClient _backgroundJobClient;
    private readonly IRecurringJobManager _recurringJobManager;
    private readonly ILogger<AdvancedJobService> _logger;

    public AdvancedJobService(
        IBackgroundJobClient backgroundJobClient,
        IRecurringJobManager recurringJobManager,
        ILogger<AdvancedJobService> logger)
    {
        _backgroundJobClient = backgroundJobClient;
        _recurringJobManager = recurringJobManager;
        _logger = logger;
    }

    // Fire-and-forget with automatic retry
    public string QueueEmailJob(EmailRequest request, string queue = "emails")
    {
        return _backgroundJobClient.Enqueue<IEmailService>(service => 
            service.SendEmailWithRetryAsync(request, null, null, false, default));
    }

    // Delayed job execution
    public string ScheduleReminderEmail(string email, string subject, string message, TimeSpan delay)
    {
        return _backgroundJobClient.Schedule<IEmailService>(
            service => service.SendEmailAsync(email, subject, message, true, default),
            delay);
    }

    // Continuation jobs
    public string ProcessOrderWithContinuations(Order order)
    {
        // Step 1: Process payment
        var paymentJobId = _backgroundJobClient.Enqueue<IPaymentService>(
            service => service.ProcessPaymentAsync(order.Id));
        
        // Step 2: After payment succeeds, update inventory
        var inventoryJobId = _backgroundJobClient.ContinueJobWith<IInventoryService>(
            paymentJobId,
            service => service.UpdateInventoryAsync(order.Id));
        
        // Step 3: After inventory update, send confirmation
        return _backgroundJobClient.ContinueJobWith<IEmailService>(
            inventoryJobId,
            service => service.SendOrderConfirmationAsync(order.Id));
    }

    // Batch processing
    public void ProcessLargeDatasetInBatches(List<DataRecord> records, int batchSize = 100)
    {
        for (int i = 0; i < records.Count; i += batchSize)
        {
            var batch = records.Skip(i).Take(batchSize).ToList();
            var batchNumber = (i / batchSize) + 1;
            
            _backgroundJobClient.Enqueue<IDataProcessor>(
                service => service.ProcessBatchAsync(batch, batchNumber));
        }
    }

    // Recurring job with custom schedule
    public void SetupCustomRecurringJobs()
    {
        // Business hours only (9 AM - 5 PM, Monday-Friday)
        _recurringJobManager.AddOrUpdate<IHealthCheckService>(
            "business-hours-healthcheck",
            service => service.RunHealthChecksAsync(),
            "0 9-17 * * 1-5", // Every hour from 9 AM to 5 PM, Monday to Friday
            TimeZoneInfo.Local);

        // End-of-month processing
        _recurringJobManager.AddOrUpdate<IAccountingService>(
            "end-of-month",
            service => service.RunEndOfMonthProcessingAsync(),
            "0 2 L * *", // 2 AM on the last day of the month
            TimeZoneInfo.Local);
    }
}

// Email service with automatic retry and logging
public class EmailService : IEmailService
{
    private readonly ILogger<EmailService> _logger;
    private readonly ISmtpService _smtpService;

    public EmailService(ILogger<EmailService> logger, ISmtpService smtpService)
    {
        _logger = logger;
        _smtpService = smtpService;
    }

    [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
    [Queue("emails")]
    public async Task SendEmailWithRetryAsync(EmailRequest request, int? delaySeconds = null, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
    {
        if (delaySeconds.HasValue)
        {
            _logger.LogInformation("Delaying email send by {DelaySeconds} seconds", delaySeconds.Value);
            await Task.Delay(TimeSpan.FromSeconds(delaySeconds.Value), cancellationToken);
        }

        try
        {
            _logger.LogInformation("Sending email to {Recipient} with subject '{Subject}'", 
                request.To, request.Subject);

            var result = await _smtpService.SendAsync(request);

            if (result.Success)
            {
                _logger.LogInformation("Successfully sent email to {Recipient}", request.To);
                
                // Update job context if provided
                if (!string.IsNullOrEmpty(jobId))
                {
                    var connection = JobStorage.Current.GetConnection();
                    connection.SetJobParameter(jobId, "SentAt", DateTime.UtcNow);
                }
            }
            else
            {
                throw new InvalidOperationException($"Failed to send email: {result.Error}");
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to send email to {Recipient}", request.To);
            
            if (throwOnError)
                throw;
                
            // Re-throw to trigger Hangfire retry
            throw new JobPerformanceException("Email sending failed", ex);
        }
    }

    public async Task SendBulkEmailsAsync(List<EmailRequest> emails, IJobCancellationToken cancellationToken)
    {
        _logger.LogInformation("Starting bulk email send for {EmailCount} emails", emails.Count);

        var semaphore = new SemaphoreSlim(10); // Limit concurrent sends
        var tasks = emails.Select(async email =>
        {
            await semaphore.WaitAsync();
            try
            {
                cancellationToken.ThrowIfCancellationRequested();
                await SendEmailWithRetryAsync(email);
            }
            finally
            {
                semaphore.Release();
            }
        });

        await Task.WhenAll(tasks);
        _logger.LogInformation("Completed bulk email send");
    }
}

4.3 Real-time Job Monitoring and Management

  
    // Job monitoring service
public class JobMonitoringService
{
    private readonly IMonitoringApi _monitoringApi;
    private readonly ILogger<JobMonitoringService> _logger;

    public JobMonitoringService(IMonitoringApi monitoringApi, ILogger<JobMonitoringService> logger)
    {
        _monitoringApi = monitoringApi;
        _logger = logger;
    }

    public async Task<JobDashboard> GetJobDashboardAsync()
    {
        var dashboard = new JobDashboard
        {
            Timestamp = DateTime.UtcNow,
            Servers = (await _monitoringApi.Servers()).Length,
            Queues = await GetQueueStatisticsAsync(),
            Jobs = await GetJobStatisticsAsync(),
            FailedJobs = await GetRecentFailedJobsAsync(),
            ScheduledJobs = await GetScheduledJobsCountAsync()
        };

        return dashboard;
    }

    private async Task<List<QueueStats>> GetQueueStatisticsAsync()
    {
        var queues = new[] { "default", "emails", "reports", "critical" };
        var stats = new List<QueueStats>();

        foreach (var queue in queues)
        {
            var queueStats = await _monitoringApi.EnqueuedCount(queue);
            var failedStats = await _monitoringApi.FailedCount(queue);
            
            stats.Add(new QueueStats
            {
                QueueName = queue,
                EnqueuedCount = queueStats,
                FailedCount = failedStats,
                ProcessingCount = await GetProcessingCount(queue)
            });
        }

        return stats;
    }

    private async Task<JobStatistics> GetJobStatisticsAsync()
    {
        var stats = _monitoringApi.GetStatistics();
        
        return new JobStatistics
        {
            TotalSucceeded = stats.Succeeded,
            TotalFailed = stats.Failed,
            TotalDeleted = stats.Deleted,
            TotalRetries = stats.Retries,
            TotalEnqueued = stats.Enqueued
        };
    }

    private async Task<List<FailedJobInfo>> GetRecentFailedJobsAsync(int count = 50)
    {
        var failedJobs = new List<FailedJobInfo>();
        var jobs = _monitoringApi.FailedJobs(0, count);

        foreach (var job in jobs)
        {
            failedJobs.Add(new FailedJobInfo
            {
                JobId = job.Key,
                JobName = job.Value.Job.ToString(),
                FailedAt = job.Value.FailedAt,
                ExceptionMessage = job.Value.ExceptionMessage,
                ExceptionDetails = job.Value.ExceptionDetails
            });
        }

        return failedJobs;
    }

    private async Task<long> GetProcessingCount(string queue)
    {
        // Implementation depends on Hangfire version
        return await Task.FromResult(0L);
    }

    private async Task<long> GetScheduledJobsCountAsync()
    {
        var stats = _monitoringApi.GetStatistics();
        return stats.Scheduled;
    }
}

public class JobDashboard
{
    public DateTime Timestamp { get; set; }
    public int Servers { get; set; }
    public List<QueueStats> Queues { get; set; } = new();
    public JobStatistics Jobs { get; set; }
    public List<FailedJobInfo> FailedJobs { get; set; } = new();
    public long ScheduledJobs { get; set; }
}

public class QueueStats
{
    public string QueueName { get; set; }
    public long EnqueuedCount { get; set; }
    public long FailedCount { get; set; }
    public long ProcessingCount { get; set; }
}

public class JobStatistics
{
    public long TotalSucceeded { get; set; }
    public long TotalFailed { get; set; }
    public long TotalDeleted { get; set; }
    public long TotalRetries { get; set; }
    public long TotalEnqueued { get; set; }
}

public class FailedJobInfo
{
    public string JobId { get; set; }
    public string JobName { get; set; }
    public DateTime FailedAt { get; set; }
    public string ExceptionMessage { get; set; }
    public string ExceptionDetails { get; set; }
}

5. Quartz.NET Enterprise Scheduling

5.1 Quartz.NET Setup and Configuration

  
    // Program.cs - Quartz.NET Configuration
using Quartz;
using Quartz.Impl;
using Quartz.Spi;

var builder = WebApplication.CreateBuilder(args);

// Add Quartz services
builder.Services.AddQuartz(q =>
{
    // Use Scoped container for jobs
    q.UseMicrosoftDependencyInjectionScopedJobFactory();
    
    // Base Quartz configuration
    q.SchedulerId = "ECommerce-Scheduler";
    q.SchedulerName = "ECommerce Background Job Scheduler";
    
    // Configure thread pool
    q.UseThreadPool(x =>
    {
        x.MaxConcurrency = 10;
    });
    
    // Configure job store (SQL Server)
    q.UsePersistentStore(store =>
    {
        store.UseProperties = true;
        store.UseSqlServer(sqlServer =>
        {
            sqlServer.ConnectionString = builder.Configuration.GetConnectionString("QuartzConnection");
            sqlServer.TablePrefix = "QRTZ_";
        });
        store.UseJsonSerializer();
        store.UseClustering(c =>
        {
            c.CheckinMisfireThreshold = TimeSpan.FromSeconds(30);
            c.CheckinInterval = TimeSpan.FromSeconds(10);
        });
    });
    
    // Register jobs and triggers
    RegisterJobsAndTriggers(q);
});

// Add Quartz hosted service
builder.Services.AddQuartzHostedService(options =>
{
    options.WaitForJobsToComplete = true;
    options.StartDelay = TimeSpan.FromSeconds(5);
});

var app = builder.Build();

app.Run();

void RegisterJobsAndTriggers(IServiceCollectionQuartzConfigurator quartz)
{
    // Email processing job
    var emailJobKey = new JobKey("EmailProcessingJob");
    quartz.AddJob<EmailProcessingJob>(opts => opts.WithIdentity(emailJobKey));
    
    quartz.AddTrigger(opts => opts
        .ForJob(emailJobKey)
        .WithIdentity("EmailProcessingJob-Trigger")
        .WithCronSchedule("0 */5 * * * ?")); // Every 5 minutes

    // Report generation job
    var reportJobKey = new JobKey("ReportGenerationJob");
    quartz.AddJob<ReportGenerationJob>(opts => opts.WithIdentity(reportJobKey));
    
    quartz.AddTrigger(opts => opts
        .ForJob(reportJobKey)
        .WithIdentity("ReportGenerationJob-Trigger")
        .WithCronSchedule("0 0 2 * * ?")); // Daily at 2 AM

    // Database maintenance job
    var maintenanceJobKey = new JobKey("DatabaseMaintenanceJob");
    quartz.AddJob<DatabaseMaintenanceJob>(opts => opts.WithIdentity(maintenanceJobKey));
    
    quartz.AddTrigger(opts => opts
        .ForJob(maintenanceJobKey)
        .WithIdentity("DatabaseMaintenanceJob-Trigger")
        .WithCronSchedule("0 0 3 * * ?")); // Daily at 3 AM

    // Health check job
    var healthCheckJobKey = new JobKey("HealthCheckJob");
    quartz.AddJob<HealthCheckJob>(opts => opts.WithIdentity(healthCheckJobKey));
    
    quartz.AddTrigger(opts => opts
        .ForJob(healthCheckJobKey)
        .WithIdentity("HealthCheckJob-Trigger")
        .WithSimpleSchedule(x => x
            .WithIntervalInMinutes(10)
            .RepeatForever()));
}

5.2 Advanced Quartz.NET Jobs

  
    // Base job with common functionality
public abstract class BaseJob : IJob
{
    protected readonly ILogger Logger;
    protected readonly IServiceProvider ServiceProvider;

    protected BaseJob(ILogger logger, IServiceProvider serviceProvider)
    {
        Logger = logger;
        ServiceProvider = serviceProvider;
    }

    public async Task Execute(IJobExecutionContext context)
    {
        var jobName = GetType().Name;
        var jobId = context.JobDetail.Key.Name;
        var fireTime = context.FireTimeUtc.LocalDateTime;
        
        Logger.LogInformation("Starting job {JobName} ({JobId}) at {FireTime}", 
            jobName, jobId, fireTime);

        try
        {
            // Create scope for dependency injection
            using var scope = ServiceProvider.CreateScope();
            
            // Execute the actual job logic
            await ExecuteJob(context, scope.ServiceProvider);
            
            Logger.LogInformation("Completed job {JobName} ({JobId}) successfully", jobName, jobId);
        }
        catch (Exception ex)
        {
            Logger.LogError(ex, "Job {JobName} ({JobId}) failed with error: {ErrorMessage}", 
                jobName, jobId, ex.Message);
            
            // Determine if we should re-throw the exception
            if (ShouldRefire(context, ex))
            {
                throw new JobExecutionException(ex, true);
            }
        }
    }

    protected abstract Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider);
    
    protected virtual bool ShouldRefire(IJobExecutionContext context, Exception exception)
    {
        // Don't refire for certain exception types
        if (exception is BusinessRuleException)
            return false;
            
        // Refire if we haven't exceeded max attempts
        var refireCount = context.RefireCount;
        return refireCount < 3;
    }
}

// Email processing job
[DisallowConcurrentExecution]
[PersistJobDataAfterExecution]
public class EmailProcessingJob : BaseJob
{
    public EmailProcessingJob(ILogger<EmailProcessingJob> logger, IServiceProvider serviceProvider) 
        : base(logger, serviceProvider)
    {
    }

    protected override async Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider)
    {
        var emailService = serviceProvider.GetRequiredService<IEmailQueueService>();
        var emailSender = serviceProvider.GetRequiredService<IEmailSender>();
        
        // Get job data
        var batchSize = context.MergedJobDataMap.GetInt("BatchSize") ?? 50;
        var maxRetries = context.MergedJobDataMap.GetInt("MaxRetries") ?? 3;
        
        Logger.LogInformation("Processing email batch with size {BatchSize}", batchSize);
        
        var pendingEmails = await emailService.GetPendingEmailsAsync(batchSize);
        var processedCount = 0;
        var failedCount = 0;
        
        foreach (var email in pendingEmails)
        {
            try
            {
                if (email.RetryCount >= maxRetries)
                {
                    await emailService.MarkAsPermanentlyFailedAsync(email.Id, "Max retries exceeded");
                    failedCount++;
                    continue;
                }
                
                await emailSender.SendAsync(email.To, email.Subject, email.Body, email.IsHtml);
                await emailService.MarkAsCompletedAsync(email.Id);
                processedCount++;
            }
            catch (Exception ex)
            {
                Logger.LogWarning(ex, "Failed to send email {EmailId}", email.Id);
                await emailService.MarkAsFailedAsync(email.Id, ex.Message);
                failedCount++;
            }
        }
        
        // Update job data for monitoring
        context.JobDetail.JobDataMap["LastProcessedCount"] = processedCount;
        context.JobDetail.JobDataMap["LastFailedCount"] = failedCount;
        context.JobDetail.JobDataMap["LastRun"] = DateTime.UtcNow;
        
        Logger.LogInformation("Email processing completed: {Processed} processed, {Failed} failed", 
            processedCount, failedCount);
    }
}

// Report generation job
[DisallowConcurrentExecution]
public class ReportGenerationJob : BaseJob
{
    public ReportGenerationJob(ILogger<ReportGenerationJob> logger, IServiceProvider serviceProvider) 
        : base(logger, serviceProvider)
    {
    }

    protected override async Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider)
    {
        var reportService = serviceProvider.GetRequiredService<IReportService>();
        var notificationService = serviceProvider.GetRequiredService<INotificationService>();
        
        var reportDate = DateTime.UtcNow.AddDays(-1); // Yesterday's report
        var reportType = context.MergedJobDataMap.GetString("ReportType") ?? "Daily";
        
        Logger.LogInformation("Generating {ReportType} report for {ReportDate}", 
            reportType, reportDate.ToString("yyyy-MM-dd"));
        
        try
        {
            var report = await reportService.GenerateReportAsync(reportType, reportDate);
            
            if (report != null)
            {
                // Store report
                await reportService.SaveReportAsync(report);
                
                // Notify administrators
                await notificationService.NotifyAdminsAsync(
                    $"New {reportType} report generated", 
                    $"The {reportType} report for {reportDate:yyyy-MM-dd} has been generated successfully.");
                
                Logger.LogInformation("Successfully generated and saved {ReportType} report", reportType);
            }
            else
            {
                Logger.LogWarning("Report generation returned null for {ReportType}", reportType);
            }
        }
        catch (Exception ex)
        {
            Logger.LogError(ex, "Failed to generate {ReportType} report", reportType);
            throw;
        }
    }
}

// Database maintenance job
public class DatabaseMaintenanceJob : BaseJob
{
    public DatabaseMaintenanceJob(ILogger<DatabaseMaintenanceJob> logger, IServiceProvider serviceProvider) 
        : base(logger, serviceProvider)
    {
    }

    protected override async Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider)
    {
        var dbContext = serviceProvider.GetRequiredService<ApplicationDbContext>();
        var maintenanceService = serviceProvider.GetRequiredService<IDatabaseMaintenanceService>();
        
        Logger.LogInformation("Starting database maintenance tasks");
        
        var tasks = new[]
        {
            maintenanceService.CleanOldAuditLogsAsync(TimeSpan.FromDays(90)),
            maintenanceService.RebuildIndexesAsync(),
            maintenanceService.UpdateStatisticsAsync(),
            maintenanceService.CleanTempFilesAsync()
        };
        
        await Task.WhenAll(tasks);
        
        Logger.LogInformation("Database maintenance tasks completed successfully");
    }
}

5.3 Dynamic Job Scheduling

  
    // Dynamic job scheduling service
public interface IDynamicScheduler
{
    Task ScheduleJobAsync<T>(string jobName, string cronExpression, IDictionary<string, object> jobData = null) where T : IJob;
    Task ScheduleOneTimeJobAsync<T>(string jobName, DateTimeOffset startTime, IDictionary<string, object> jobData = null) where T : IJob;
    Task<bool> DeleteJobAsync(string jobName);
    Task<bool> PauseJobAsync(string jobName);
    Task<bool> ResumeJobAsync(string jobName);
    Task<bool> UpdateJobScheduleAsync(string jobName, string newCronExpression);
    Task<List<ScheduledJobInfo>> GetScheduledJobsAsync();
}

public class DynamicScheduler : IDynamicScheduler
{
    private readonly ISchedulerFactory _schedulerFactory;
    private readonly ILogger<DynamicScheduler> _logger;

    public DynamicScheduler(ISchedulerFactory schedulerFactory, ILogger<DynamicScheduler> logger)
    {
        _schedulerFactory = schedulerFactory;
        _logger = logger;
    }

    public async Task ScheduleJobAsync<T>(string jobName, string cronExpression, IDictionary<string, object> jobData = null) where T : IJob
    {
        var scheduler = await _schedulerFactory.GetScheduler();
        
        var jobKey = new JobKey(jobName);
        if (await scheduler.CheckExists(jobKey))
        {
            _logger.LogWarning("Job {JobName} already exists. Updating schedule.", jobName);
            await UpdateJobScheduleAsync(jobName, cronExpression);
            return;
        }

        var job = JobBuilder.Create<T>()
            .WithIdentity(jobKey)
            .UsingJobData(new JobDataMap(jobData ?? new Dictionary<string, object>()))
            .Build();

        var trigger = TriggerBuilder.Create()
            .WithIdentity($"{jobName}-Trigger")
            .WithCronSchedule(cronExpression)
            .Build();

        await scheduler.ScheduleJob(job, trigger);
        _logger.LogInformation("Scheduled job {JobName} with cron expression: {CronExpression}", jobName, cronExpression);
    }

    public async Task ScheduleOneTimeJobAsync<T>(string jobName, DateTimeOffset startTime, IDictionary<string, object> jobData = null) where T : IJob
    {
        var scheduler = await _schedulerFactory.GetScheduler();
        
        var jobKey = new JobKey(jobName);
        if (await scheduler.CheckExists(jobKey))
        {
            throw new InvalidOperationException($"Job {jobName} already exists");
        }

        var job = JobBuilder.Create<T>()
            .WithIdentity(jobKey)
            .UsingJobData(new JobDataMap(jobData ?? new Dictionary<string, object>()))
            .Build();

        var trigger = TriggerBuilder.Create()
            .WithIdentity($"{jobName}-Trigger")
            .StartAt(startTime)
            .Build();

        await scheduler.ScheduleJob(job, trigger);
        _logger.LogInformation("Scheduled one-time job {JobName} to run at {StartTime}", jobName, startTime);
    }

    public async Task<bool> DeleteJobAsync(string jobName)
    {
        var scheduler = await _schedulerFactory.GetScheduler();
        var jobKey = new JobKey(jobName);
        
        if (await scheduler.CheckExists(jobKey))
        {
            var result = await scheduler.DeleteJob(jobKey);
            _logger.LogInformation("Deleted job {JobName}: {Result}", jobName, result);
            return result;
        }
        
        _logger.LogWarning("Job {JobName} not found for deletion", jobName);
        return false;
    }

    public async Task<bool> PauseJobAsync(string jobName)
    {
        var scheduler = await _schedulerFactory.GetScheduler();
        var jobKey = new JobKey(jobName);
        
        if (await scheduler.CheckExists(jobKey))
        {
            await scheduler.PauseJob(jobKey);
            _logger.LogInformation("Paused job {JobName}", jobName);
            return true;
        }
        
        _logger.LogWarning("Job {JobName} not found for pausing", jobName);
        return false;
    }

    public async Task<bool> ResumeJobAsync(string jobName)
    {
        var scheduler = await _schedulerFactory.GetScheduler();
        var jobKey = new JobKey(jobName);
        
        if (await scheduler.CheckExists(jobKey))
        {
            await scheduler.ResumeJob(jobKey);
            _logger.LogInformation("Resumed job {JobName}", jobName);
            return true;
        }
        
        _logger.LogWarning("Job {JobName} not found for resuming", jobName);
        return false;
    }

    public async Task<bool> UpdateJobScheduleAsync(string jobName, string newCronExpression)
    {
        var scheduler = await _schedulerFactory.GetScheduler();
        var jobKey = new JobKey(jobName);
        var triggerKey = new TriggerKey($"{jobName}-Trigger");
        
        if (!await scheduler.CheckExists(jobKey) || !await scheduler.CheckExists(triggerKey))
        {
            _logger.LogWarning("Job or trigger not found for {JobName}", jobName);
            return false;
        }

        var newTrigger = TriggerBuilder.Create()
            .WithIdentity(triggerKey)
            .WithCronSchedule(newCronExpression)
            .Build();

        await scheduler.RescheduleJob(triggerKey, newTrigger);
        _logger.LogInformation("Updated schedule for job {JobName} to: {CronExpression}", jobName, newCronExpression);
        return true;
    }

    public async Task<List<ScheduledJobInfo>> GetScheduledJobsAsync()
    {
        var scheduler = await _schedulerFactory.GetScheduler();
        var jobGroups = await scheduler.GetJobGroupNames();
        var jobs = new List<ScheduledJobInfo>();

        foreach (var group in jobGroups)
        {
            var groupMatcher = GroupMatcher<JobKey>.GroupContains(group);
            var jobKeys = await scheduler.GetJobKeys(groupMatcher);
            
            foreach (var jobKey in jobKeys)
            {
                var detail = await scheduler.GetJobDetail(jobKey);
                var triggers = await scheduler.GetTriggersOfJob(jobKey);
                
                foreach (var trigger in triggers)
                {
                    var state = await scheduler.GetTriggerState(trigger.Key);
                    var nextFireTime = trigger.GetNextFireTimeUtc();
                    
                    jobs.Add(new ScheduledJobInfo
                    {
                        JobName = jobKey.Name,
                        Group = jobKey.Group,
                        Description = detail.Description,
                        JobType = detail.JobType.FullName,
                        TriggerState = state.ToString(),
                        NextFireTime = nextFireTime?.LocalDateTime,
                        PreviousFireTime = trigger.GetPreviousFireTimeUtc()?.LocalDateTime
                    });
                }
            }
        }

        return jobs;
    }
}

public class ScheduledJobInfo
{
    public string JobName { get; set; }
    public string Group { get; set; }
    public string Description { get; set; }
    public string JobType { get; set; }
    public string TriggerState { get; set; }
    public DateTime? NextFireTime { get; set; }
    public DateTime? PreviousFireTime { get; set; }
}

6. Coravel Simplified Scheduling

6.1 Coravel Setup and Basic Usage

  
    // Program.cs - Coravel Configuration
using Coravel;

var builder = WebApplication.CreateBuilder(args);

// Add Coravel services
builder.Services.AddScheduler();
builder.Services.AddQueue();

// Register background services
builder.Services.AddTransient<EmailNotificationInvokable>();
builder.Services.AddTransient<ReportGenerationInvokable>();
builder.Services.AddTransient<DataCleanupInvokable>();

// Register caching if needed
builder.Services.AddCache();

var app = builder.Build();

// Configure Coravel scheduling
app.Services.UseScheduler(scheduler =>
{
    // Email notifications every 10 minutes
    scheduler.Schedule<EmailNotificationInvokable>()
        .EveryMinute()
        .PreventOverlapping("EmailNotifications")
        .Weekday();

    // Daily report generation at 2 AM
    scheduler.Schedule<ReportGenerationInvokable>()
        .DailyAtHour(2)
        .Zoned(TimeZoneInfo.Local);

    // Data cleanup every Sunday at 3 AM
    scheduler.Schedule<DataCleanupInvokable>()
        .Cron("0 3 * * 0") // Sunday at 3 AM
        .Zoned(TimeZoneInfo.Local);

    // Hourly cache priming
    scheduler.Schedule<CachePrimingInvokable>()
        .Hourly()
        .RunOnceAtStart();

}).OnError(ex =>
{
    // Global error handling
    var logger = app.Services.GetRequiredService<ILogger<Program>>();
    logger.LogError(ex, "Scheduler error occurred");
});

app.Run();

6.2 Coravel Invocables and Queuing

  
    // Email notification invocable
public class EmailNotificationInvokable : IInvocable, ICancellableInvocable
{
    private readonly IEmailQueueService _emailQueueService;
    private readonly IEmailSender _emailSender;
    private readonly ILogger<EmailNotificationInvokable> _logger;

    public EmailNotificationInvokable(
        IEmailQueueService emailQueueService,
        IEmailSender emailSender,
        ILogger<EmailNotificationInvokable> logger)
    {
        _emailQueueService = emailQueueService;
        _emailSender = emailSender;
        _logger = logger;
    }

    public async Task Invoke()
    {
        _logger.LogInformation("Starting email notification processing");
        
        var pendingEmails = await _emailQueueService.GetPendingEmailsAsync(100);
        var processedCount = 0;
        var failedCount = 0;

        foreach (var email in pendingEmails)
        {
            try
            {
                await _emailQueueService.MarkAsProcessingAsync(email.Id);
                
                var result = await _emailSender.SendAsync(
                    email.To, 
                    email.Subject, 
                    email.Body, 
                    email.IsHtml);

                if (result.Success)
                {
                    await _emailQueueService.MarkAsCompletedAsync(email.Id);
                    processedCount++;
                }
                else
                {
                    await _emailQueueService.MarkAsFailedAsync(email.Id, result.Error);
                    failedCount++;
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to process email {EmailId}", email.Id);
                await _emailQueueService.MarkAsFailedAsync(email.Id, ex.Message);
                failedCount++;
            }
        }

        _logger.LogInformation("Email processing completed: {Processed} processed, {Failed} failed", 
            processedCount, failedCount);
    }

    public CancellationToken CancellationToken { get; set; }
}

// Report generation invocable
public class ReportGenerationInvokable : IInvocable
{
    private readonly IReportService _reportService;
    private readonly INotificationService _notificationService;
    private readonly ILogger<ReportGenerationInvokable> _logger;

    public ReportGenerationInvokable(
        IReportService reportService,
        INotificationService notificationService,
        ILogger<ReportGenerationInvokable> logger)
    {
        _reportService = reportService;
        _notificationService = notificationService;
        _logger = logger;
    }

    public async Task Invoke()
    {
        _logger.LogInformation("Starting daily report generation");
        
        var reportDate = DateTime.Today.AddDays(-1); // Yesterday's report
        var reportTypes = new[] { "Sales", "Inventory", "UserActivity" };

        foreach (var reportType in reportTypes)
        {
            try
            {
                _logger.LogInformation("Generating {ReportType} report for {ReportDate}", 
                    reportType, reportDate.ToString("yyyy-MM-dd"));

                var report = await _reportService.GenerateReportAsync(reportType, reportDate);
                
                if (report != null)
                {
                    await _reportService.SaveReportAsync(report);
                    
                    _logger.LogInformation("Successfully generated and saved {ReportType} report", reportType);
                }
                else
                {
                    _logger.LogWarning("Report generation returned null for {ReportType}", reportType);
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to generate {ReportType} report", reportType);
            }
        }

        // Notify administrators
        await _notificationService.NotifyAdminsAsync(
            "Daily Reports Generated",
            $"Daily reports for {reportDate:yyyy-MM-dd} have been generated successfully.");
        
        _logger.LogInformation("Daily report generation completed");
    }
}

// Data cleanup invocable
public class DataCleanupInvokable : IInvocable
{
    private readonly ApplicationDbContext _dbContext;
    private readonly IFileService _fileService;
    private readonly ILogger<DataCleanupInvokable> _logger;

    public DataCleanupInvokable(
        ApplicationDbContext dbContext,
        IFileService fileService,
        ILogger<DataCleanupInvokable> logger)
    {
        _dbContext = dbContext;
        _fileService = fileService;
        _logger = logger;
    }

    public async Task Invoke()
    {
        _logger.LogInformation("Starting weekly data cleanup");
        
        var tasks = new[]
        {
            CleanOldAuditLogsAsync(),
            CleanOldNotificationsAsync(),
            CleanTemporaryFilesAsync(),
            CleanExpiredSessionsAsync()
        };

        await Task.WhenAll(tasks);
        
        _logger.LogInformation("Weekly data cleanup completed successfully");
    }

    private async Task CleanOldAuditLogsAsync()
    {
        var cutoffDate = DateTime.UtcNow.AddDays(-90); // 90 days retention
        var oldLogs = await _dbContext.AuditLogs
            .Where(log => log.CreatedAt < cutoffDate)
            .ToListAsync();

        if (oldLogs.Any())
        {
            _dbContext.AuditLogs.RemoveRange(oldLogs);
            await _dbContext.SaveChangesAsync();
            _logger.LogInformation("Cleaned {Count} old audit logs", oldLogs.Count);
        }
    }

    private async Task CleanOldNotificationsAsync()
    {
        var cutoffDate = DateTime.UtcNow.AddDays(-30); // 30 days retention
        var oldNotifications = await _dbContext.Notifications
            .Where(n => n.CreatedAt < cutoffDate && n.IsRead)
            .ToListAsync();

        if (oldNotifications.Any())
        {
            _dbContext.Notifications.RemoveRange(oldNotifications);
            await _dbContext.SaveChangesAsync();
            _logger.LogInformation("Cleaned {Count} old notifications", oldNotifications.Count);
        }
    }

    private async Task CleanTemporaryFilesAsync()
    {
        var tempPath = Path.GetTempPath();
        var cutoffDate = DateTime.UtcNow.AddDays(-1);
        
        var filesDeleted = await _fileService.CleanOldFilesAsync(tempPath, "*.tmp", cutoffDate);
        _logger.LogInformation("Cleaned {Count} temporary files", filesDeleted);
    }

    private async Task CleanExpiredSessionsAsync()
    {
        var cutoffDate = DateTime.UtcNow.AddHours(-24); // 24 hours expiry
        var expiredSessions = await _dbContext.UserSessions
            .Where(s => s.LastActivity < cutoffDate)
            .ToListAsync();

        if (expiredSessions.Any())
        {
            _dbContext.UserSessions.RemoveRange(expiredSessions);
            await _dbContext.SaveChangesAsync();
            _logger.LogInformation("Cleaned {Count} expired sessions", expiredSessions.Count);
        }
    }
}

6.3 Coravel Queuing for Background Processing

  
    // Queue-based email service
public class QueueEmailService
{
    private readonly IQueue _queue;
    private readonly ILogger<QueueEmailService> _logger;

    public QueueEmailService(IQueue queue, ILogger<QueueEmailService> logger)
    {
        _queue = queue;
        _logger = logger;
    }

    public async Task QueueWelcomeEmailAsync(string email, string userName)
    {
        var emailData = new WelcomeEmailData
        {
            Email = email,
            UserName = userName,
            SentAt = DateTime.UtcNow
        };

        await _queue.QueueInvocableAsync<SendWelcomeEmailInvocable, WelcomeEmailData>(emailData);
        _logger.LogInformation("Queued welcome email for {Email}", email);
    }

    public async Task QueueBulkEmailsAsync(List<BulkEmailData> emails)
    {
        foreach (var email in emails)
        {
            await _queue.QueueInvocableAsync<SendBulkEmailInvocable, BulkEmailData>(email);
        }
        
        _logger.LogInformation("Queued {Count} bulk emails", emails.Count);
    }

    public async Task QueuePasswordResetEmailAsync(string email, string resetToken)
    {
        var emailData = new PasswordResetEmailData
        {
            Email = email,
            ResetToken = resetToken,
            ExpiresAt = DateTime.UtcNow.AddHours(24)
        };

        await _queue.QueueInvocableAsync<SendPasswordResetEmailInvocable, PasswordResetEmailData>(emailData);
        _logger.LogInformation("Queued password reset email for {Email}", email);
    }
}

// Welcome email invocable
public class SendWelcomeEmailInvocable : IInvocable, IInvocableWithPayload<WelcomeEmailData>
{
    private readonly IEmailSender _emailSender;
    private readonly ILogger<SendWelcomeEmailInvocable> _logger;

    public SendWelcomeEmailInvocable(IEmailSender emailSender, ILogger<SendWelcomeEmailInvocable> logger)
    {
        _emailSender = emailSender;
        _logger = logger;
    }

    public WelcomeEmailData Payload { get; set; }

    public async Task Invoke()
    {
        try
        {
            var subject = $"Welcome to Our Service, {Payload.UserName}!";
            var body = $@"
                <h1>Welcome aboard, {Payload.UserName}!</h1>
                <p>Thank you for joining our service. We're excited to have you!</p>
                <p>If you have any questions, don't hesitate to contact our support team.</p>
                <br>
                <p>Best regards,<br>The Team</p>";

            var result = await _emailSender.SendAsync(Payload.Email, subject, body, true);

            if (result.Success)
            {
                _logger.LogInformation("Successfully sent welcome email to {Email}", Payload.Email);
            }
            else
            {
                _logger.LogError("Failed to send welcome email to {Email}: {Error}", 
                    Payload.Email, result.Error);
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Error sending welcome email to {Email}", Payload.Email);
            throw;
        }
    }
}

// Bulk email invocable
public class SendBulkEmailInvocable : IInvocable, IInvocableWithPayload<BulkEmailData>
{
    private readonly IEmailSender _emailSender;
    private readonly ILogger<SendBulkEmailInvocable> _logger;

    public SendBulkEmailInvocable(IEmailSender emailSender, ILogger<SendBulkEmailInvocable> logger)
    {
        _emailSender = emailSender;
        _logger = logger;
    }

    public BulkEmailData Payload { get; set; }

    public async Task Invoke()
    {
        try
        {
            var result = await _emailSender.SendAsync(
                Payload.To, 
                Payload.Subject, 
                Payload.Body, 
                Payload.IsHtml);

            if (result.Success)
            {
                _logger.LogInformation("Successfully sent bulk email to {Email}", Payload.To);
            }
            else
            {
                _logger.LogError("Failed to send bulk email to {Email}: {Error}", 
                    Payload.To, result.Error);
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Error sending bulk email to {Email}", Payload.To);
            throw;
        }
    }
}

// Data models for queuing
public class WelcomeEmailData
{
    public string Email { get; set; }
    public string UserName { get; set; }
    public DateTime SentAt { get; set; }
}

public class BulkEmailData
{
    public string To { get; set; }
    public string Subject { get; set; }
    public string Body { get; set; }
    public bool IsHtml { get; set; }
}

public class PasswordResetEmailData
{
    public string Email { get; set; }
    public string ResetToken { get; set; }
    public DateTime ExpiresAt { get; set; }
}

7. Real-World E-Commerce Automation

7.1 Complete Order Processing Pipeline

  
    // Order processing service with background jobs
public class OrderProcessingService
{
    private readonly IBackgroundJobClient _backgroundJobClient;
    private readonly IRecurringJobManager _recurringJobManager;
    private readonly ILogger<OrderProcessingService> _logger;

    public OrderProcessingService(
        IBackgroundJobClient backgroundJobClient,
        IRecurringJobManager recurringJobManager,
        ILogger<OrderProcessingService> logger)
    {
        _backgroundJobClient = backgroundJobClient;
        _recurringJobManager = recurringJobManager;
        _logger = logger;
    }

    public async Task<string> ProcessOrderAsync(Order order)
    {
        _logger.LogInformation("Starting order processing for order {OrderId}", order.Id);

        try
        {
            // Step 1: Validate order (immediate)
            await ValidateOrderAsync(order);

            // Step 2: Process payment (background)
            var paymentJobId = _backgroundJobClient.Enqueue<IPaymentService>(
                service => service.ProcessPaymentAsync(order.Id, null, false, default));

            // Step 3: Update inventory after payment (continuation)
            var inventoryJobId = _backgroundJobClient.ContinueJobWith<IInventoryService>(
                paymentJobId,
                service => service.UpdateInventoryAsync(order.Id, null, false, default));

            // Step 4: Send confirmation after inventory update (continuation)
            var confirmationJobId = _backgroundJobClient.ContinueJobWith<IEmailService>(
                inventoryJobId,
                service => service.SendOrderConfirmationAsync(order.Id, null, false, default));

            // Step 5: Schedule follow-up email for 7 days later
            _backgroundJobClient.Schedule<IEmailService>(
                service => service.SendFollowUpEmailAsync(order.Id, null, false, default),
                TimeSpan.FromDays(7));

            _logger.LogInformation("Order {OrderId} processing pipeline started successfully", order.Id);
            return confirmationJobId;
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to process order {OrderId}", order.Id);
            throw;
        }
    }

    public async Task<string> ProcessBulkOrdersAsync(List<Order> orders)
    {
        _logger.LogInformation("Processing {OrderCount} orders in bulk", orders.Count);

        var batchId = Guid.NewGuid().ToString();
        var batchJobIds = new List<string>();

        foreach (var order in orders)
        {
            var jobId = _backgroundJobClient.Enqueue<IOrderService>(
                service => service.ProcessSingleOrderAsync(order.Id, batchId, null, false, default));
            
            batchJobIds.Add(jobId);
        }

        // Monitor batch completion
        _backgroundJobClient.Enqueue<IBatchMonitorService>(
            service => service.MonitorBatchCompletionAsync(batchId, batchJobIds, null, false, default));

        return batchId;
    }

    public void SetupRecurringOrderJobs()
    {
        // Daily order summary at 6 AM
        _recurringJobManager.AddOrUpdate<IOrderReportService>(
            "daily-order-summary",
            service => service.GenerateDailyOrderSummaryAsync(null, false, default),
            Cron.Daily(6, 0));

        // Abandoned cart cleanup every 6 hours
        _recurringJobManager.AddOrUpdate<ICartService>(
            "abandoned-cart-cleanup",
            service => service.CleanAbandonedCartsAsync(null, false, default),
            "0 */6 * * *");

        // Monthly sales report on 1st of month at 3 AM
        _recurringJobManager.AddOrUpdate<ISalesReportService>(
            "monthly-sales-report",
            service => service.GenerateMonthlySalesReportAsync(null, false, default),
            "0 3 1 * *");
    }

    private async Task ValidateOrderAsync(Order order)
    {
        // Perform synchronous validation
        if (order.TotalAmount <= 0)
            throw new InvalidOperationException("Order total must be greater than zero");

        if (string.IsNullOrEmpty(order.CustomerEmail))
            throw new InvalidOperationException("Customer email is required");

        // Check product availability
        foreach (var item in order.Items)
        {
            var available = await CheckProductAvailabilityAsync(item.ProductId, item.Quantity);
            if (!available)
                throw new InvalidOperationException($"Product {item.ProductId} is not available in requested quantity");
        }
    }

    private async Task<bool> CheckProductAvailabilityAsync(int productId, int quantity)
    {
        // Implementation depends on your data access
        await Task.Delay(10); // Simulate database call
        return quantity <= 100; // Simplified check
    }
}

// Payment service with retry logic
public class PaymentService : IPaymentService
{
    private readonly ILogger<PaymentService> _logger;
    private readonly IPaymentGateway _paymentGateway;

    public PaymentService(ILogger<PaymentService> logger, IPaymentGateway paymentGateway)
    {
        _logger = logger;
        _paymentGateway = paymentGateway;
    }

    [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
    [Queue("payments")]
    public async Task ProcessPaymentAsync(int orderId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
    {
        _logger.LogInformation("Processing payment for order {OrderId}", orderId);

        try
        {
            // Retrieve order details
            var order = await GetOrderAsync(orderId);
            
            // Process payment through gateway
            var paymentResult = await _paymentGateway.ProcessPaymentAsync(
                order.TotalAmount,
                order.Currency,
                order.PaymentMethod);

            if (paymentResult.Success)
            {
                await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Completed, paymentResult.TransactionId);
                _logger.LogInformation("Successfully processed payment for order {OrderId}", orderId);
            }
            else
            {
                await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Failed, null, paymentResult.Error);
                throw new InvalidOperationException($"Payment failed: {paymentResult.Error}");
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Payment processing failed for order {OrderId}", orderId);
            
            if (throwOnError)
                throw;
                
            throw new JobPerformanceException("Payment processing failed", ex);
        }
    }

    private async Task<Order> GetOrderAsync(int orderId)
    {
        // Implementation to retrieve order from database
        await Task.Delay(10);
        return new Order { Id = orderId, TotalAmount = 99.99m, Currency = "USD", PaymentMethod = "CreditCard" };
    }

    private async Task UpdateOrderPaymentStatusAsync(int orderId, PaymentStatus status, string transactionId = null, string error = null)
    {
        // Implementation to update order in database
        await Task.Delay(10);
        _logger.LogInformation("Updated order {OrderId} payment status to {Status}", orderId, status);
    }
}

public enum PaymentStatus
{
    Pending = 0,
    Completed = 1,
    Failed = 2,
    Refunded = 3
}

7.2 Inventory Management with Background Jobs

  
    // Inventory management service
public class InventoryService : IInventoryService
{
    private readonly IBackgroundJobClient _backgroundJobClient;
    private readonly ILogger<InventoryService> _logger;
    private readonly ApplicationDbContext _dbContext;

    public InventoryService(
        IBackgroundJobClient backgroundJobClient,
        ILogger<InventoryService> logger,
        ApplicationDbContext dbContext)
    {
        _backgroundJobClient = backgroundJobClient;
        _logger = logger;
        _dbContext = dbContext;
    }

    [AutomaticRetry(Attempts = 2)]
    [Queue("inventory")]
    public async Task UpdateInventoryAsync(int orderId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
    {
        _logger.LogInformation("Updating inventory for order {OrderId}", orderId);

        try
        {
            var order = await _dbContext.Orders
                .Include(o => o.Items)
                .ThenInclude(i => i.Product)
                .FirstOrDefaultAsync(o => o.Id == orderId, cancellationToken);

            if (order == null)
                throw new InvalidOperationException($"Order {orderId} not found");

            foreach (var item in order.Items)
            {
                var product = item.Product;
                
                if (product.StockQuantity < item.Quantity)
                {
                    throw new InvalidOperationException(
                        $"Insufficient stock for product {product.Name}. Available: {product.StockQuantity}, Requested: {item.Quantity}");
                }

                // Update stock
                product.StockQuantity -= item.Quantity;
                product.LastStockUpdate = DateTime.UtcNow;

                // Log inventory change
                var inventoryLog = new InventoryLog
                {
                    ProductId = product.Id,
                    OrderId = orderId,
                    QuantityChange = -item.Quantity,
                    NewStockLevel = product.StockQuantity,
                    Reason = "Order fulfillment",
                    CreatedAt = DateTime.UtcNow
                };

                _dbContext.InventoryLogs.Add(inventoryLog);

                // Check for low stock and trigger reorder if needed
                if (product.StockQuantity <= product.ReorderLevel)
                {
                    _backgroundJobClient.Enqueue<IInventoryService>(
                        service => service.TriggerReorderAsync(product.Id, null, false, default));
                }
            }

            await _dbContext.SaveChangesAsync(cancellationToken);
            _logger.LogInformation("Successfully updated inventory for order {OrderId}", orderId);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to update inventory for order {OrderId}", orderId);
            
            if (throwOnError)
                throw;
                
            throw new JobPerformanceException("Inventory update failed", ex);
        }
    }

    public async Task TriggerReorderAsync(int productId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
    {
        _logger.LogInformation("Triggering reorder for product {ProductId}", productId);

        try
        {
            var product = await _dbContext.Products.FindAsync(new object[] { productId }, cancellationToken);
            
            if (product == null)
                throw new InvalidOperationException($"Product {productId} not found");

            // Create purchase order or notify supplier
            var reorderRequest = new ReorderRequest
            {
                ProductId = product.Id,
                ProductName = product.Name,
                CurrentStock = product.StockQuantity,
                ReorderLevel = product.ReorderLevel,
                ReorderQuantity = product.ReorderQuantity,
                RequestedAt = DateTime.UtcNow
            };

            // Notify purchasing department
            _backgroundJobClient.Enqueue<INotificationService>(
                service => service.NotifyPurchasingDepartmentAsync(reorderRequest, null, false, default));

            // Update product reorder status
            product.LastReorderDate = DateTime.UtcNow;
            await _dbContext.SaveChangesAsync(cancellationToken);

            _logger.LogInformation("Reorder triggered for product {ProductName}", product.Name);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to trigger reorder for product {ProductId}", productId);
            
            if (throwOnError)
                throw;
        }
    }

    public void ScheduleInventoryJobs()
    {
        // Daily stock level check at 8 AM
        _backgroundJobClient.Recurring<IInventoryService>(
            "daily-stock-check",
            service => service.CheckStockLevelsAsync(null, false, default),
            Cron.Daily(8, 0));

        // Weekly inventory report on Monday at 6 AM
        _backgroundJobClient.Recurring<IInventoryReportService>(
            "weekly-inventory-report",
            service => service.GenerateWeeklyInventoryReportAsync(null, false, default),
            Cron.Weekly(DayOfWeek.Monday, 6, 0));

        // Monthly dead stock identification on 1st at 2 AM
        _backgroundJobClient.Recurring<IInventoryAnalysisService>(
            "dead-stock-analysis",
            service => service.IdentifyDeadStockAsync(null, false, default),
            "0 2 1 * *");
    }

    public async Task CheckStockLevelsAsync(string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
    {
        _logger.LogInformation("Starting daily stock level check");

        try
        {
            var lowStockProducts = await _dbContext.Products
                .Where(p => p.StockQuantity <= p.ReorderLevel && p.IsActive)
                .ToListAsync(cancellationToken);

            foreach (var product in lowStockProducts)
            {
                _backgroundJobClient.Enqueue<IInventoryService>(
                    service => service.TriggerReorderAsync(product.Id, null, false, default));
            }

            _logger.LogInformation("Stock level check completed. {Count} products need reordering", lowStockProducts.Count);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to check stock levels");
            
            if (throwOnError)
                throw;
        }
    }
}

public class ReorderRequest
{
    public int ProductId { get; set; }
    public string ProductName { get; set; }
    public int CurrentStock { get; set; }
    public int ReorderLevel { get; set; }
    public int ReorderQuantity { get; set; }
    public DateTime RequestedAt { get; set; }
}

public class InventoryLog
{
    public int Id { get; set; }
    public int ProductId { get; set; }
    public int? OrderId { get; set; }
    public int QuantityChange { get; set; }
    public int NewStockLevel { get; set; }
    public string Reason { get; set; }
    public DateTime CreatedAt { get; set; }
}

8. Advanced Patterns & Scenarios

8.1 Distributed Job Processing

  
    // Distributed job coordinator
public interface IDistributedJobCoordinator
{
    Task<string> ScheduleDistributedJobAsync<T>(string jobName, object payload, string queue = "default") where T : IBackgroundJob;
    Task<bool> CancelDistributedJobAsync(string jobId);
    Task<DistributedJobStatus> GetJobStatusAsync(string jobId);
    Task<List<DistributedJobInfo>> GetActiveJobsAsync();
}

public class DistributedJobCoordinator : IDistributedJobCoordinator
{
    private readonly IServiceProvider _serviceProvider;
    private readonly ILogger<DistributedJobCoordinator> _logger;
    private readonly IBackgroundJobClient _backgroundJobClient;
    private readonly ApplicationDbContext _dbContext;

    public DistributedJobCoordinator(
        IServiceProvider serviceProvider,
        ILogger<DistributedJobCoordinator> logger,
        IBackgroundJobClient backgroundJobClient,
        ApplicationDbContext dbContext)
    {
        _serviceProvider = serviceProvider;
        _logger = logger;
        _backgroundJobClient = backgroundJobClient;
        _dbContext = dbContext;
    }

    public async Task<string> ScheduleDistributedJobAsync<T>(string jobName, object payload, string queue = "default") where T : IBackgroundJob
    {
        var jobId = Guid.NewGuid().ToString();
        
        var distributedJob = new DistributedJob
        {
            Id = jobId,
            JobName = jobName,
            JobType = typeof(T).FullName,
            Payload = JsonSerializer.Serialize(payload),
            Queue = queue,
            Status = DistributedJobStatus.Scheduled,
            CreatedAt = DateTime.UtcNow,
            UpdatedAt = DateTime.UtcNow
        };

        _dbContext.DistributedJobs.Add(distributedJob);
        await _dbContext.SaveChangesAsync();

        // Schedule the actual background job
        var hangfireJobId = _backgroundJobClient.Enqueue<T>(
            job => job.ExecuteAsync(jobId, null, false, default));

        // Store Hangfire job ID reference
        distributedJob.HangfireJobId = hangfireJobId;
        await _dbContext.SaveChangesAsync();

        _logger.LogInformation("Scheduled distributed job {JobName} with ID {JobId}", jobName, jobId);
        
        return jobId;
    }

    public async Task<bool> CancelDistributedJobAsync(string jobId)
    {
        var job = await _dbContext.DistributedJobs
            .FirstOrDefaultAsync(j => j.Id == jobId);

        if (job == null)
        {
            _logger.LogWarning("Distributed job {JobId} not found for cancellation", jobId);
            return false;
        }

        if (job.Status == DistributedJobStatus.Running || job.Status == DistributedJobStatus.Scheduled)
        {
            if (!string.IsNullOrEmpty(job.HangfireJobId))
            {
                _backgroundJobClient.Delete(job.HangfireJobId);
            }

            job.Status = DistributedJobStatus.Cancelled;
            job.UpdatedAt = DateTime.UtcNow;
            await _dbContext.SaveChangesAsync();

            _logger.LogInformation("Cancelled distributed job {JobId}", jobId);
            return true;
        }

        _logger.LogWarning("Distributed job {JobId} cannot be cancelled in current status: {Status}", jobId, job.Status);
        return false;
    }

    public async Task<DistributedJobStatus> GetJobStatusAsync(string jobId)
    {
        var job = await _dbContext.DistributedJobs
            .FirstOrDefaultAsync(j => j.Id == jobId);

        return job?.Status ?? DistributedJobStatus.NotFound;
    }

    public async Task<List<DistributedJobInfo>> GetActiveJobsAsync()
    {
        var activeJobs = await _dbContext.DistributedJobs
            .Where(j => j.Status == DistributedJobStatus.Scheduled || j.Status == DistributedJobStatus.Running)
            .OrderByDescending(j => j.CreatedAt)
            .Take(100)
            .Select(j => new DistributedJobInfo
            {
                Id = j.Id,
                JobName = j.JobName,
                Status = j.Status,
                CreatedAt = j.CreatedAt,
                UpdatedAt = j.UpdatedAt,
                Queue = j.Queue
            })
            .ToListAsync();

        return activeJobs;
    }
}

// Base distributed job interface
public interface IBackgroundJob
{
    Task ExecuteAsync(string jobId, string hangfireJobId = null, bool throwOnError = false, CancellationToken cancellationToken = default);
}

// Example distributed job implementation
public class DataExportJob : IBackgroundJob
{
    private readonly ILogger<DataExportJob> _logger;
    private readonly ApplicationDbContext _dbContext;
    private readonly IExportService _exportService;
    private readonly IDistributedJobCoordinator _jobCoordinator;

    public DataExportJob(
        ILogger<DataExportJob> logger,
        ApplicationDbContext dbContext,
        IExportService exportService,
        IDistributedJobCoordinator jobCoordinator)
    {
        _logger = logger;
        _dbContext = dbContext;
        _exportService = exportService;
        _jobCoordinator = jobCoordinator;
    }

    public async Task ExecuteAsync(string jobId, string hangfireJobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
    {
        _logger.LogInformation("Starting data export job {JobId}", jobId);

        try
        {
            // Update job status to running
            await UpdateJobStatusAsync(jobId, DistributedJobStatus.Running);

            // Retrieve job parameters
            var job = await _dbContext.DistributedJobs.FindAsync(new object[] { jobId }, cancellationToken);
            if (job == null)
                throw new InvalidOperationException($"Job {jobId} not found");

            var parameters = JsonSerializer.Deserialize<DataExportParameters>(job.Payload);

            // Execute export
            var exportResult = await _exportService.ExportDataAsync(
                parameters.DataType,
                parameters.StartDate,
                parameters.EndDate,
                parameters.Format);

            if (exportResult.Success)
            {
                await UpdateJobStatusAsync(jobId, DistributedJobStatus.Completed, exportResult.FilePath);
                _logger.LogInformation("Data export job {JobId} completed successfully", jobId);
            }
            else
            {
                await UpdateJobStatusAsync(jobId, DistributedJobStatus.Failed, error: exportResult.Error);
                throw new InvalidOperationException($"Export failed: {exportResult.Error}");
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Data export job {JobId} failed", jobId);
            await UpdateJobStatusAsync(jobId, DistributedJobStatus.Failed, error: ex.Message);
            
            if (throwOnError)
                throw;
        }
    }

    private async Task UpdateJobStatusAsync(string jobId, DistributedJobStatus status, string result = null, string error = null)
    {
        var job = await _dbContext.DistributedJobs.FindAsync(jobId);
        if (job != null)
        {
            job.Status = status;
            job.Result = result;
            job.Error = error;
            job.UpdatedAt = DateTime.UtcNow;
            
            if (status == DistributedJobStatus.Completed)
                job.CompletedAt = DateTime.UtcNow;

            await _dbContext.SaveChangesAsync();
        }
    }
}

public class DataExportParameters
{
    public string DataType { get; set; } // "Orders", "Products", "Users"
    public DateTime StartDate { get; set; }
    public DateTime EndDate { get; set; }
    public string Format { get; set; } // "CSV", "Excel", "JSON"
}

public class DistributedJob
{
    public string Id { get; set; }
    public string JobName { get; set; }
    public string JobType { get; set; }
    public string Payload { get; set; }
    public string Queue { get; set; }
    public DistributedJobStatus Status { get; set; }
    public string HangfireJobId { get; set; }
    public string Result { get; set; }
    public string Error { get; set; }
    public DateTime CreatedAt { get; set; }
    public DateTime UpdatedAt { get; set; }
    public DateTime? CompletedAt { get; set; }
}

public enum DistributedJobStatus
{
    Scheduled = 0,
    Running = 1,
    Completed = 2,
    Failed = 3,
    Cancelled = 4,
    NotFound = 5
}

public class DistributedJobInfo
{
    public string Id { get; set; }
    public string JobName { get; set; }
    public DistributedJobStatus Status { get; set; }
    public string Queue { get; set; }
    public DateTime CreatedAt { get; set; }
    public DateTime UpdatedAt { get; set; }
}

8.2 Job Chaining and Workflows

  
    // Workflow orchestration service
public interface IWorkflowOrchestrator
{
    Task<string> StartOrderFulfillmentWorkflowAsync(int orderId);
    Task<string> StartDataProcessingWorkflowAsync(DataProcessingRequest request);
    Task<bool> CancelWorkflowAsync(string workflowId);
    Task<WorkflowStatus> GetWorkflowStatusAsync(string workflowId);
}

public class WorkflowOrchestrator : IWorkflowOrchestrator
{
    private readonly IBackgroundJobClient _backgroundJobClient;
    private readonly ILogger<WorkflowOrchestrator> _logger;
    private readonly ApplicationDbContext _dbContext;

    public WorkflowOrchestrator(
        IBackgroundJobClient backgroundJobClient,
        ILogger<WorkflowOrchestrator> logger,
        ApplicationDbContext dbContext)
    {
        _backgroundJobClient = backgroundJobClient;
        _logger = logger;
        _dbContext = dbContext;
    }

    public async Task<string> StartOrderFulfillmentWorkflowAsync(int orderId)
    {
        var workflowId = Guid.NewGuid().ToString();
        
        var workflow = new Workflow
        {
            Id = workflowId,
            Type = WorkflowType.OrderFulfillment,
            Status = WorkflowStatus.Running,
            ReferenceId = orderId.ToString(),
            CreatedAt = DateTime.UtcNow,
            UpdatedAt = DateTime.UtcNow
        };

        _dbContext.Workflows.Add(workflow);
        await _dbContext.SaveChangesAsync();

        // Start the workflow chain
        var paymentJobId = _backgroundJobClient.Enqueue<IPaymentService>(
            service => service.ProcessPaymentAsync(orderId, workflowId, null, false, default));

        _logger.LogInformation("Started order fulfillment workflow {WorkflowId} for order {OrderId}", workflowId, orderId);
        
        return workflowId;
    }

    public async Task<string> StartDataProcessingWorkflowAsync(DataProcessingRequest request)
    {
        var workflowId = Guid.NewGuid().ToString();
        
        var workflow = new Workflow
        {
            Id = workflowId,
            Type = WorkflowType.DataProcessing,
            Status = WorkflowStatus.Running,
            ReferenceId = request.Source,
            CreatedAt = DateTime.UtcNow,
            UpdatedAt = DateTime.UtcNow
        };

        _dbContext.Workflows.Add(workflow);
        await _dbContext.SaveChangesAsync();

        // Chain data processing steps
        var extractionJobId = _backgroundJobClient.Enqueue<IDataExtractionService>(
            service => service.ExtractDataAsync(request.Source, workflowId, null, false, default));

        var transformationJobId = _backgroundJobClient.ContinueJobWith<IDataTransformationService>(
            extractionJobId,
            service => service.TransformDataAsync(workflowId, null, false, default));

        var loadingJobId = _backgroundJobClient.ContinueJobWith<IDataLoadingService>(
            transformationJobId,
            service => service.LoadDataAsync(request.Destination, workflowId, null, false, default));

        var notificationJobId = _backgroundJobClient.ContinueJobWith<INotificationService>(
            loadingJobId,
            service => service.NotifyDataProcessingCompleteAsync(workflowId, null, false, default));

        _logger.LogInformation("Started data processing workflow {WorkflowId}", workflowId);
        
        return workflowId;
    }

    public async Task<bool> CancelWorkflowAsync(string workflowId)
    {
        var workflow = await _dbContext.Workflows
            .FirstOrDefaultAsync(w => w.Id == workflowId);

        if (workflow == null)
        {
            _logger.LogWarning("Workflow {WorkflowId} not found for cancellation", workflowId);
            return false;
        }

        if (workflow.Status == WorkflowStatus.Running)
        {
            // Cancel all associated jobs
            var jobs = await _dbContext.WorkflowJobs
                .Where(j => j.WorkflowId == workflowId && j.Status == WorkflowJobStatus.Running)
                .ToListAsync();

            foreach (var job in jobs)
            {
                if (!string.IsNullOrEmpty(job.HangfireJobId))
                {
                    _backgroundJobClient.Delete(job.HangfireJobId);
                }
                
                job.Status = WorkflowJobStatus.Cancelled;
                job.UpdatedAt = DateTime.UtcNow;
            }

            workflow.Status = WorkflowStatus.Cancelled;
            workflow.UpdatedAt = DateTime.UtcNow;
            await _dbContext.SaveChangesAsync();

            _logger.LogInformation("Cancelled workflow {WorkflowId}", workflowId);
            return true;
        }

        _logger.LogWarning("Workflow {WorkflowId} cannot be cancelled in current status: {Status}", workflowId, workflow.Status);
        return false;
    }

    public async Task<WorkflowStatus> GetWorkflowStatusAsync(string workflowId)
    {
        var workflow = await _dbContext.Workflows
            .FirstOrDefaultAsync(w => w.Id == workflowId);

        return workflow?.Status ?? WorkflowStatus.NotFound;
    }
}

// Workflow-aware job base class
public abstract class WorkflowJobBase
{
    protected readonly ILogger Logger;
    protected readonly ApplicationDbContext DbContext;

    protected WorkflowJobBase(ILogger logger, ApplicationDbContext dbContext)
    {
        Logger = logger;
        DbContext = dbContext;
    }

    protected async Task UpdateWorkflowJobStatusAsync(string workflowId, string jobName, WorkflowJobStatus status, string result = null, string error = null)
    {
        var workflowJob = await DbContext.WorkflowJobs
            .FirstOrDefaultAsync(j => j.WorkflowId == workflowId && j.JobName == jobName);

        if (workflowJob == null)
        {
            workflowJob = new WorkflowJob
            {
                WorkflowId = workflowId,
                JobName = jobName,
                Status = status,
                Result = result,
                Error = error,
                CreatedAt = DateTime.UtcNow,
                UpdatedAt = DateTime.UtcNow
            };
            DbContext.WorkflowJobs.Add(workflowJob);
        }
        else
        {
            workflowJob.Status = status;
            workflowJob.Result = result;
            workflowJob.Error = error;
            workflowJob.UpdatedAt = DateTime.UtcNow;
        }

        await DbContext.SaveChangesAsync();
    }

    protected async Task UpdateWorkflowStatusAsync(string workflowId, WorkflowStatus status)
    {
        var workflow = await DbContext.Workflows.FindAsync(workflowId);
        if (workflow != null)
        {
            workflow.Status = status;
            workflow.UpdatedAt = DateTime.UtcNow;
            await DbContext.SaveChangesAsync();
        }
    }
}

// Example workflow job implementation
public class PaymentServiceWithWorkflow : WorkflowJobBase, IPaymentService
{
    private readonly IPaymentGateway _paymentGateway;

    public PaymentServiceWithWorkflow(
        ILogger<PaymentServiceWithWorkflow> logger,
        ApplicationDbContext dbContext,
        IPaymentGateway paymentGateway) 
        : base(logger, dbContext)
    {
        _paymentGateway = paymentGateway;
    }

    public async Task ProcessPaymentAsync(int orderId, string workflowId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
    {
        const string jobName = "ProcessPayment";
        
        Logger.LogInformation("Processing payment for order {OrderId} in workflow {WorkflowId}", orderId, workflowId);

        try
        {
            await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Running);

            // Retrieve order and process payment
            var order = await GetOrderAsync(orderId);
            var paymentResult = await _paymentGateway.ProcessPaymentAsync(
                order.TotalAmount, 
                order.Currency, 
                order.PaymentMethod);

            if (paymentResult.Success)
            {
                await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Completed, paymentResult.TransactionId);
                await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Completed, paymentResult.TransactionId);
                Logger.LogInformation("Payment processed successfully for order {OrderId}", orderId);
            }
            else
            {
                await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Failed, null, paymentResult.Error);
                await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Failed, error: paymentResult.Error);
                await UpdateWorkflowStatusAsync(workflowId, WorkflowStatus.Failed);
                throw new InvalidOperationException($"Payment failed: {paymentResult.Error}");
            }
        }
        catch (Exception ex)
        {
            Logger.LogError(ex, "Payment processing failed for order {OrderId}", orderId);
            await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Failed, error: ex.Message);
            await UpdateWorkflowStatusAsync(workflowId, WorkflowStatus.Failed);
            
            if (throwOnError)
                throw;
        }
    }

    private async Task<Order> GetOrderAsync(int orderId)
    {
        // Implementation to retrieve order
        await Task.Delay(10);
        return new Order { Id = orderId, TotalAmount = 99.99m, Currency = "USD", PaymentMethod = "CreditCard" };
    }

    private async Task UpdateOrderPaymentStatusAsync(int orderId, PaymentStatus status, string transactionId = null, string error = null)
    {
        // Implementation to update order
        await Task.Delay(10);
    }
}

public class Workflow
{
    public string Id { get; set; }
    public WorkflowType Type { get; set; }
    public WorkflowStatus Status { get; set; }
    public string ReferenceId { get; set; }
    public DateTime CreatedAt { get; set; }
    public DateTime UpdatedAt { get; set; }
    public DateTime? CompletedAt { get; set; }
    
    public ICollection<WorkflowJob> Jobs { get; set; } = new List<WorkflowJob>();
}

public class WorkflowJob
{
    public int Id { get; set; }
    public string WorkflowId { get; set; }
    public string JobName { get; set; }
    public string HangfireJobId { get; set; }
    public WorkflowJobStatus Status { get; set; }
    public string Result { get; set; }
    public string Error { get; set; }
    public DateTime CreatedAt { get; set; }
    public DateTime UpdatedAt { get; set; }
    
    public Workflow Workflow { get; set; }
}

public enum WorkflowType
{
    OrderFulfillment = 0,
    DataProcessing = 1,
    ReportGeneration = 2,
    DataMigration = 3
}

public enum WorkflowStatus
{
    Running = 0,
    Completed = 1,
    Failed = 2,
    Cancelled = 3,
    NotFound = 4
}

public enum WorkflowJobStatus
{
    Pending = 0,
    Running = 1,
    Completed = 2,
    Failed = 3,
    Cancelled = 4
}

public class DataProcessingRequest
{
    public string Source { get; set; }
    public string Destination { get; set; }
    public string Format { get; set; }
}

9. Monitoring & Diagnostics

9.1 Comprehensive Job Monitoring

  
    // Job monitoring and alerting service
public interface IJobMonitor
{
    Task<JobHealthReport> GetHealthReportAsync();
    Task<List<JobExecutionRecord>> GetRecentExecutionsAsync(int count = 50);
    Task<JobStatistics> GetJobStatisticsAsync(DateTime fromDate, DateTime toDate);
    Task<bool> CheckJobHealthAsync(string jobName);
    Task SendAlertAsync(string jobName, string message, AlertLevel level);
}

public class JobMonitor : IJobMonitor
{
    private readonly IMonitoringApi _monitoringApi;
    private readonly ILogger<JobMonitor> _logger;
    private readonly ApplicationDbContext _dbContext;
    private readonly INotificationService _notificationService;

    public JobMonitor(
        IMonitoringApi monitoringApi,
        ILogger<JobMonitor> logger,
        ApplicationDbContext dbContext,
        INotificationService notificationService)
    {
        _monitoringApi = monitoringApi;
        _logger = logger;
        _dbContext = dbContext;
        _notificationService = notificationService;
    }

    public async Task<JobHealthReport> GetHealthReportAsync()
    {
        var report = new JobHealthReport
        {
            GeneratedAt = DateTime.UtcNow,
            ServerCount = (await _monitoringApi.Servers()).Length,
            QueueStats = await GetQueueStatisticsAsync(),
            FailedJobs = await GetRecentFailedJobsAsync(20),
            LongRunningJobs = await GetLongRunningJobsAsync(),
            SystemMetrics = await GetSystemMetricsAsync()
        };

        report.OverallHealth = CalculateOverallHealth(report);
        
        return report;
    }

    public async Task<List<JobExecutionRecord>> GetRecentExecutionsAsync(int count = 50)
    {
        var executions = new List<JobExecutionRecord>();
        
        // Get succeeded jobs
        var succeededJobs = _monitoringApi.SucceededJobs(0, count / 2);
        executions.AddRange(succeededJobs.Select(job => new JobExecutionRecord
        {
            JobId = job.Key,
            JobName = job.Value.Job.ToString(),
            Status = JobExecutionStatus.Succeeded,
            StartedAt = job.Value.SucceededAt?.Add(-job.Value.TotalDuration) ?? DateTime.MinValue,
            CompletedAt = job.Value.SucceededAt,
            Duration = job.Value.TotalDuration
        }));

        // Get failed jobs
        var failedJobs = _monitoringApi.FailedJobs(0, count / 2);
        executions.AddRange(failedJobs.Select(job => new JobExecutionRecord
        {
            JobId = job.Key,
            JobName = job.Value.Job.ToString(),
            Status = JobExecutionStatus.Failed,
            StartedAt = job.Value.FailedAt?.Add(-TimeSpan.FromSeconds(30)) ?? DateTime.MinValue, // Estimate
            CompletedAt = job.Value.FailedAt,
            Duration = TimeSpan.FromSeconds(30), // Estimate
            Error = job.Value.ExceptionMessage
        }));

        return executions.OrderByDescending(e => e.CompletedAt).Take(count).ToList();
    }

    public async Task<JobStatistics> GetJobStatisticsAsync(DateTime fromDate, DateTime toDate)
    {
        var stats = new JobStatistics
        {
            Period = new DateRange(fromDate, toDate),
            TotalExecutions = await GetTotalExecutionsAsync(fromDate, toDate),
            SuccessfulExecutions = await GetSuccessfulExecutionsAsync(fromDate, toDate),
            FailedExecutions = await GetFailedExecutionsAsync(fromDate, toDate),
            AverageDuration = await GetAverageDurationAsync(fromDate, toDate),
            MaxDuration = await GetMaxDurationAsync(fromDate, toDate)
        };

        stats.SuccessRate = stats.TotalExecutions > 0 
            ? (double)stats.SuccessfulExecutions / stats.TotalExecutions * 100 
            : 0;

        return stats;
    }

    public async Task<bool> CheckJobHealthAsync(string jobName)
    {
        try
        {
            var recentExecutions = await GetRecentExecutionsAsync(10);
            var jobExecutions = recentExecutions
                .Where(e => e.JobName.Contains(jobName))
                .ToList();

            if (!jobExecutions.Any())
                return true; // No recent executions, considered healthy

            var failureRate = (double)jobExecutions.Count(e => e.Status == JobExecutionStatus.Failed) / jobExecutions.Count;
            
            return failureRate < 0.2; // Consider unhealthy if more than 20% failures
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Error checking health for job {JobName}", jobName);
            return false;
        }
    }

    public async Task SendAlertAsync(string jobName, string message, AlertLevel level)
    {
        var alert = new JobAlert
        {
            JobName = jobName,
            Message = message,
            Level = level,
            CreatedAt = DateTime.UtcNow,
            IsAcknowledged = false
        };

        _dbContext.JobAlerts.Add(alert);
        await _dbContext.SaveChangesAsync();

        // Send immediate notification for critical alerts
        if (level == AlertLevel.Critical)
        {
            await _notificationService.NotifyAdminsAsync(
                $"CRITICAL: Job {jobName} Alert",
                message);
        }

        _logger.LogWarning("Job alert created: {JobName} - {Message} (Level: {Level})", jobName, message, level);
    }

    private async Task<List<QueueStatistics>> GetQueueStatisticsAsync()
    {
        var queues = new[] { "default", "emails", "reports", "critical", "payments", "inventory" };
        var stats = new List<QueueStatistics>();

        foreach (var queue in queues)
        {
            stats.Add(new QueueStatistics
            {
                QueueName = queue,
                EnqueuedCount = await _monitoringApi.EnqueuedCount(queue),
                FailedCount = await _monitoringApi.FailedCount(queue),
                ScheduledCount = await _monitoringApi.ScheduledCount(queue)
            });
        }

        return stats;
    }

    private async Task<List<FailedJobInfo>> GetRecentFailedJobsAsync(int count)
    {
        var failedJobs = _monitoringApi.FailedJobs(0, count);
        return failedJobs.Select(job => new FailedJobInfo
        {
            JobId = job.Key,
            JobName = job.Value.Job.ToString(),
            FailedAt = job.Value.FailedAt?.LocalDateTime ?? DateTime.MinValue,
            ExceptionMessage = job.Value.ExceptionMessage,
            ExceptionDetails = job.Value.ExceptionDetails,
            InFailedState = job.Value.InFailedState
        }).ToList();
    }

    private async Task<List<LongRunningJob>> GetLongRunningJobsAsync()
    {
        // Implementation to identify jobs running longer than expected
        // This would require custom tracking
        return new List<LongRunningJob>();
    }

    private async Task<SystemMetrics> GetSystemMetricsAsync()
    {
        // Implementation to get system-level metrics
        return new SystemMetrics
        {
            MemoryUsage = GC.GetTotalMemory(false),
            ActiveThreads = System.Diagnostics.Process.GetCurrentProcess().Threads.Count,
            CpuUsage = 0 // Would require more sophisticated monitoring
        };
    }

    private JobHealth CalculateOverallHealth(JobHealthReport report)
    {
        if (report.FailedJobs.Count > 10)
            return JobHealth.Critical;
        
        if (report.FailedJobs.Count > 5)
            return JobHealth.Warning;
            
        return JobHealth.Healthy;
    }

    // Additional helper methods for statistics...
    private async Task<int> GetTotalExecutionsAsync(DateTime fromDate, DateTime toDate)
    {
        // Implementation to get total job executions from storage
        return await Task.FromResult(1000);
    }

    private async Task<int> GetSuccessfulExecutionsAsync(DateTime fromDate, DateTime toDate)
    {
        // Implementation to get successful job executions from storage
        return await Task.FromResult(950);
    }

    private async Task<int> GetFailedExecutionsAsync(DateTime fromDate, DateTime toDate)
    {
        // Implementation to get failed job executions from storage
        return await Task.FromResult(50);
    }

    private async Task<TimeSpan> GetAverageDurationAsync(DateTime fromDate, DateTime toDate)
    {
        // Implementation to calculate average job duration
        return await Task.FromResult(TimeSpan.FromSeconds(30));
    }

    private async Task<TimeSpan> GetMaxDurationAsync(DateTime fromDate, DateTime toDate)
    {
        // Implementation to get maximum job duration
        return await Task.FromResult(TimeSpan.FromMinutes(5));
    }
}

public class JobHealthReport
{
    public DateTime GeneratedAt { get; set; }
    public JobHealth OverallHealth { get; set; }
    public int ServerCount { get; set; }
    public List<QueueStatistics> QueueStats { get; set; } = new();
    public List<FailedJobInfo> FailedJobs { get; set; } = new();
    public List<LongRunningJob> LongRunningJobs { get; set; } = new();
    public SystemMetrics SystemMetrics { get; set; }
}

public class JobExecutionRecord
{
    public string JobId { get; set; }
    public string JobName { get; set; }
    public JobExecutionStatus Status { get; set; }
    public DateTime StartedAt { get; set; }
    public DateTime? CompletedAt { get; set; }
    public TimeSpan? Duration { get; set; }
    public string Error { get; set; }
}

public class JobStatistics
{
    public DateRange Period { get; set; }
    public int TotalExecutions { get; set; }
    public int SuccessfulExecutions { get; set; }
    public int FailedExecutions { get; set; }
    public double SuccessRate { get; set; }
    public TimeSpan AverageDuration { get; set; }
    public TimeSpan MaxDuration { get; set; }
}

public class QueueStatistics
{
    public string QueueName { get; set; }
    public long EnqueuedCount { get; set; }
    public long FailedCount { get; set; }
    public long ScheduledCount { get; set; }
}

public class SystemMetrics
{
    public long MemoryUsage { get; set; }
    public int ActiveThreads { get; set; }
    public double CpuUsage { get; set; }
}

public class LongRunningJob
{
    public string JobId { get; set; }
    public string JobName { get; set; }
    public DateTime StartedAt { get; set; }
    public TimeSpan Duration { get; set; }
    public TimeSpan ExpectedDuration { get; set; }
}

public class JobAlert
{
    public int Id { get; set; }
    public string JobName { get; set; }
    public string Message { get; set; }
    public AlertLevel Level { get; set; }
    public DateTime CreatedAt { get; set; }
    public bool IsAcknowledged { get; set; }
}

public enum JobHealth
{
    Healthy = 0,
    Warning = 1,
    Critical = 2
}

public enum JobExecutionStatus
{
    Succeeded = 0,
    Failed = 1,
    Running = 2
}

public enum AlertLevel
{
    Info = 0,
    Warning = 1,
    Critical = 2
}

public record DateRange(DateTime Start, DateTime End);

9.2 Performance Monitoring and Metrics