![automation]()
Previous article: Testing Mastery: Catch Bugs Early with Comprehensive Testing in ASP.NET Core (Part- 29 of 40)
Table of Contents
The Background Jobs Revolution
Core Background Processing Concepts
Building with IHostedService
Hangfire Mastery
Quartz.NET Enterprise Scheduling
Coravel Simplified Scheduling
Real-World E-Commerce Automation
Advanced Patterns & Scenarios
Monitoring & Diagnostics
Production Ready Deployment
1. The Background Jobs Revolution
Why Background Jobs Matter in Modern Applications
In today's digital landscape, users expect instant responses and seamless experiences. However, many tasks in web applications are time-consuming and would degrade user experience if processed synchronously. This is where background jobs revolutionize application architecture.
Real-World Scenarios:
Sending welcome emails to new users
Generating monthly sales reports
Processing image and video uploads
Data synchronization between systems
Cleaning up temporary files
Sending push notifications
The Evolution of Background Processing
Traditional Approach:
csharp
// Problem: Blocking the main thread
public async Task<IActionResult> ProcessOrder(Order order)
{
// This blocks the user for several seconds
await _emailService.SendOrderConfirmation(order);
await _reportService.GenerateInvoice(order);
await _analyticsService.TrackOrder(order);
return Ok();
}
Modern Background Jobs Approach
// Solution: Immediate response with background processing
public async Task<IActionResult> ProcessOrder(Order order)
{
// Immediate response
_backgroundJobService.Enqueue(() => _emailService.SendOrderConfirmation(order));
_backgroundJobService.Enqueue(() => _reportService.GenerateInvoice(order));
_backgroundJobService.Schedule(() => _analyticsService.TrackOrder(order), TimeSpan.FromMinutes(30));
return Ok("Order processing started!");
}
2. Core Background Processing Concepts
2.1 Understanding Different Job Types
public enum JobType
{
// Fire-and-forget: Execute once, immediately
FireAndForget,
// Delayed: Execute once, after specified delay
Delayed,
// Recurring: Execute repeatedly on a schedule
Recurring,
// Continuation: Execute after another job completes
Continuation,
// Batch: Execute multiple jobs as a unit
Batch
}
public class JobCharacteristics
{
public TimeSpan ExpectedDuration { get; set; }
public int RetryCount { get; set; }
public bool IsIdempotent { get; set; }
public JobPriority Priority { get; set; }
public DateTime? Timeout { get; set; }
}
public enum JobPriority
{
Low = 0,
Normal = 1,
High = 2,
Critical = 3
}
2.2 Storage Considerations for Background Jobs
// Job Storage Options Configuration
public class JobStorageConfig
{
public StorageType Type { get; set; }
public string ConnectionString { get; set; }
public TimeSpan JobExpiration { get; set; } = TimeSpan.FromDays(30);
public int QueuePollInterval { get; set; } = 1000; // ms
}
public enum StorageType
{
InMemory, // Development only
SQLServer, // Enterprise applications
PostgreSQL, // Cross-platform
Redis, // High performance
MongoDB // Document storage
}
3. Building with IHostedService
3.1 Custom Background Service Implementation
// Basic background service template
public abstract class BackgroundService : IHostedService, IDisposable
{
private Task _executingTask;
private readonly CancellationTokenSource _stoppingCts = new();
private readonly ILogger<BackgroundService> _logger;
protected BackgroundService(ILogger<BackgroundService> logger)
{
_logger = logger;
}
protected abstract Task ExecuteAsync(CancellationToken stoppingToken);
public virtual Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Background service {ServiceName} is starting", GetType().Name);
_executingTask = ExecuteAsync(_stoppingCts.Token);
if (_executingTask.IsCompleted)
{
return _executingTask;
}
return Task.CompletedTask;
}
public virtual async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Background service {ServiceName} is stopping", GetType().Name);
if (_executingTask == null)
{
return;
}
try
{
_stoppingCts.Cancel();
}
finally
{
await Task.WhenAny(_executingTask, Task.Delay(Timeout.Infinite, cancellationToken));
}
}
public virtual void Dispose()
{
_stoppingCts.Cancel();
_stoppingCts?.Dispose();
}
}
3.2 Email Notification Service
// Real-world email background service
public class EmailNotificationService : BackgroundService
{
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<EmailNotificationService> _logger;
private readonly IConfiguration _configuration;
private readonly TimeSpan _processingInterval = TimeSpan.FromSeconds(30);
public EmailNotificationService(
IServiceProvider serviceProvider,
ILogger<EmailNotificationService> logger,
IConfiguration configuration)
{
_serviceProvider = serviceProvider;
_logger = logger;
_configuration = configuration;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Email Notification Service started");
while (!stoppingToken.IsCancellationRequested)
{
try
{
await ProcessPendingEmails(stoppingToken);
await Task.Delay(_processingInterval, stoppingToken);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogError(ex, "Error occurred processing emails");
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); // Backoff on error
}
}
_logger.LogInformation("Email Notification Service stopped");
}
private async Task ProcessPendingEmails(CancellationToken stoppingToken)
{
using var scope = _serviceProvider.CreateScope();
var emailQueueService = scope.ServiceProvider.GetRequiredService<IEmailQueueService>();
var emailSender = scope.ServiceProvider.GetRequiredService<IEmailSender>();
var pendingEmails = await emailQueueService.GetPendingEmailsAsync(50);
_logger.LogInformation("Processing {EmailCount} pending emails", pendingEmails.Count);
var tasks = pendingEmails.Select(email => ProcessSingleEmail(email, emailSender, emailQueueService, stoppingToken));
await Task.WhenAll(tasks);
}
private async Task ProcessSingleEmail(
EmailQueueItem email,
IEmailSender emailSender,
IEmailQueueService emailQueueService,
CancellationToken stoppingToken)
{
try
{
// Mark as processing
await emailQueueService.MarkAsProcessingAsync(email.Id);
// Send email
var result = await emailSender.SendAsync(
email.To,
email.Subject,
email.Body,
email.IsHtml);
if (result.Success)
{
await emailQueueService.MarkAsCompletedAsync(email.Id);
_logger.LogInformation("Successfully sent email to {Recipient}", email.To);
}
else
{
await emailQueueService.MarkAsFailedAsync(email.Id, result.Error);
_logger.LogWarning("Failed to send email to {Recipient}: {Error}", email.To, result.Error);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing email {EmailId} to {Recipient}", email.Id, email.To);
await emailQueueService.MarkAsFailedAsync(email.Id, ex.Message);
}
}
}
// Supporting models and interfaces
public interface IEmailQueueService
{
Task<List<EmailQueueItem>> GetPendingEmailsAsync(int batchSize);
Task MarkAsProcessingAsync(Guid emailId);
Task MarkAsCompletedAsync(Guid emailId);
Task MarkAsFailedAsync(Guid emailId, string error);
Task<Guid> QueueEmailAsync(string to, string subject, string body, bool isHtml = true);
}
public class EmailQueueItem
{
public Guid Id { get; set; }
public string To { get; set; }
public string Subject { get; set; }
public string Body { get; set; }
public bool IsHtml { get; set; }
public EmailStatus Status { get; set; }
public int RetryCount { get; set; }
public string Error { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime? ProcessedAt { get; set; }
}
public enum EmailStatus
{
Pending = 0,
Processing = 1,
Completed = 2,
Failed = 3
}
public class EmailResult
{
public bool Success { get; set; }
public string Error { get; set; }
public string MessageId { get; set; }
}
3.3 Database Maintenance Service
// Automated database maintenance service
public class DatabaseMaintenanceService : BackgroundService
{
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<DatabaseMaintenanceService> _logger;
private readonly List<MaintenanceTask> _maintenanceTasks;
public DatabaseMaintenanceService(
IServiceProvider serviceProvider,
ILogger<DatabaseMaintenanceService> logger)
{
_serviceProvider = serviceProvider;
_logger = logger;
_maintenanceTasks = new List<MaintenanceTask>
{
new MaintenanceTask
{
Name = "Clean Old Audit Logs",
Schedule = TimeSpan.FromHours(24),
LastRun = DateTime.MinValue,
Action = CleanOldAuditLogs
},
new MaintenanceTask
{
Name = "Rebuild Indexes",
Schedule = TimeSpan.FromDays(7),
LastRun = DateTime.MinValue,
Action = RebuildIndexes
},
new MaintenanceTask
{
Name = "Update Statistics",
Schedule = TimeSpan.FromDays(1),
LastRun = DateTime.MinValue,
Action = UpdateStatistics
},
new MaintenanceTask
{
Name = "Clean Temporary Files",
Schedule = TimeSpan.FromHours(6),
LastRun = DateTime.MinValue,
Action = CleanTemporaryFiles
}
};
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Database Maintenance Service started");
while (!stoppingToken.IsCancellationRequested)
{
try
{
await ProcessMaintenanceTasks(stoppingToken);
await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken); // Check every 5 minutes
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogError(ex, "Error in database maintenance service");
await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken);
}
}
_logger.LogInformation("Database Maintenance Service stopped");
}
private async Task ProcessMaintenanceTasks(CancellationToken stoppingToken)
{
var now = DateTime.UtcNow;
var tasksToRun = _maintenanceTasks
.Where(task => now - task.LastRun >= task.Schedule)
.ToList();
if (!tasksToRun.Any())
return;
_logger.LogInformation("Running {TaskCount} maintenance tasks", tasksToRun.Count);
using var scope = _serviceProvider.CreateScope();
foreach (var task in tasksToRun)
{
if (stoppingToken.IsCancellationRequested)
break;
try
{
_logger.LogInformation("Starting maintenance task: {TaskName}", task.Name);
await task.Action(scope.ServiceProvider, stoppingToken);
task.LastRun = now;
_logger.LogInformation("Completed maintenance task: {TaskName}", task.Name);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error running maintenance task: {TaskName}", task.Name);
}
}
}
private async Task CleanOldAuditLogs(IServiceProvider services, CancellationToken cancellationToken)
{
var dbContext = services.GetRequiredService<ApplicationDbContext>();
var retentionDays = 90; // Keep logs for 90 days
var cutoffDate = DateTime.UtcNow.AddDays(-retentionDays);
var oldLogs = await dbContext.AuditLogs
.Where(log => log.CreatedAt < cutoffDate)
.ToListAsync(cancellationToken);
if (oldLogs.Any())
{
dbContext.AuditLogs.RemoveRange(oldLogs);
await dbContext.SaveChangesAsync(cancellationToken);
_logger.LogInformation("Cleaned {LogCount} old audit logs", oldLogs.Count);
}
}
private async Task RebuildIndexes(IServiceProvider services, CancellationToken cancellationToken)
{
var dbContext = services.GetRequiredService<ApplicationDbContext>();
// Rebuild indexes for large tables
var tables = new[] { "Orders", "Products", "Users", "AuditLogs" };
foreach (var table in tables)
{
var sql = $"ALTER INDEX ALL ON [{table}] REBUILD";
await dbContext.Database.ExecuteSqlRawAsync(sql, cancellationToken);
_logger.LogInformation("Rebuilt indexes for table: {TableName}", table);
}
}
private async Task UpdateStatistics(IServiceProvider services, CancellationToken cancellationToken)
{
var dbContext = services.GetRequiredService<ApplicationDbContext>();
await dbContext.Database.ExecuteSqlRawAsync("EXEC sp_updatestats", cancellationToken);
_logger.LogInformation("Updated database statistics");
}
private async Task CleanTemporaryFiles(IServiceProvider services, CancellationToken cancellationToken)
{
var fileService = services.GetRequiredService<IFileService>();
var tempPath = Path.GetTempPath();
var cutoffDate = DateTime.UtcNow.AddDays(-1);
var filesDeleted = await fileService.CleanOldFilesAsync(tempPath, "*.tmp", cutoffDate);
_logger.LogInformation("Cleaned {FileCount} temporary files", filesDeleted);
}
}
public class MaintenanceTask
{
public string Name { get; set; }
public TimeSpan Schedule { get; set; }
public DateTime LastRun { get; set; }
public Func<IServiceProvider, CancellationToken, Task> Action { get; set; }
}
4. Hangfire Mastery
4.1 Hangfire Setup and Configuration
// Program.cs - Hangfire Configuration
using Hangfire;
using Hangfire.SqlServer;
var builder = WebApplication.CreateBuilder(args);
// Add Hangfire services
builder.Services.AddHangfire(configuration => configuration
.SetDataCompatibilityLevel(CompatibilityLevel.Version_170)
.UseSimpleAssemblyNameTypeSerializer()
.UseRecommendedSerializerSettings()
.UseSqlServerStorage(builder.Configuration.GetConnectionString("HangfireConnection"), new SqlServerStorageOptions
{
CommandBatchMaxTimeout = TimeSpan.FromMinutes(5),
SlidingInvisibilityTimeout = TimeSpan.FromMinutes(5),
QueuePollInterval = TimeSpan.Zero,
UseRecommendedIsolationLevel = true,
DisableGlobalLocks = true,
SchemaName = "Hangfire"
}));
// Add the processing server as IHostedService
builder.Services.AddHangfireServer(options =>
{
options.WorkerCount = Environment.ProcessorCount * 2;
options.Queues = new[] { "default", "emails", "reports", "critical" };
options.ServerName = $"Server_{Environment.MachineName}";
});
// Register job-related services
builder.Services.AddScoped<IEmailService, EmailService>();
builder.Services.AddScoped<IReportService, ReportService>();
builder.Services.AddScoped<IDataProcessingService, DataProcessingService>();
var app = builder.Build();
// Configure Hangfire dashboard (only in development or for authorized users)
if (app.Environment.IsDevelopment())
{
app.UseHangfireDashboard("/hangfire", new DashboardOptions
{
Authorization = new[] { new HangfireAuthorizationFilter() },
DashboardTitle = "ECommerce Background Jobs",
StatsPollingInterval = 10000
});
}
else
{
app.UseHangfireDashboard("/hangfire", new DashboardOptions
{
Authorization = new[] { new HangfireAuthorizationFilter() },
IsReadOnlyFunc = context => false
});
}
// Configure recurring jobs
ConfigureRecurringJobs();
app.Run();
void ConfigureRecurringJobs()
{
// Daily cleanup job
RecurringJob.AddOrUpdate<DatabaseMaintenanceService>(
"daily-cleanup",
service => service.CleanOldDataAsync(),
Cron.Daily(2, 0), // 2 AM daily
TimeZoneInfo.Local);
// Hourly statistics update
RecurringJob.AddOrUpdate<StatisticsService>(
"update-statistics",
service => service.UpdateStatisticsAsync(),
Cron.Hourly,
TimeZoneInfo.Local);
// Weekly report generation
RecurringJob.AddOrUpdate<ReportService>(
"weekly-reports",
service => service.GenerateWeeklyReportsAsync(),
Cron.Weekly(DayOfWeek.Monday, 3, 0), // Monday 3 AM
TimeZoneInfo.Local);
}
// Hangfire authorization filter
public class HangfireAuthorizationFilter : IDashboardAuthorizationFilter
{
public bool Authorize(DashboardContext context)
{
var httpContext = context.GetHttpContext();
// Allow all in development
if (httpContext.Request.Host.Host.Contains("localhost"))
return true;
// Implement your authorization logic here
return httpContext.User.Identity.IsAuthenticated &&
httpContext.User.IsInRole("Administrator");
}
}
4.2 Advanced Job Scenarios with Hangfire
// Advanced job service with retry policies and monitoring
public class AdvancedJobService
{
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly IRecurringJobManager _recurringJobManager;
private readonly ILogger<AdvancedJobService> _logger;
public AdvancedJobService(
IBackgroundJobClient backgroundJobClient,
IRecurringJobManager recurringJobManager,
ILogger<AdvancedJobService> logger)
{
_backgroundJobClient = backgroundJobClient;
_recurringJobManager = recurringJobManager;
_logger = logger;
}
// Fire-and-forget with automatic retry
public string QueueEmailJob(EmailRequest request, string queue = "emails")
{
return _backgroundJobClient.Enqueue<IEmailService>(service =>
service.SendEmailWithRetryAsync(request, null, null, false, default));
}
// Delayed job execution
public string ScheduleReminderEmail(string email, string subject, string message, TimeSpan delay)
{
return _backgroundJobClient.Schedule<IEmailService>(
service => service.SendEmailAsync(email, subject, message, true, default),
delay);
}
// Continuation jobs
public string ProcessOrderWithContinuations(Order order)
{
// Step 1: Process payment
var paymentJobId = _backgroundJobClient.Enqueue<IPaymentService>(
service => service.ProcessPaymentAsync(order.Id));
// Step 2: After payment succeeds, update inventory
var inventoryJobId = _backgroundJobClient.ContinueJobWith<IInventoryService>(
paymentJobId,
service => service.UpdateInventoryAsync(order.Id));
// Step 3: After inventory update, send confirmation
return _backgroundJobClient.ContinueJobWith<IEmailService>(
inventoryJobId,
service => service.SendOrderConfirmationAsync(order.Id));
}
// Batch processing
public void ProcessLargeDatasetInBatches(List<DataRecord> records, int batchSize = 100)
{
for (int i = 0; i < records.Count; i += batchSize)
{
var batch = records.Skip(i).Take(batchSize).ToList();
var batchNumber = (i / batchSize) + 1;
_backgroundJobClient.Enqueue<IDataProcessor>(
service => service.ProcessBatchAsync(batch, batchNumber));
}
}
// Recurring job with custom schedule
public void SetupCustomRecurringJobs()
{
// Business hours only (9 AM - 5 PM, Monday-Friday)
_recurringJobManager.AddOrUpdate<IHealthCheckService>(
"business-hours-healthcheck",
service => service.RunHealthChecksAsync(),
"0 9-17 * * 1-5", // Every hour from 9 AM to 5 PM, Monday to Friday
TimeZoneInfo.Local);
// End-of-month processing
_recurringJobManager.AddOrUpdate<IAccountingService>(
"end-of-month",
service => service.RunEndOfMonthProcessingAsync(),
"0 2 L * *", // 2 AM on the last day of the month
TimeZoneInfo.Local);
}
}
// Email service with automatic retry and logging
public class EmailService : IEmailService
{
private readonly ILogger<EmailService> _logger;
private readonly ISmtpService _smtpService;
public EmailService(ILogger<EmailService> logger, ISmtpService smtpService)
{
_logger = logger;
_smtpService = smtpService;
}
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
[Queue("emails")]
public async Task SendEmailWithRetryAsync(EmailRequest request, int? delaySeconds = null, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
{
if (delaySeconds.HasValue)
{
_logger.LogInformation("Delaying email send by {DelaySeconds} seconds", delaySeconds.Value);
await Task.Delay(TimeSpan.FromSeconds(delaySeconds.Value), cancellationToken);
}
try
{
_logger.LogInformation("Sending email to {Recipient} with subject '{Subject}'",
request.To, request.Subject);
var result = await _smtpService.SendAsync(request);
if (result.Success)
{
_logger.LogInformation("Successfully sent email to {Recipient}", request.To);
// Update job context if provided
if (!string.IsNullOrEmpty(jobId))
{
var connection = JobStorage.Current.GetConnection();
connection.SetJobParameter(jobId, "SentAt", DateTime.UtcNow);
}
}
else
{
throw new InvalidOperationException($"Failed to send email: {result.Error}");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to send email to {Recipient}", request.To);
if (throwOnError)
throw;
// Re-throw to trigger Hangfire retry
throw new JobPerformanceException("Email sending failed", ex);
}
}
public async Task SendBulkEmailsAsync(List<EmailRequest> emails, IJobCancellationToken cancellationToken)
{
_logger.LogInformation("Starting bulk email send for {EmailCount} emails", emails.Count);
var semaphore = new SemaphoreSlim(10); // Limit concurrent sends
var tasks = emails.Select(async email =>
{
await semaphore.WaitAsync();
try
{
cancellationToken.ThrowIfCancellationRequested();
await SendEmailWithRetryAsync(email);
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(tasks);
_logger.LogInformation("Completed bulk email send");
}
}
4.3 Real-time Job Monitoring and Management
// Job monitoring service
public class JobMonitoringService
{
private readonly IMonitoringApi _monitoringApi;
private readonly ILogger<JobMonitoringService> _logger;
public JobMonitoringService(IMonitoringApi monitoringApi, ILogger<JobMonitoringService> logger)
{
_monitoringApi = monitoringApi;
_logger = logger;
}
public async Task<JobDashboard> GetJobDashboardAsync()
{
var dashboard = new JobDashboard
{
Timestamp = DateTime.UtcNow,
Servers = (await _monitoringApi.Servers()).Length,
Queues = await GetQueueStatisticsAsync(),
Jobs = await GetJobStatisticsAsync(),
FailedJobs = await GetRecentFailedJobsAsync(),
ScheduledJobs = await GetScheduledJobsCountAsync()
};
return dashboard;
}
private async Task<List<QueueStats>> GetQueueStatisticsAsync()
{
var queues = new[] { "default", "emails", "reports", "critical" };
var stats = new List<QueueStats>();
foreach (var queue in queues)
{
var queueStats = await _monitoringApi.EnqueuedCount(queue);
var failedStats = await _monitoringApi.FailedCount(queue);
stats.Add(new QueueStats
{
QueueName = queue,
EnqueuedCount = queueStats,
FailedCount = failedStats,
ProcessingCount = await GetProcessingCount(queue)
});
}
return stats;
}
private async Task<JobStatistics> GetJobStatisticsAsync()
{
var stats = _monitoringApi.GetStatistics();
return new JobStatistics
{
TotalSucceeded = stats.Succeeded,
TotalFailed = stats.Failed,
TotalDeleted = stats.Deleted,
TotalRetries = stats.Retries,
TotalEnqueued = stats.Enqueued
};
}
private async Task<List<FailedJobInfo>> GetRecentFailedJobsAsync(int count = 50)
{
var failedJobs = new List<FailedJobInfo>();
var jobs = _monitoringApi.FailedJobs(0, count);
foreach (var job in jobs)
{
failedJobs.Add(new FailedJobInfo
{
JobId = job.Key,
JobName = job.Value.Job.ToString(),
FailedAt = job.Value.FailedAt,
ExceptionMessage = job.Value.ExceptionMessage,
ExceptionDetails = job.Value.ExceptionDetails
});
}
return failedJobs;
}
private async Task<long> GetProcessingCount(string queue)
{
// Implementation depends on Hangfire version
return await Task.FromResult(0L);
}
private async Task<long> GetScheduledJobsCountAsync()
{
var stats = _monitoringApi.GetStatistics();
return stats.Scheduled;
}
}
public class JobDashboard
{
public DateTime Timestamp { get; set; }
public int Servers { get; set; }
public List<QueueStats> Queues { get; set; } = new();
public JobStatistics Jobs { get; set; }
public List<FailedJobInfo> FailedJobs { get; set; } = new();
public long ScheduledJobs { get; set; }
}
public class QueueStats
{
public string QueueName { get; set; }
public long EnqueuedCount { get; set; }
public long FailedCount { get; set; }
public long ProcessingCount { get; set; }
}
public class JobStatistics
{
public long TotalSucceeded { get; set; }
public long TotalFailed { get; set; }
public long TotalDeleted { get; set; }
public long TotalRetries { get; set; }
public long TotalEnqueued { get; set; }
}
public class FailedJobInfo
{
public string JobId { get; set; }
public string JobName { get; set; }
public DateTime FailedAt { get; set; }
public string ExceptionMessage { get; set; }
public string ExceptionDetails { get; set; }
}
5. Quartz.NET Enterprise Scheduling
5.1 Quartz.NET Setup and Configuration
// Program.cs - Quartz.NET Configuration
using Quartz;
using Quartz.Impl;
using Quartz.Spi;
var builder = WebApplication.CreateBuilder(args);
// Add Quartz services
builder.Services.AddQuartz(q =>
{
// Use Scoped container for jobs
q.UseMicrosoftDependencyInjectionScopedJobFactory();
// Base Quartz configuration
q.SchedulerId = "ECommerce-Scheduler";
q.SchedulerName = "ECommerce Background Job Scheduler";
// Configure thread pool
q.UseThreadPool(x =>
{
x.MaxConcurrency = 10;
});
// Configure job store (SQL Server)
q.UsePersistentStore(store =>
{
store.UseProperties = true;
store.UseSqlServer(sqlServer =>
{
sqlServer.ConnectionString = builder.Configuration.GetConnectionString("QuartzConnection");
sqlServer.TablePrefix = "QRTZ_";
});
store.UseJsonSerializer();
store.UseClustering(c =>
{
c.CheckinMisfireThreshold = TimeSpan.FromSeconds(30);
c.CheckinInterval = TimeSpan.FromSeconds(10);
});
});
// Register jobs and triggers
RegisterJobsAndTriggers(q);
});
// Add Quartz hosted service
builder.Services.AddQuartzHostedService(options =>
{
options.WaitForJobsToComplete = true;
options.StartDelay = TimeSpan.FromSeconds(5);
});
var app = builder.Build();
app.Run();
void RegisterJobsAndTriggers(IServiceCollectionQuartzConfigurator quartz)
{
// Email processing job
var emailJobKey = new JobKey("EmailProcessingJob");
quartz.AddJob<EmailProcessingJob>(opts => opts.WithIdentity(emailJobKey));
quartz.AddTrigger(opts => opts
.ForJob(emailJobKey)
.WithIdentity("EmailProcessingJob-Trigger")
.WithCronSchedule("0 */5 * * * ?")); // Every 5 minutes
// Report generation job
var reportJobKey = new JobKey("ReportGenerationJob");
quartz.AddJob<ReportGenerationJob>(opts => opts.WithIdentity(reportJobKey));
quartz.AddTrigger(opts => opts
.ForJob(reportJobKey)
.WithIdentity("ReportGenerationJob-Trigger")
.WithCronSchedule("0 0 2 * * ?")); // Daily at 2 AM
// Database maintenance job
var maintenanceJobKey = new JobKey("DatabaseMaintenanceJob");
quartz.AddJob<DatabaseMaintenanceJob>(opts => opts.WithIdentity(maintenanceJobKey));
quartz.AddTrigger(opts => opts
.ForJob(maintenanceJobKey)
.WithIdentity("DatabaseMaintenanceJob-Trigger")
.WithCronSchedule("0 0 3 * * ?")); // Daily at 3 AM
// Health check job
var healthCheckJobKey = new JobKey("HealthCheckJob");
quartz.AddJob<HealthCheckJob>(opts => opts.WithIdentity(healthCheckJobKey));
quartz.AddTrigger(opts => opts
.ForJob(healthCheckJobKey)
.WithIdentity("HealthCheckJob-Trigger")
.WithSimpleSchedule(x => x
.WithIntervalInMinutes(10)
.RepeatForever()));
}
5.2 Advanced Quartz.NET Jobs
// Base job with common functionality
public abstract class BaseJob : IJob
{
protected readonly ILogger Logger;
protected readonly IServiceProvider ServiceProvider;
protected BaseJob(ILogger logger, IServiceProvider serviceProvider)
{
Logger = logger;
ServiceProvider = serviceProvider;
}
public async Task Execute(IJobExecutionContext context)
{
var jobName = GetType().Name;
var jobId = context.JobDetail.Key.Name;
var fireTime = context.FireTimeUtc.LocalDateTime;
Logger.LogInformation("Starting job {JobName} ({JobId}) at {FireTime}",
jobName, jobId, fireTime);
try
{
// Create scope for dependency injection
using var scope = ServiceProvider.CreateScope();
// Execute the actual job logic
await ExecuteJob(context, scope.ServiceProvider);
Logger.LogInformation("Completed job {JobName} ({JobId}) successfully", jobName, jobId);
}
catch (Exception ex)
{
Logger.LogError(ex, "Job {JobName} ({JobId}) failed with error: {ErrorMessage}",
jobName, jobId, ex.Message);
// Determine if we should re-throw the exception
if (ShouldRefire(context, ex))
{
throw new JobExecutionException(ex, true);
}
}
}
protected abstract Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider);
protected virtual bool ShouldRefire(IJobExecutionContext context, Exception exception)
{
// Don't refire for certain exception types
if (exception is BusinessRuleException)
return false;
// Refire if we haven't exceeded max attempts
var refireCount = context.RefireCount;
return refireCount < 3;
}
}
// Email processing job
[DisallowConcurrentExecution]
[PersistJobDataAfterExecution]
public class EmailProcessingJob : BaseJob
{
public EmailProcessingJob(ILogger<EmailProcessingJob> logger, IServiceProvider serviceProvider)
: base(logger, serviceProvider)
{
}
protected override async Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider)
{
var emailService = serviceProvider.GetRequiredService<IEmailQueueService>();
var emailSender = serviceProvider.GetRequiredService<IEmailSender>();
// Get job data
var batchSize = context.MergedJobDataMap.GetInt("BatchSize") ?? 50;
var maxRetries = context.MergedJobDataMap.GetInt("MaxRetries") ?? 3;
Logger.LogInformation("Processing email batch with size {BatchSize}", batchSize);
var pendingEmails = await emailService.GetPendingEmailsAsync(batchSize);
var processedCount = 0;
var failedCount = 0;
foreach (var email in pendingEmails)
{
try
{
if (email.RetryCount >= maxRetries)
{
await emailService.MarkAsPermanentlyFailedAsync(email.Id, "Max retries exceeded");
failedCount++;
continue;
}
await emailSender.SendAsync(email.To, email.Subject, email.Body, email.IsHtml);
await emailService.MarkAsCompletedAsync(email.Id);
processedCount++;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Failed to send email {EmailId}", email.Id);
await emailService.MarkAsFailedAsync(email.Id, ex.Message);
failedCount++;
}
}
// Update job data for monitoring
context.JobDetail.JobDataMap["LastProcessedCount"] = processedCount;
context.JobDetail.JobDataMap["LastFailedCount"] = failedCount;
context.JobDetail.JobDataMap["LastRun"] = DateTime.UtcNow;
Logger.LogInformation("Email processing completed: {Processed} processed, {Failed} failed",
processedCount, failedCount);
}
}
// Report generation job
[DisallowConcurrentExecution]
public class ReportGenerationJob : BaseJob
{
public ReportGenerationJob(ILogger<ReportGenerationJob> logger, IServiceProvider serviceProvider)
: base(logger, serviceProvider)
{
}
protected override async Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider)
{
var reportService = serviceProvider.GetRequiredService<IReportService>();
var notificationService = serviceProvider.GetRequiredService<INotificationService>();
var reportDate = DateTime.UtcNow.AddDays(-1); // Yesterday's report
var reportType = context.MergedJobDataMap.GetString("ReportType") ?? "Daily";
Logger.LogInformation("Generating {ReportType} report for {ReportDate}",
reportType, reportDate.ToString("yyyy-MM-dd"));
try
{
var report = await reportService.GenerateReportAsync(reportType, reportDate);
if (report != null)
{
// Store report
await reportService.SaveReportAsync(report);
// Notify administrators
await notificationService.NotifyAdminsAsync(
$"New {reportType} report generated",
$"The {reportType} report for {reportDate:yyyy-MM-dd} has been generated successfully.");
Logger.LogInformation("Successfully generated and saved {ReportType} report", reportType);
}
else
{
Logger.LogWarning("Report generation returned null for {ReportType}", reportType);
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Failed to generate {ReportType} report", reportType);
throw;
}
}
}
// Database maintenance job
public class DatabaseMaintenanceJob : BaseJob
{
public DatabaseMaintenanceJob(ILogger<DatabaseMaintenanceJob> logger, IServiceProvider serviceProvider)
: base(logger, serviceProvider)
{
}
protected override async Task ExecuteJob(IJobExecutionContext context, IServiceProvider serviceProvider)
{
var dbContext = serviceProvider.GetRequiredService<ApplicationDbContext>();
var maintenanceService = serviceProvider.GetRequiredService<IDatabaseMaintenanceService>();
Logger.LogInformation("Starting database maintenance tasks");
var tasks = new[]
{
maintenanceService.CleanOldAuditLogsAsync(TimeSpan.FromDays(90)),
maintenanceService.RebuildIndexesAsync(),
maintenanceService.UpdateStatisticsAsync(),
maintenanceService.CleanTempFilesAsync()
};
await Task.WhenAll(tasks);
Logger.LogInformation("Database maintenance tasks completed successfully");
}
}
5.3 Dynamic Job Scheduling
// Dynamic job scheduling service
public interface IDynamicScheduler
{
Task ScheduleJobAsync<T>(string jobName, string cronExpression, IDictionary<string, object> jobData = null) where T : IJob;
Task ScheduleOneTimeJobAsync<T>(string jobName, DateTimeOffset startTime, IDictionary<string, object> jobData = null) where T : IJob;
Task<bool> DeleteJobAsync(string jobName);
Task<bool> PauseJobAsync(string jobName);
Task<bool> ResumeJobAsync(string jobName);
Task<bool> UpdateJobScheduleAsync(string jobName, string newCronExpression);
Task<List<ScheduledJobInfo>> GetScheduledJobsAsync();
}
public class DynamicScheduler : IDynamicScheduler
{
private readonly ISchedulerFactory _schedulerFactory;
private readonly ILogger<DynamicScheduler> _logger;
public DynamicScheduler(ISchedulerFactory schedulerFactory, ILogger<DynamicScheduler> logger)
{
_schedulerFactory = schedulerFactory;
_logger = logger;
}
public async Task ScheduleJobAsync<T>(string jobName, string cronExpression, IDictionary<string, object> jobData = null) where T : IJob
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobName);
if (await scheduler.CheckExists(jobKey))
{
_logger.LogWarning("Job {JobName} already exists. Updating schedule.", jobName);
await UpdateJobScheduleAsync(jobName, cronExpression);
return;
}
var job = JobBuilder.Create<T>()
.WithIdentity(jobKey)
.UsingJobData(new JobDataMap(jobData ?? new Dictionary<string, object>()))
.Build();
var trigger = TriggerBuilder.Create()
.WithIdentity($"{jobName}-Trigger")
.WithCronSchedule(cronExpression)
.Build();
await scheduler.ScheduleJob(job, trigger);
_logger.LogInformation("Scheduled job {JobName} with cron expression: {CronExpression}", jobName, cronExpression);
}
public async Task ScheduleOneTimeJobAsync<T>(string jobName, DateTimeOffset startTime, IDictionary<string, object> jobData = null) where T : IJob
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobName);
if (await scheduler.CheckExists(jobKey))
{
throw new InvalidOperationException($"Job {jobName} already exists");
}
var job = JobBuilder.Create<T>()
.WithIdentity(jobKey)
.UsingJobData(new JobDataMap(jobData ?? new Dictionary<string, object>()))
.Build();
var trigger = TriggerBuilder.Create()
.WithIdentity($"{jobName}-Trigger")
.StartAt(startTime)
.Build();
await scheduler.ScheduleJob(job, trigger);
_logger.LogInformation("Scheduled one-time job {JobName} to run at {StartTime}", jobName, startTime);
}
public async Task<bool> DeleteJobAsync(string jobName)
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobName);
if (await scheduler.CheckExists(jobKey))
{
var result = await scheduler.DeleteJob(jobKey);
_logger.LogInformation("Deleted job {JobName}: {Result}", jobName, result);
return result;
}
_logger.LogWarning("Job {JobName} not found for deletion", jobName);
return false;
}
public async Task<bool> PauseJobAsync(string jobName)
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobName);
if (await scheduler.CheckExists(jobKey))
{
await scheduler.PauseJob(jobKey);
_logger.LogInformation("Paused job {JobName}", jobName);
return true;
}
_logger.LogWarning("Job {JobName} not found for pausing", jobName);
return false;
}
public async Task<bool> ResumeJobAsync(string jobName)
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobName);
if (await scheduler.CheckExists(jobKey))
{
await scheduler.ResumeJob(jobKey);
_logger.LogInformation("Resumed job {JobName}", jobName);
return true;
}
_logger.LogWarning("Job {JobName} not found for resuming", jobName);
return false;
}
public async Task<bool> UpdateJobScheduleAsync(string jobName, string newCronExpression)
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobKey = new JobKey(jobName);
var triggerKey = new TriggerKey($"{jobName}-Trigger");
if (!await scheduler.CheckExists(jobKey) || !await scheduler.CheckExists(triggerKey))
{
_logger.LogWarning("Job or trigger not found for {JobName}", jobName);
return false;
}
var newTrigger = TriggerBuilder.Create()
.WithIdentity(triggerKey)
.WithCronSchedule(newCronExpression)
.Build();
await scheduler.RescheduleJob(triggerKey, newTrigger);
_logger.LogInformation("Updated schedule for job {JobName} to: {CronExpression}", jobName, newCronExpression);
return true;
}
public async Task<List<ScheduledJobInfo>> GetScheduledJobsAsync()
{
var scheduler = await _schedulerFactory.GetScheduler();
var jobGroups = await scheduler.GetJobGroupNames();
var jobs = new List<ScheduledJobInfo>();
foreach (var group in jobGroups)
{
var groupMatcher = GroupMatcher<JobKey>.GroupContains(group);
var jobKeys = await scheduler.GetJobKeys(groupMatcher);
foreach (var jobKey in jobKeys)
{
var detail = await scheduler.GetJobDetail(jobKey);
var triggers = await scheduler.GetTriggersOfJob(jobKey);
foreach (var trigger in triggers)
{
var state = await scheduler.GetTriggerState(trigger.Key);
var nextFireTime = trigger.GetNextFireTimeUtc();
jobs.Add(new ScheduledJobInfo
{
JobName = jobKey.Name,
Group = jobKey.Group,
Description = detail.Description,
JobType = detail.JobType.FullName,
TriggerState = state.ToString(),
NextFireTime = nextFireTime?.LocalDateTime,
PreviousFireTime = trigger.GetPreviousFireTimeUtc()?.LocalDateTime
});
}
}
}
return jobs;
}
}
public class ScheduledJobInfo
{
public string JobName { get; set; }
public string Group { get; set; }
public string Description { get; set; }
public string JobType { get; set; }
public string TriggerState { get; set; }
public DateTime? NextFireTime { get; set; }
public DateTime? PreviousFireTime { get; set; }
}
6. Coravel Simplified Scheduling
6.1 Coravel Setup and Basic Usage
// Program.cs - Coravel Configuration
using Coravel;
var builder = WebApplication.CreateBuilder(args);
// Add Coravel services
builder.Services.AddScheduler();
builder.Services.AddQueue();
// Register background services
builder.Services.AddTransient<EmailNotificationInvokable>();
builder.Services.AddTransient<ReportGenerationInvokable>();
builder.Services.AddTransient<DataCleanupInvokable>();
// Register caching if needed
builder.Services.AddCache();
var app = builder.Build();
// Configure Coravel scheduling
app.Services.UseScheduler(scheduler =>
{
// Email notifications every 10 minutes
scheduler.Schedule<EmailNotificationInvokable>()
.EveryMinute()
.PreventOverlapping("EmailNotifications")
.Weekday();
// Daily report generation at 2 AM
scheduler.Schedule<ReportGenerationInvokable>()
.DailyAtHour(2)
.Zoned(TimeZoneInfo.Local);
// Data cleanup every Sunday at 3 AM
scheduler.Schedule<DataCleanupInvokable>()
.Cron("0 3 * * 0") // Sunday at 3 AM
.Zoned(TimeZoneInfo.Local);
// Hourly cache priming
scheduler.Schedule<CachePrimingInvokable>()
.Hourly()
.RunOnceAtStart();
}).OnError(ex =>
{
// Global error handling
var logger = app.Services.GetRequiredService<ILogger<Program>>();
logger.LogError(ex, "Scheduler error occurred");
});
app.Run();
6.2 Coravel Invocables and Queuing
// Email notification invocable
public class EmailNotificationInvokable : IInvocable, ICancellableInvocable
{
private readonly IEmailQueueService _emailQueueService;
private readonly IEmailSender _emailSender;
private readonly ILogger<EmailNotificationInvokable> _logger;
public EmailNotificationInvokable(
IEmailQueueService emailQueueService,
IEmailSender emailSender,
ILogger<EmailNotificationInvokable> logger)
{
_emailQueueService = emailQueueService;
_emailSender = emailSender;
_logger = logger;
}
public async Task Invoke()
{
_logger.LogInformation("Starting email notification processing");
var pendingEmails = await _emailQueueService.GetPendingEmailsAsync(100);
var processedCount = 0;
var failedCount = 0;
foreach (var email in pendingEmails)
{
try
{
await _emailQueueService.MarkAsProcessingAsync(email.Id);
var result = await _emailSender.SendAsync(
email.To,
email.Subject,
email.Body,
email.IsHtml);
if (result.Success)
{
await _emailQueueService.MarkAsCompletedAsync(email.Id);
processedCount++;
}
else
{
await _emailQueueService.MarkAsFailedAsync(email.Id, result.Error);
failedCount++;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to process email {EmailId}", email.Id);
await _emailQueueService.MarkAsFailedAsync(email.Id, ex.Message);
failedCount++;
}
}
_logger.LogInformation("Email processing completed: {Processed} processed, {Failed} failed",
processedCount, failedCount);
}
public CancellationToken CancellationToken { get; set; }
}
// Report generation invocable
public class ReportGenerationInvokable : IInvocable
{
private readonly IReportService _reportService;
private readonly INotificationService _notificationService;
private readonly ILogger<ReportGenerationInvokable> _logger;
public ReportGenerationInvokable(
IReportService reportService,
INotificationService notificationService,
ILogger<ReportGenerationInvokable> logger)
{
_reportService = reportService;
_notificationService = notificationService;
_logger = logger;
}
public async Task Invoke()
{
_logger.LogInformation("Starting daily report generation");
var reportDate = DateTime.Today.AddDays(-1); // Yesterday's report
var reportTypes = new[] { "Sales", "Inventory", "UserActivity" };
foreach (var reportType in reportTypes)
{
try
{
_logger.LogInformation("Generating {ReportType} report for {ReportDate}",
reportType, reportDate.ToString("yyyy-MM-dd"));
var report = await _reportService.GenerateReportAsync(reportType, reportDate);
if (report != null)
{
await _reportService.SaveReportAsync(report);
_logger.LogInformation("Successfully generated and saved {ReportType} report", reportType);
}
else
{
_logger.LogWarning("Report generation returned null for {ReportType}", reportType);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to generate {ReportType} report", reportType);
}
}
// Notify administrators
await _notificationService.NotifyAdminsAsync(
"Daily Reports Generated",
$"Daily reports for {reportDate:yyyy-MM-dd} have been generated successfully.");
_logger.LogInformation("Daily report generation completed");
}
}
// Data cleanup invocable
public class DataCleanupInvokable : IInvocable
{
private readonly ApplicationDbContext _dbContext;
private readonly IFileService _fileService;
private readonly ILogger<DataCleanupInvokable> _logger;
public DataCleanupInvokable(
ApplicationDbContext dbContext,
IFileService fileService,
ILogger<DataCleanupInvokable> logger)
{
_dbContext = dbContext;
_fileService = fileService;
_logger = logger;
}
public async Task Invoke()
{
_logger.LogInformation("Starting weekly data cleanup");
var tasks = new[]
{
CleanOldAuditLogsAsync(),
CleanOldNotificationsAsync(),
CleanTemporaryFilesAsync(),
CleanExpiredSessionsAsync()
};
await Task.WhenAll(tasks);
_logger.LogInformation("Weekly data cleanup completed successfully");
}
private async Task CleanOldAuditLogsAsync()
{
var cutoffDate = DateTime.UtcNow.AddDays(-90); // 90 days retention
var oldLogs = await _dbContext.AuditLogs
.Where(log => log.CreatedAt < cutoffDate)
.ToListAsync();
if (oldLogs.Any())
{
_dbContext.AuditLogs.RemoveRange(oldLogs);
await _dbContext.SaveChangesAsync();
_logger.LogInformation("Cleaned {Count} old audit logs", oldLogs.Count);
}
}
private async Task CleanOldNotificationsAsync()
{
var cutoffDate = DateTime.UtcNow.AddDays(-30); // 30 days retention
var oldNotifications = await _dbContext.Notifications
.Where(n => n.CreatedAt < cutoffDate && n.IsRead)
.ToListAsync();
if (oldNotifications.Any())
{
_dbContext.Notifications.RemoveRange(oldNotifications);
await _dbContext.SaveChangesAsync();
_logger.LogInformation("Cleaned {Count} old notifications", oldNotifications.Count);
}
}
private async Task CleanTemporaryFilesAsync()
{
var tempPath = Path.GetTempPath();
var cutoffDate = DateTime.UtcNow.AddDays(-1);
var filesDeleted = await _fileService.CleanOldFilesAsync(tempPath, "*.tmp", cutoffDate);
_logger.LogInformation("Cleaned {Count} temporary files", filesDeleted);
}
private async Task CleanExpiredSessionsAsync()
{
var cutoffDate = DateTime.UtcNow.AddHours(-24); // 24 hours expiry
var expiredSessions = await _dbContext.UserSessions
.Where(s => s.LastActivity < cutoffDate)
.ToListAsync();
if (expiredSessions.Any())
{
_dbContext.UserSessions.RemoveRange(expiredSessions);
await _dbContext.SaveChangesAsync();
_logger.LogInformation("Cleaned {Count} expired sessions", expiredSessions.Count);
}
}
}
6.3 Coravel Queuing for Background Processing
// Queue-based email service
public class QueueEmailService
{
private readonly IQueue _queue;
private readonly ILogger<QueueEmailService> _logger;
public QueueEmailService(IQueue queue, ILogger<QueueEmailService> logger)
{
_queue = queue;
_logger = logger;
}
public async Task QueueWelcomeEmailAsync(string email, string userName)
{
var emailData = new WelcomeEmailData
{
Email = email,
UserName = userName,
SentAt = DateTime.UtcNow
};
await _queue.QueueInvocableAsync<SendWelcomeEmailInvocable, WelcomeEmailData>(emailData);
_logger.LogInformation("Queued welcome email for {Email}", email);
}
public async Task QueueBulkEmailsAsync(List<BulkEmailData> emails)
{
foreach (var email in emails)
{
await _queue.QueueInvocableAsync<SendBulkEmailInvocable, BulkEmailData>(email);
}
_logger.LogInformation("Queued {Count} bulk emails", emails.Count);
}
public async Task QueuePasswordResetEmailAsync(string email, string resetToken)
{
var emailData = new PasswordResetEmailData
{
Email = email,
ResetToken = resetToken,
ExpiresAt = DateTime.UtcNow.AddHours(24)
};
await _queue.QueueInvocableAsync<SendPasswordResetEmailInvocable, PasswordResetEmailData>(emailData);
_logger.LogInformation("Queued password reset email for {Email}", email);
}
}
// Welcome email invocable
public class SendWelcomeEmailInvocable : IInvocable, IInvocableWithPayload<WelcomeEmailData>
{
private readonly IEmailSender _emailSender;
private readonly ILogger<SendWelcomeEmailInvocable> _logger;
public SendWelcomeEmailInvocable(IEmailSender emailSender, ILogger<SendWelcomeEmailInvocable> logger)
{
_emailSender = emailSender;
_logger = logger;
}
public WelcomeEmailData Payload { get; set; }
public async Task Invoke()
{
try
{
var subject = $"Welcome to Our Service, {Payload.UserName}!";
var body = $@"
<h1>Welcome aboard, {Payload.UserName}!</h1>
<p>Thank you for joining our service. We're excited to have you!</p>
<p>If you have any questions, don't hesitate to contact our support team.</p>
<br>
<p>Best regards,<br>The Team</p>";
var result = await _emailSender.SendAsync(Payload.Email, subject, body, true);
if (result.Success)
{
_logger.LogInformation("Successfully sent welcome email to {Email}", Payload.Email);
}
else
{
_logger.LogError("Failed to send welcome email to {Email}: {Error}",
Payload.Email, result.Error);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error sending welcome email to {Email}", Payload.Email);
throw;
}
}
}
// Bulk email invocable
public class SendBulkEmailInvocable : IInvocable, IInvocableWithPayload<BulkEmailData>
{
private readonly IEmailSender _emailSender;
private readonly ILogger<SendBulkEmailInvocable> _logger;
public SendBulkEmailInvocable(IEmailSender emailSender, ILogger<SendBulkEmailInvocable> logger)
{
_emailSender = emailSender;
_logger = logger;
}
public BulkEmailData Payload { get; set; }
public async Task Invoke()
{
try
{
var result = await _emailSender.SendAsync(
Payload.To,
Payload.Subject,
Payload.Body,
Payload.IsHtml);
if (result.Success)
{
_logger.LogInformation("Successfully sent bulk email to {Email}", Payload.To);
}
else
{
_logger.LogError("Failed to send bulk email to {Email}: {Error}",
Payload.To, result.Error);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error sending bulk email to {Email}", Payload.To);
throw;
}
}
}
// Data models for queuing
public class WelcomeEmailData
{
public string Email { get; set; }
public string UserName { get; set; }
public DateTime SentAt { get; set; }
}
public class BulkEmailData
{
public string To { get; set; }
public string Subject { get; set; }
public string Body { get; set; }
public bool IsHtml { get; set; }
}
public class PasswordResetEmailData
{
public string Email { get; set; }
public string ResetToken { get; set; }
public DateTime ExpiresAt { get; set; }
}
7. Real-World E-Commerce Automation
7.1 Complete Order Processing Pipeline
// Order processing service with background jobs
public class OrderProcessingService
{
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly IRecurringJobManager _recurringJobManager;
private readonly ILogger<OrderProcessingService> _logger;
public OrderProcessingService(
IBackgroundJobClient backgroundJobClient,
IRecurringJobManager recurringJobManager,
ILogger<OrderProcessingService> logger)
{
_backgroundJobClient = backgroundJobClient;
_recurringJobManager = recurringJobManager;
_logger = logger;
}
public async Task<string> ProcessOrderAsync(Order order)
{
_logger.LogInformation("Starting order processing for order {OrderId}", order.Id);
try
{
// Step 1: Validate order (immediate)
await ValidateOrderAsync(order);
// Step 2: Process payment (background)
var paymentJobId = _backgroundJobClient.Enqueue<IPaymentService>(
service => service.ProcessPaymentAsync(order.Id, null, false, default));
// Step 3: Update inventory after payment (continuation)
var inventoryJobId = _backgroundJobClient.ContinueJobWith<IInventoryService>(
paymentJobId,
service => service.UpdateInventoryAsync(order.Id, null, false, default));
// Step 4: Send confirmation after inventory update (continuation)
var confirmationJobId = _backgroundJobClient.ContinueJobWith<IEmailService>(
inventoryJobId,
service => service.SendOrderConfirmationAsync(order.Id, null, false, default));
// Step 5: Schedule follow-up email for 7 days later
_backgroundJobClient.Schedule<IEmailService>(
service => service.SendFollowUpEmailAsync(order.Id, null, false, default),
TimeSpan.FromDays(7));
_logger.LogInformation("Order {OrderId} processing pipeline started successfully", order.Id);
return confirmationJobId;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to process order {OrderId}", order.Id);
throw;
}
}
public async Task<string> ProcessBulkOrdersAsync(List<Order> orders)
{
_logger.LogInformation("Processing {OrderCount} orders in bulk", orders.Count);
var batchId = Guid.NewGuid().ToString();
var batchJobIds = new List<string>();
foreach (var order in orders)
{
var jobId = _backgroundJobClient.Enqueue<IOrderService>(
service => service.ProcessSingleOrderAsync(order.Id, batchId, null, false, default));
batchJobIds.Add(jobId);
}
// Monitor batch completion
_backgroundJobClient.Enqueue<IBatchMonitorService>(
service => service.MonitorBatchCompletionAsync(batchId, batchJobIds, null, false, default));
return batchId;
}
public void SetupRecurringOrderJobs()
{
// Daily order summary at 6 AM
_recurringJobManager.AddOrUpdate<IOrderReportService>(
"daily-order-summary",
service => service.GenerateDailyOrderSummaryAsync(null, false, default),
Cron.Daily(6, 0));
// Abandoned cart cleanup every 6 hours
_recurringJobManager.AddOrUpdate<ICartService>(
"abandoned-cart-cleanup",
service => service.CleanAbandonedCartsAsync(null, false, default),
"0 */6 * * *");
// Monthly sales report on 1st of month at 3 AM
_recurringJobManager.AddOrUpdate<ISalesReportService>(
"monthly-sales-report",
service => service.GenerateMonthlySalesReportAsync(null, false, default),
"0 3 1 * *");
}
private async Task ValidateOrderAsync(Order order)
{
// Perform synchronous validation
if (order.TotalAmount <= 0)
throw new InvalidOperationException("Order total must be greater than zero");
if (string.IsNullOrEmpty(order.CustomerEmail))
throw new InvalidOperationException("Customer email is required");
// Check product availability
foreach (var item in order.Items)
{
var available = await CheckProductAvailabilityAsync(item.ProductId, item.Quantity);
if (!available)
throw new InvalidOperationException($"Product {item.ProductId} is not available in requested quantity");
}
}
private async Task<bool> CheckProductAvailabilityAsync(int productId, int quantity)
{
// Implementation depends on your data access
await Task.Delay(10); // Simulate database call
return quantity <= 100; // Simplified check
}
}
// Payment service with retry logic
public class PaymentService : IPaymentService
{
private readonly ILogger<PaymentService> _logger;
private readonly IPaymentGateway _paymentGateway;
public PaymentService(ILogger<PaymentService> logger, IPaymentGateway paymentGateway)
{
_logger = logger;
_paymentGateway = paymentGateway;
}
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
[Queue("payments")]
public async Task ProcessPaymentAsync(int orderId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Processing payment for order {OrderId}", orderId);
try
{
// Retrieve order details
var order = await GetOrderAsync(orderId);
// Process payment through gateway
var paymentResult = await _paymentGateway.ProcessPaymentAsync(
order.TotalAmount,
order.Currency,
order.PaymentMethod);
if (paymentResult.Success)
{
await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Completed, paymentResult.TransactionId);
_logger.LogInformation("Successfully processed payment for order {OrderId}", orderId);
}
else
{
await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Failed, null, paymentResult.Error);
throw new InvalidOperationException($"Payment failed: {paymentResult.Error}");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Payment processing failed for order {OrderId}", orderId);
if (throwOnError)
throw;
throw new JobPerformanceException("Payment processing failed", ex);
}
}
private async Task<Order> GetOrderAsync(int orderId)
{
// Implementation to retrieve order from database
await Task.Delay(10);
return new Order { Id = orderId, TotalAmount = 99.99m, Currency = "USD", PaymentMethod = "CreditCard" };
}
private async Task UpdateOrderPaymentStatusAsync(int orderId, PaymentStatus status, string transactionId = null, string error = null)
{
// Implementation to update order in database
await Task.Delay(10);
_logger.LogInformation("Updated order {OrderId} payment status to {Status}", orderId, status);
}
}
public enum PaymentStatus
{
Pending = 0,
Completed = 1,
Failed = 2,
Refunded = 3
}
7.2 Inventory Management with Background Jobs
// Inventory management service
public class InventoryService : IInventoryService
{
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly ILogger<InventoryService> _logger;
private readonly ApplicationDbContext _dbContext;
public InventoryService(
IBackgroundJobClient backgroundJobClient,
ILogger<InventoryService> logger,
ApplicationDbContext dbContext)
{
_backgroundJobClient = backgroundJobClient;
_logger = logger;
_dbContext = dbContext;
}
[AutomaticRetry(Attempts = 2)]
[Queue("inventory")]
public async Task UpdateInventoryAsync(int orderId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Updating inventory for order {OrderId}", orderId);
try
{
var order = await _dbContext.Orders
.Include(o => o.Items)
.ThenInclude(i => i.Product)
.FirstOrDefaultAsync(o => o.Id == orderId, cancellationToken);
if (order == null)
throw new InvalidOperationException($"Order {orderId} not found");
foreach (var item in order.Items)
{
var product = item.Product;
if (product.StockQuantity < item.Quantity)
{
throw new InvalidOperationException(
$"Insufficient stock for product {product.Name}. Available: {product.StockQuantity}, Requested: {item.Quantity}");
}
// Update stock
product.StockQuantity -= item.Quantity;
product.LastStockUpdate = DateTime.UtcNow;
// Log inventory change
var inventoryLog = new InventoryLog
{
ProductId = product.Id,
OrderId = orderId,
QuantityChange = -item.Quantity,
NewStockLevel = product.StockQuantity,
Reason = "Order fulfillment",
CreatedAt = DateTime.UtcNow
};
_dbContext.InventoryLogs.Add(inventoryLog);
// Check for low stock and trigger reorder if needed
if (product.StockQuantity <= product.ReorderLevel)
{
_backgroundJobClient.Enqueue<IInventoryService>(
service => service.TriggerReorderAsync(product.Id, null, false, default));
}
}
await _dbContext.SaveChangesAsync(cancellationToken);
_logger.LogInformation("Successfully updated inventory for order {OrderId}", orderId);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to update inventory for order {OrderId}", orderId);
if (throwOnError)
throw;
throw new JobPerformanceException("Inventory update failed", ex);
}
}
public async Task TriggerReorderAsync(int productId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Triggering reorder for product {ProductId}", productId);
try
{
var product = await _dbContext.Products.FindAsync(new object[] { productId }, cancellationToken);
if (product == null)
throw new InvalidOperationException($"Product {productId} not found");
// Create purchase order or notify supplier
var reorderRequest = new ReorderRequest
{
ProductId = product.Id,
ProductName = product.Name,
CurrentStock = product.StockQuantity,
ReorderLevel = product.ReorderLevel,
ReorderQuantity = product.ReorderQuantity,
RequestedAt = DateTime.UtcNow
};
// Notify purchasing department
_backgroundJobClient.Enqueue<INotificationService>(
service => service.NotifyPurchasingDepartmentAsync(reorderRequest, null, false, default));
// Update product reorder status
product.LastReorderDate = DateTime.UtcNow;
await _dbContext.SaveChangesAsync(cancellationToken);
_logger.LogInformation("Reorder triggered for product {ProductName}", product.Name);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to trigger reorder for product {ProductId}", productId);
if (throwOnError)
throw;
}
}
public void ScheduleInventoryJobs()
{
// Daily stock level check at 8 AM
_backgroundJobClient.Recurring<IInventoryService>(
"daily-stock-check",
service => service.CheckStockLevelsAsync(null, false, default),
Cron.Daily(8, 0));
// Weekly inventory report on Monday at 6 AM
_backgroundJobClient.Recurring<IInventoryReportService>(
"weekly-inventory-report",
service => service.GenerateWeeklyInventoryReportAsync(null, false, default),
Cron.Weekly(DayOfWeek.Monday, 6, 0));
// Monthly dead stock identification on 1st at 2 AM
_backgroundJobClient.Recurring<IInventoryAnalysisService>(
"dead-stock-analysis",
service => service.IdentifyDeadStockAsync(null, false, default),
"0 2 1 * *");
}
public async Task CheckStockLevelsAsync(string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Starting daily stock level check");
try
{
var lowStockProducts = await _dbContext.Products
.Where(p => p.StockQuantity <= p.ReorderLevel && p.IsActive)
.ToListAsync(cancellationToken);
foreach (var product in lowStockProducts)
{
_backgroundJobClient.Enqueue<IInventoryService>(
service => service.TriggerReorderAsync(product.Id, null, false, default));
}
_logger.LogInformation("Stock level check completed. {Count} products need reordering", lowStockProducts.Count);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to check stock levels");
if (throwOnError)
throw;
}
}
}
public class ReorderRequest
{
public int ProductId { get; set; }
public string ProductName { get; set; }
public int CurrentStock { get; set; }
public int ReorderLevel { get; set; }
public int ReorderQuantity { get; set; }
public DateTime RequestedAt { get; set; }
}
public class InventoryLog
{
public int Id { get; set; }
public int ProductId { get; set; }
public int? OrderId { get; set; }
public int QuantityChange { get; set; }
public int NewStockLevel { get; set; }
public string Reason { get; set; }
public DateTime CreatedAt { get; set; }
}
8. Advanced Patterns & Scenarios
8.1 Distributed Job Processing
// Distributed job coordinator
public interface IDistributedJobCoordinator
{
Task<string> ScheduleDistributedJobAsync<T>(string jobName, object payload, string queue = "default") where T : IBackgroundJob;
Task<bool> CancelDistributedJobAsync(string jobId);
Task<DistributedJobStatus> GetJobStatusAsync(string jobId);
Task<List<DistributedJobInfo>> GetActiveJobsAsync();
}
public class DistributedJobCoordinator : IDistributedJobCoordinator
{
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<DistributedJobCoordinator> _logger;
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly ApplicationDbContext _dbContext;
public DistributedJobCoordinator(
IServiceProvider serviceProvider,
ILogger<DistributedJobCoordinator> logger,
IBackgroundJobClient backgroundJobClient,
ApplicationDbContext dbContext)
{
_serviceProvider = serviceProvider;
_logger = logger;
_backgroundJobClient = backgroundJobClient;
_dbContext = dbContext;
}
public async Task<string> ScheduleDistributedJobAsync<T>(string jobName, object payload, string queue = "default") where T : IBackgroundJob
{
var jobId = Guid.NewGuid().ToString();
var distributedJob = new DistributedJob
{
Id = jobId,
JobName = jobName,
JobType = typeof(T).FullName,
Payload = JsonSerializer.Serialize(payload),
Queue = queue,
Status = DistributedJobStatus.Scheduled,
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow
};
_dbContext.DistributedJobs.Add(distributedJob);
await _dbContext.SaveChangesAsync();
// Schedule the actual background job
var hangfireJobId = _backgroundJobClient.Enqueue<T>(
job => job.ExecuteAsync(jobId, null, false, default));
// Store Hangfire job ID reference
distributedJob.HangfireJobId = hangfireJobId;
await _dbContext.SaveChangesAsync();
_logger.LogInformation("Scheduled distributed job {JobName} with ID {JobId}", jobName, jobId);
return jobId;
}
public async Task<bool> CancelDistributedJobAsync(string jobId)
{
var job = await _dbContext.DistributedJobs
.FirstOrDefaultAsync(j => j.Id == jobId);
if (job == null)
{
_logger.LogWarning("Distributed job {JobId} not found for cancellation", jobId);
return false;
}
if (job.Status == DistributedJobStatus.Running || job.Status == DistributedJobStatus.Scheduled)
{
if (!string.IsNullOrEmpty(job.HangfireJobId))
{
_backgroundJobClient.Delete(job.HangfireJobId);
}
job.Status = DistributedJobStatus.Cancelled;
job.UpdatedAt = DateTime.UtcNow;
await _dbContext.SaveChangesAsync();
_logger.LogInformation("Cancelled distributed job {JobId}", jobId);
return true;
}
_logger.LogWarning("Distributed job {JobId} cannot be cancelled in current status: {Status}", jobId, job.Status);
return false;
}
public async Task<DistributedJobStatus> GetJobStatusAsync(string jobId)
{
var job = await _dbContext.DistributedJobs
.FirstOrDefaultAsync(j => j.Id == jobId);
return job?.Status ?? DistributedJobStatus.NotFound;
}
public async Task<List<DistributedJobInfo>> GetActiveJobsAsync()
{
var activeJobs = await _dbContext.DistributedJobs
.Where(j => j.Status == DistributedJobStatus.Scheduled || j.Status == DistributedJobStatus.Running)
.OrderByDescending(j => j.CreatedAt)
.Take(100)
.Select(j => new DistributedJobInfo
{
Id = j.Id,
JobName = j.JobName,
Status = j.Status,
CreatedAt = j.CreatedAt,
UpdatedAt = j.UpdatedAt,
Queue = j.Queue
})
.ToListAsync();
return activeJobs;
}
}
// Base distributed job interface
public interface IBackgroundJob
{
Task ExecuteAsync(string jobId, string hangfireJobId = null, bool throwOnError = false, CancellationToken cancellationToken = default);
}
// Example distributed job implementation
public class DataExportJob : IBackgroundJob
{
private readonly ILogger<DataExportJob> _logger;
private readonly ApplicationDbContext _dbContext;
private readonly IExportService _exportService;
private readonly IDistributedJobCoordinator _jobCoordinator;
public DataExportJob(
ILogger<DataExportJob> logger,
ApplicationDbContext dbContext,
IExportService exportService,
IDistributedJobCoordinator jobCoordinator)
{
_logger = logger;
_dbContext = dbContext;
_exportService = exportService;
_jobCoordinator = jobCoordinator;
}
public async Task ExecuteAsync(string jobId, string hangfireJobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Starting data export job {JobId}", jobId);
try
{
// Update job status to running
await UpdateJobStatusAsync(jobId, DistributedJobStatus.Running);
// Retrieve job parameters
var job = await _dbContext.DistributedJobs.FindAsync(new object[] { jobId }, cancellationToken);
if (job == null)
throw new InvalidOperationException($"Job {jobId} not found");
var parameters = JsonSerializer.Deserialize<DataExportParameters>(job.Payload);
// Execute export
var exportResult = await _exportService.ExportDataAsync(
parameters.DataType,
parameters.StartDate,
parameters.EndDate,
parameters.Format);
if (exportResult.Success)
{
await UpdateJobStatusAsync(jobId, DistributedJobStatus.Completed, exportResult.FilePath);
_logger.LogInformation("Data export job {JobId} completed successfully", jobId);
}
else
{
await UpdateJobStatusAsync(jobId, DistributedJobStatus.Failed, error: exportResult.Error);
throw new InvalidOperationException($"Export failed: {exportResult.Error}");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Data export job {JobId} failed", jobId);
await UpdateJobStatusAsync(jobId, DistributedJobStatus.Failed, error: ex.Message);
if (throwOnError)
throw;
}
}
private async Task UpdateJobStatusAsync(string jobId, DistributedJobStatus status, string result = null, string error = null)
{
var job = await _dbContext.DistributedJobs.FindAsync(jobId);
if (job != null)
{
job.Status = status;
job.Result = result;
job.Error = error;
job.UpdatedAt = DateTime.UtcNow;
if (status == DistributedJobStatus.Completed)
job.CompletedAt = DateTime.UtcNow;
await _dbContext.SaveChangesAsync();
}
}
}
public class DataExportParameters
{
public string DataType { get; set; } // "Orders", "Products", "Users"
public DateTime StartDate { get; set; }
public DateTime EndDate { get; set; }
public string Format { get; set; } // "CSV", "Excel", "JSON"
}
public class DistributedJob
{
public string Id { get; set; }
public string JobName { get; set; }
public string JobType { get; set; }
public string Payload { get; set; }
public string Queue { get; set; }
public DistributedJobStatus Status { get; set; }
public string HangfireJobId { get; set; }
public string Result { get; set; }
public string Error { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime UpdatedAt { get; set; }
public DateTime? CompletedAt { get; set; }
}
public enum DistributedJobStatus
{
Scheduled = 0,
Running = 1,
Completed = 2,
Failed = 3,
Cancelled = 4,
NotFound = 5
}
public class DistributedJobInfo
{
public string Id { get; set; }
public string JobName { get; set; }
public DistributedJobStatus Status { get; set; }
public string Queue { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime UpdatedAt { get; set; }
}
8.2 Job Chaining and Workflows
// Workflow orchestration service
public interface IWorkflowOrchestrator
{
Task<string> StartOrderFulfillmentWorkflowAsync(int orderId);
Task<string> StartDataProcessingWorkflowAsync(DataProcessingRequest request);
Task<bool> CancelWorkflowAsync(string workflowId);
Task<WorkflowStatus> GetWorkflowStatusAsync(string workflowId);
}
public class WorkflowOrchestrator : IWorkflowOrchestrator
{
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly ILogger<WorkflowOrchestrator> _logger;
private readonly ApplicationDbContext _dbContext;
public WorkflowOrchestrator(
IBackgroundJobClient backgroundJobClient,
ILogger<WorkflowOrchestrator> logger,
ApplicationDbContext dbContext)
{
_backgroundJobClient = backgroundJobClient;
_logger = logger;
_dbContext = dbContext;
}
public async Task<string> StartOrderFulfillmentWorkflowAsync(int orderId)
{
var workflowId = Guid.NewGuid().ToString();
var workflow = new Workflow
{
Id = workflowId,
Type = WorkflowType.OrderFulfillment,
Status = WorkflowStatus.Running,
ReferenceId = orderId.ToString(),
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow
};
_dbContext.Workflows.Add(workflow);
await _dbContext.SaveChangesAsync();
// Start the workflow chain
var paymentJobId = _backgroundJobClient.Enqueue<IPaymentService>(
service => service.ProcessPaymentAsync(orderId, workflowId, null, false, default));
_logger.LogInformation("Started order fulfillment workflow {WorkflowId} for order {OrderId}", workflowId, orderId);
return workflowId;
}
public async Task<string> StartDataProcessingWorkflowAsync(DataProcessingRequest request)
{
var workflowId = Guid.NewGuid().ToString();
var workflow = new Workflow
{
Id = workflowId,
Type = WorkflowType.DataProcessing,
Status = WorkflowStatus.Running,
ReferenceId = request.Source,
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow
};
_dbContext.Workflows.Add(workflow);
await _dbContext.SaveChangesAsync();
// Chain data processing steps
var extractionJobId = _backgroundJobClient.Enqueue<IDataExtractionService>(
service => service.ExtractDataAsync(request.Source, workflowId, null, false, default));
var transformationJobId = _backgroundJobClient.ContinueJobWith<IDataTransformationService>(
extractionJobId,
service => service.TransformDataAsync(workflowId, null, false, default));
var loadingJobId = _backgroundJobClient.ContinueJobWith<IDataLoadingService>(
transformationJobId,
service => service.LoadDataAsync(request.Destination, workflowId, null, false, default));
var notificationJobId = _backgroundJobClient.ContinueJobWith<INotificationService>(
loadingJobId,
service => service.NotifyDataProcessingCompleteAsync(workflowId, null, false, default));
_logger.LogInformation("Started data processing workflow {WorkflowId}", workflowId);
return workflowId;
}
public async Task<bool> CancelWorkflowAsync(string workflowId)
{
var workflow = await _dbContext.Workflows
.FirstOrDefaultAsync(w => w.Id == workflowId);
if (workflow == null)
{
_logger.LogWarning("Workflow {WorkflowId} not found for cancellation", workflowId);
return false;
}
if (workflow.Status == WorkflowStatus.Running)
{
// Cancel all associated jobs
var jobs = await _dbContext.WorkflowJobs
.Where(j => j.WorkflowId == workflowId && j.Status == WorkflowJobStatus.Running)
.ToListAsync();
foreach (var job in jobs)
{
if (!string.IsNullOrEmpty(job.HangfireJobId))
{
_backgroundJobClient.Delete(job.HangfireJobId);
}
job.Status = WorkflowJobStatus.Cancelled;
job.UpdatedAt = DateTime.UtcNow;
}
workflow.Status = WorkflowStatus.Cancelled;
workflow.UpdatedAt = DateTime.UtcNow;
await _dbContext.SaveChangesAsync();
_logger.LogInformation("Cancelled workflow {WorkflowId}", workflowId);
return true;
}
_logger.LogWarning("Workflow {WorkflowId} cannot be cancelled in current status: {Status}", workflowId, workflow.Status);
return false;
}
public async Task<WorkflowStatus> GetWorkflowStatusAsync(string workflowId)
{
var workflow = await _dbContext.Workflows
.FirstOrDefaultAsync(w => w.Id == workflowId);
return workflow?.Status ?? WorkflowStatus.NotFound;
}
}
// Workflow-aware job base class
public abstract class WorkflowJobBase
{
protected readonly ILogger Logger;
protected readonly ApplicationDbContext DbContext;
protected WorkflowJobBase(ILogger logger, ApplicationDbContext dbContext)
{
Logger = logger;
DbContext = dbContext;
}
protected async Task UpdateWorkflowJobStatusAsync(string workflowId, string jobName, WorkflowJobStatus status, string result = null, string error = null)
{
var workflowJob = await DbContext.WorkflowJobs
.FirstOrDefaultAsync(j => j.WorkflowId == workflowId && j.JobName == jobName);
if (workflowJob == null)
{
workflowJob = new WorkflowJob
{
WorkflowId = workflowId,
JobName = jobName,
Status = status,
Result = result,
Error = error,
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow
};
DbContext.WorkflowJobs.Add(workflowJob);
}
else
{
workflowJob.Status = status;
workflowJob.Result = result;
workflowJob.Error = error;
workflowJob.UpdatedAt = DateTime.UtcNow;
}
await DbContext.SaveChangesAsync();
}
protected async Task UpdateWorkflowStatusAsync(string workflowId, WorkflowStatus status)
{
var workflow = await DbContext.Workflows.FindAsync(workflowId);
if (workflow != null)
{
workflow.Status = status;
workflow.UpdatedAt = DateTime.UtcNow;
await DbContext.SaveChangesAsync();
}
}
}
// Example workflow job implementation
public class PaymentServiceWithWorkflow : WorkflowJobBase, IPaymentService
{
private readonly IPaymentGateway _paymentGateway;
public PaymentServiceWithWorkflow(
ILogger<PaymentServiceWithWorkflow> logger,
ApplicationDbContext dbContext,
IPaymentGateway paymentGateway)
: base(logger, dbContext)
{
_paymentGateway = paymentGateway;
}
public async Task ProcessPaymentAsync(int orderId, string workflowId, string jobId = null, bool throwOnError = false, CancellationToken cancellationToken = default)
{
const string jobName = "ProcessPayment";
Logger.LogInformation("Processing payment for order {OrderId} in workflow {WorkflowId}", orderId, workflowId);
try
{
await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Running);
// Retrieve order and process payment
var order = await GetOrderAsync(orderId);
var paymentResult = await _paymentGateway.ProcessPaymentAsync(
order.TotalAmount,
order.Currency,
order.PaymentMethod);
if (paymentResult.Success)
{
await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Completed, paymentResult.TransactionId);
await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Completed, paymentResult.TransactionId);
Logger.LogInformation("Payment processed successfully for order {OrderId}", orderId);
}
else
{
await UpdateOrderPaymentStatusAsync(orderId, PaymentStatus.Failed, null, paymentResult.Error);
await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Failed, error: paymentResult.Error);
await UpdateWorkflowStatusAsync(workflowId, WorkflowStatus.Failed);
throw new InvalidOperationException($"Payment failed: {paymentResult.Error}");
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Payment processing failed for order {OrderId}", orderId);
await UpdateWorkflowJobStatusAsync(workflowId, jobName, WorkflowJobStatus.Failed, error: ex.Message);
await UpdateWorkflowStatusAsync(workflowId, WorkflowStatus.Failed);
if (throwOnError)
throw;
}
}
private async Task<Order> GetOrderAsync(int orderId)
{
// Implementation to retrieve order
await Task.Delay(10);
return new Order { Id = orderId, TotalAmount = 99.99m, Currency = "USD", PaymentMethod = "CreditCard" };
}
private async Task UpdateOrderPaymentStatusAsync(int orderId, PaymentStatus status, string transactionId = null, string error = null)
{
// Implementation to update order
await Task.Delay(10);
}
}
public class Workflow
{
public string Id { get; set; }
public WorkflowType Type { get; set; }
public WorkflowStatus Status { get; set; }
public string ReferenceId { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime UpdatedAt { get; set; }
public DateTime? CompletedAt { get; set; }
public ICollection<WorkflowJob> Jobs { get; set; } = new List<WorkflowJob>();
}
public class WorkflowJob
{
public int Id { get; set; }
public string WorkflowId { get; set; }
public string JobName { get; set; }
public string HangfireJobId { get; set; }
public WorkflowJobStatus Status { get; set; }
public string Result { get; set; }
public string Error { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime UpdatedAt { get; set; }
public Workflow Workflow { get; set; }
}
public enum WorkflowType
{
OrderFulfillment = 0,
DataProcessing = 1,
ReportGeneration = 2,
DataMigration = 3
}
public enum WorkflowStatus
{
Running = 0,
Completed = 1,
Failed = 2,
Cancelled = 3,
NotFound = 4
}
public enum WorkflowJobStatus
{
Pending = 0,
Running = 1,
Completed = 2,
Failed = 3,
Cancelled = 4
}
public class DataProcessingRequest
{
public string Source { get; set; }
public string Destination { get; set; }
public string Format { get; set; }
}
9. Monitoring & Diagnostics
9.1 Comprehensive Job Monitoring
// Job monitoring and alerting service
public interface IJobMonitor
{
Task<JobHealthReport> GetHealthReportAsync();
Task<List<JobExecutionRecord>> GetRecentExecutionsAsync(int count = 50);
Task<JobStatistics> GetJobStatisticsAsync(DateTime fromDate, DateTime toDate);
Task<bool> CheckJobHealthAsync(string jobName);
Task SendAlertAsync(string jobName, string message, AlertLevel level);
}
public class JobMonitor : IJobMonitor
{
private readonly IMonitoringApi _monitoringApi;
private readonly ILogger<JobMonitor> _logger;
private readonly ApplicationDbContext _dbContext;
private readonly INotificationService _notificationService;
public JobMonitor(
IMonitoringApi monitoringApi,
ILogger<JobMonitor> logger,
ApplicationDbContext dbContext,
INotificationService notificationService)
{
_monitoringApi = monitoringApi;
_logger = logger;
_dbContext = dbContext;
_notificationService = notificationService;
}
public async Task<JobHealthReport> GetHealthReportAsync()
{
var report = new JobHealthReport
{
GeneratedAt = DateTime.UtcNow,
ServerCount = (await _monitoringApi.Servers()).Length,
QueueStats = await GetQueueStatisticsAsync(),
FailedJobs = await GetRecentFailedJobsAsync(20),
LongRunningJobs = await GetLongRunningJobsAsync(),
SystemMetrics = await GetSystemMetricsAsync()
};
report.OverallHealth = CalculateOverallHealth(report);
return report;
}
public async Task<List<JobExecutionRecord>> GetRecentExecutionsAsync(int count = 50)
{
var executions = new List<JobExecutionRecord>();
// Get succeeded jobs
var succeededJobs = _monitoringApi.SucceededJobs(0, count / 2);
executions.AddRange(succeededJobs.Select(job => new JobExecutionRecord
{
JobId = job.Key,
JobName = job.Value.Job.ToString(),
Status = JobExecutionStatus.Succeeded,
StartedAt = job.Value.SucceededAt?.Add(-job.Value.TotalDuration) ?? DateTime.MinValue,
CompletedAt = job.Value.SucceededAt,
Duration = job.Value.TotalDuration
}));
// Get failed jobs
var failedJobs = _monitoringApi.FailedJobs(0, count / 2);
executions.AddRange(failedJobs.Select(job => new JobExecutionRecord
{
JobId = job.Key,
JobName = job.Value.Job.ToString(),
Status = JobExecutionStatus.Failed,
StartedAt = job.Value.FailedAt?.Add(-TimeSpan.FromSeconds(30)) ?? DateTime.MinValue, // Estimate
CompletedAt = job.Value.FailedAt,
Duration = TimeSpan.FromSeconds(30), // Estimate
Error = job.Value.ExceptionMessage
}));
return executions.OrderByDescending(e => e.CompletedAt).Take(count).ToList();
}
public async Task<JobStatistics> GetJobStatisticsAsync(DateTime fromDate, DateTime toDate)
{
var stats = new JobStatistics
{
Period = new DateRange(fromDate, toDate),
TotalExecutions = await GetTotalExecutionsAsync(fromDate, toDate),
SuccessfulExecutions = await GetSuccessfulExecutionsAsync(fromDate, toDate),
FailedExecutions = await GetFailedExecutionsAsync(fromDate, toDate),
AverageDuration = await GetAverageDurationAsync(fromDate, toDate),
MaxDuration = await GetMaxDurationAsync(fromDate, toDate)
};
stats.SuccessRate = stats.TotalExecutions > 0
? (double)stats.SuccessfulExecutions / stats.TotalExecutions * 100
: 0;
return stats;
}
public async Task<bool> CheckJobHealthAsync(string jobName)
{
try
{
var recentExecutions = await GetRecentExecutionsAsync(10);
var jobExecutions = recentExecutions
.Where(e => e.JobName.Contains(jobName))
.ToList();
if (!jobExecutions.Any())
return true; // No recent executions, considered healthy
var failureRate = (double)jobExecutions.Count(e => e.Status == JobExecutionStatus.Failed) / jobExecutions.Count;
return failureRate < 0.2; // Consider unhealthy if more than 20% failures
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking health for job {JobName}", jobName);
return false;
}
}
public async Task SendAlertAsync(string jobName, string message, AlertLevel level)
{
var alert = new JobAlert
{
JobName = jobName,
Message = message,
Level = level,
CreatedAt = DateTime.UtcNow,
IsAcknowledged = false
};
_dbContext.JobAlerts.Add(alert);
await _dbContext.SaveChangesAsync();
// Send immediate notification for critical alerts
if (level == AlertLevel.Critical)
{
await _notificationService.NotifyAdminsAsync(
$"CRITICAL: Job {jobName} Alert",
message);
}
_logger.LogWarning("Job alert created: {JobName} - {Message} (Level: {Level})", jobName, message, level);
}
private async Task<List<QueueStatistics>> GetQueueStatisticsAsync()
{
var queues = new[] { "default", "emails", "reports", "critical", "payments", "inventory" };
var stats = new List<QueueStatistics>();
foreach (var queue in queues)
{
stats.Add(new QueueStatistics
{
QueueName = queue,
EnqueuedCount = await _monitoringApi.EnqueuedCount(queue),
FailedCount = await _monitoringApi.FailedCount(queue),
ScheduledCount = await _monitoringApi.ScheduledCount(queue)
});
}
return stats;
}
private async Task<List<FailedJobInfo>> GetRecentFailedJobsAsync(int count)
{
var failedJobs = _monitoringApi.FailedJobs(0, count);
return failedJobs.Select(job => new FailedJobInfo
{
JobId = job.Key,
JobName = job.Value.Job.ToString(),
FailedAt = job.Value.FailedAt?.LocalDateTime ?? DateTime.MinValue,
ExceptionMessage = job.Value.ExceptionMessage,
ExceptionDetails = job.Value.ExceptionDetails,
InFailedState = job.Value.InFailedState
}).ToList();
}
private async Task<List<LongRunningJob>> GetLongRunningJobsAsync()
{
// Implementation to identify jobs running longer than expected
// This would require custom tracking
return new List<LongRunningJob>();
}
private async Task<SystemMetrics> GetSystemMetricsAsync()
{
// Implementation to get system-level metrics
return new SystemMetrics
{
MemoryUsage = GC.GetTotalMemory(false),
ActiveThreads = System.Diagnostics.Process.GetCurrentProcess().Threads.Count,
CpuUsage = 0 // Would require more sophisticated monitoring
};
}
private JobHealth CalculateOverallHealth(JobHealthReport report)
{
if (report.FailedJobs.Count > 10)
return JobHealth.Critical;
if (report.FailedJobs.Count > 5)
return JobHealth.Warning;
return JobHealth.Healthy;
}
// Additional helper methods for statistics...
private async Task<int> GetTotalExecutionsAsync(DateTime fromDate, DateTime toDate)
{
// Implementation to get total job executions from storage
return await Task.FromResult(1000);
}
private async Task<int> GetSuccessfulExecutionsAsync(DateTime fromDate, DateTime toDate)
{
// Implementation to get successful job executions from storage
return await Task.FromResult(950);
}
private async Task<int> GetFailedExecutionsAsync(DateTime fromDate, DateTime toDate)
{
// Implementation to get failed job executions from storage
return await Task.FromResult(50);
}
private async Task<TimeSpan> GetAverageDurationAsync(DateTime fromDate, DateTime toDate)
{
// Implementation to calculate average job duration
return await Task.FromResult(TimeSpan.FromSeconds(30));
}
private async Task<TimeSpan> GetMaxDurationAsync(DateTime fromDate, DateTime toDate)
{
// Implementation to get maximum job duration
return await Task.FromResult(TimeSpan.FromMinutes(5));
}
}
public class JobHealthReport
{
public DateTime GeneratedAt { get; set; }
public JobHealth OverallHealth { get; set; }
public int ServerCount { get; set; }
public List<QueueStatistics> QueueStats { get; set; } = new();
public List<FailedJobInfo> FailedJobs { get; set; } = new();
public List<LongRunningJob> LongRunningJobs { get; set; } = new();
public SystemMetrics SystemMetrics { get; set; }
}
public class JobExecutionRecord
{
public string JobId { get; set; }
public string JobName { get; set; }
public JobExecutionStatus Status { get; set; }
public DateTime StartedAt { get; set; }
public DateTime? CompletedAt { get; set; }
public TimeSpan? Duration { get; set; }
public string Error { get; set; }
}
public class JobStatistics
{
public DateRange Period { get; set; }
public int TotalExecutions { get; set; }
public int SuccessfulExecutions { get; set; }
public int FailedExecutions { get; set; }
public double SuccessRate { get; set; }
public TimeSpan AverageDuration { get; set; }
public TimeSpan MaxDuration { get; set; }
}
public class QueueStatistics
{
public string QueueName { get; set; }
public long EnqueuedCount { get; set; }
public long FailedCount { get; set; }
public long ScheduledCount { get; set; }
}
public class SystemMetrics
{
public long MemoryUsage { get; set; }
public int ActiveThreads { get; set; }
public double CpuUsage { get; set; }
}
public class LongRunningJob
{
public string JobId { get; set; }
public string JobName { get; set; }
public DateTime StartedAt { get; set; }
public TimeSpan Duration { get; set; }
public TimeSpan ExpectedDuration { get; set; }
}
public class JobAlert
{
public int Id { get; set; }
public string JobName { get; set; }
public string Message { get; set; }
public AlertLevel Level { get; set; }
public DateTime CreatedAt { get; set; }
public bool IsAcknowledged { get; set; }
}
public enum JobHealth
{
Healthy = 0,
Warning = 1,
Critical = 2
}
public enum JobExecutionStatus
{
Succeeded = 0,
Failed = 1,
Running = 2
}
public enum AlertLevel
{
Info = 0,
Warning = 1,
Critical = 2
}
public record DateRange(DateTime Start, DateTime End);
9.2 Performance Monitoring and Metrics
csharp
// Performance monitoring service
public interface IJobPerformanceMonitor
{
Task RecordJobExecutionAsync(string jobName, TimeSpan duration, bool success, string error = null);
Task<JobPerformanceSummary> GetPerformanceSummaryAsync(string jobName, TimeSpan period);
Task<List<PerformanceIssue>> DetectPerformanceIssuesAsync();
Task CleanOldMetricsAsync(DateTime cutoffDate);
}
public class JobPerformanceMonitor : IJobPerformanceMonitor
{
private readonly ApplicationDbContext _dbContext;
private readonly ILogger<JobPerformanceMonitor> _logger;
public JobPerformanceMonitor(ApplicationDbContext dbContext, ILogger<JobPerformanceMonitor> logger)
{
_dbContext = dbContext;
_logger = logger;
}
public async Task RecordJobExecutionAsync(string jobName, TimeSpan duration, bool success, string error = null)
{
var metric = new JobPerformanceMetric
{
JobName = jobName,
DurationMs = (long)duration.TotalMilliseconds,
Success = success,
Error = error,
Timestamp = DateTime.UtcNow
};
_dbContext.JobPerformanceMetrics.Add(metric);
await _dbContext.SaveChangesAsync();
_logger.LogDebug("Recorded performance metric for {JobName}: {Duration}ms, Success: {Success}",
jobName, duration.TotalMilliseconds, success);
}
public async Task<JobPerformanceSummary> GetPerformanceSummaryAsync(string jobName, TimeSpan period)
{
var cutoffDate = DateTime.UtcNow - period;
var metrics = await _dbContext.JobPerformanceMetrics
.Where(m => m.JobName == jobName && m.Timestamp >= cutoffDate)
.ToListAsync();
if (!metrics.Any())
return new JobPerformanceSummary { JobName = jobName, Period = period };
var successfulMetrics = metrics.Where(m => m.Success).ToList();
return new JobPerformanceSummary
{
JobName = jobName,
Period = period,
TotalExecutions = metrics.Count,
SuccessfulExecutions = successfulMetrics.Count,
FailedExecutions = metrics.Count - successfulMetrics.Count,
SuccessRate = (double)successfulMetrics.Count / metrics.Count * 100,
AverageDuration = TimeSpan.FromMilliseconds(successfulMetrics.Average(m => m.DurationMs)),
MinDuration = TimeSpan.FromMilliseconds(successfulMetrics.Min(m => m.DurationMs)),
MaxDuration = TimeSpan.FromMilliseconds(successfulMetrics.Max(m => m.DurationMs)),
Percentile95 = TimeSpan.FromMilliseconds(CalculatePercentile(successfulMetrics.Select(m => m.DurationMs), 0.95)),
RecentErrors = metrics
.Where(m => !m.Success)
.OrderByDescending(m => m.Timestamp)
.Take(10)
.Select(m => new JobError { Timestamp = m.Timestamp, Error = m.Error })
.ToList()
};
}
public async Task<List<PerformanceIssue>> DetectPerformanceIssuesAsync()
{
var issues = new List<PerformanceIssue>();
var period = TimeSpan.FromHours(24);
var cutoffDate = DateTime.UtcNow - period;
// Get unique job names
var jobNames = await _dbContext.JobPerformanceMetrics
.Where(m => m.Timestamp >= cutoffDate)
.Select(m => m.JobName)
.Distinct()
.ToListAsync();
foreach (var jobName in jobNames)
{
var summary = await GetPerformanceSummaryAsync(jobName, period);
// Check for high failure rate
if (summary.SuccessRate < 90) // Less than 90% success rate
{
issues.Add(new PerformanceIssue
{
JobName = jobName,
IssueType = PerformanceIssueType.HighFailureRate,
Severity = summary.SuccessRate < 70 ? IssueSeverity.Critical : IssueSeverity.Warning,
Metric = $"Success Rate: {summary.SuccessRate:F1}%",
Recommendation = "Investigate recent errors and implement retry logic"
});
}
// Check for performance degradation
var historicalSummary = await GetPerformanceSummaryAsync(jobName, TimeSpan.FromDays(7));
if (summary.AverageDuration > historicalSummary.AverageDuration * 1.5) // 50% slower than historical
{
issues.Add(new PerformanceIssue
{
JobName = jobName,
IssueType = PerformanceIssueType.PerformanceDegradation,
Severity = IssueSeverity.Warning,
Metric = $"Current: {summary.AverageDuration.TotalSeconds:F1}s, Historical: {historicalSummary.AverageDuration.TotalSeconds:F1}s",
Recommendation = "Optimize database queries or check for resource constraints"
});
}
// Check for long-running jobs
if (summary.MaxDuration > TimeSpan.FromMinutes(10)) // Jobs taking more than 10 minutes
{
issues.Add(new PerformanceIssue
{
JobName = jobName,
IssueType = PerformanceIssueType.LongRunning,
Severity = summary.MaxDuration > TimeSpan.FromMinutes(30) ? IssueSeverity.Critical : IssueSeverity.Warning,
Metric = $"Max Duration: {summary.MaxDuration.TotalMinutes:F1} minutes",
Recommendation = "Consider breaking job into smaller chunks or optimizing performance"
});
}
}
return issues.OrderByDescending(i => i.Severity).ToList();
}
public async Task CleanOldMetricsAsync(DateTime cutoffDate)
{
var oldMetrics = _dbContext.JobPerformanceMetrics
.Where(m => m.Timestamp < cutoffDate);
_dbContext.JobPerformanceMetrics.RemoveRange(oldMetrics);
var deletedCount = await _dbContext.SaveChangesAsync();
_logger.LogInformation("Cleaned {Count} old performance metrics older than {CutoffDate}",
deletedCount, cutoffDate);
}
private double CalculatePercentile(IEnumerable<long> values, double percentile)
{
var sortedValues = values.OrderBy(v => v).ToArray();
double index = percentile * (sortedValues.Length - 1);
int lowerIndex = (int)Math.Floor(index);
int upperIndex = (int)Math.Ceiling(index);
if (lowerIndex == upperIndex)
return sortedValues[lowerIndex];
double weight = index - lowerIndex;
return sortedValues[lowerIndex] * (1 - weight) + sortedValues[upperIndex] * weight;
}
}
public class JobPerformanceMetric
{
public int Id { get; set; }
public string JobName { get; set; }
public long DurationMs { get; set; }
public bool Success { get; set; }
public string Error { get; set; }
public DateTime Timestamp { get; set; }
}
public class JobPerformanceSummary
{
public string JobName { get; set; }
public TimeSpan Period { get; set; }
public int TotalExecutions { get; set; }
public int SuccessfulExecutions { get; set; }
public int FailedExecutions { get; set; }
public double SuccessRate { get; set; }
public TimeSpan AverageDuration { get; set; }
public TimeSpan MinDuration { get; set; }
public TimeSpan MaxDuration { get; set; }
public TimeSpan Percentile95 { get; set; }
public List<JobError> RecentErrors { get; set; } = new();
}
public class PerformanceIssue
{
public string JobName { get; set; }
public PerformanceIssueType IssueType { get; set; }
public IssueSeverity Severity { get; set; }
public string Metric { get; set; }
public string Recommendation { get; set; }
public DateTime DetectedAt { get; set; } = DateTime.UtcNow;
}
public class JobError
{
public DateTime Timestamp { get; set; }
public string Error { get; set; }
}
public enum PerformanceIssueType
{
HighFailureRate = 0,
PerformanceDegradation = 1,
LongRunning = 2,
ResourceExhaustion = 3
}
public enum IssueSeverity
{
Info = 0,
Warning = 1,
Critical = 2
}
10. Production Ready Deployment
10.1 Docker Configuration for Background Jobs
# Dockerfile for ASP.NET Core with Background Jobs
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
WORKDIR /app
EXPOSE 80
EXPOSE 443
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
WORKDIR /src
COPY ["ECommerce.API/ECommerce.API.csproj", "ECommerce.API/"]
COPY ["ECommerce.Application/ECommerce.Application.csproj", "ECommerce.Application/"]
COPY ["ECommerce.Domain/ECommerce.Domain.csproj", "ECommerce.Domain/"]
COPY ["ECommerce.Infrastructure/ECommerce.Infrastructure.csproj", "ECommerce.Infrastructure/"]
COPY ["ECommerce.Shared/ECommerce.Shared.csproj", "ECommerce.Shared/"]
RUN dotnet restore "ECommerce.API/ECommerce.API.csproj"
COPY . .
WORKDIR "/src/ECommerce.API"
RUN dotnet build "ECommerce.API.csproj" -c Release -o /app/build
FROM build AS publish
RUN dotnet publish "ECommerce.API.csproj" -c Release -o /app/publish /p:UseAppHost=false
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
# Install monitoring tools
RUN apt-get update && apt-get install -y curl
# Health check endpoint
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD curl -f http://localhost/health || exit 1
ENTRYPOINT ["dotnet", "ECommerce.API.dll"]
10.2 Docker Compose for Production
# docker-compose.production.yml
version: '3.8'
services:
# Database
postgres:
image: postgres:15
environment:
POSTGRES_DB: ecommerce
POSTGRES_USER: admin
POSTGRES_PASSWORD: ${DB_PASSWORD}
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
networks:
- ecommerce-network
restart: unless-stopped
# Redis for caching and Hangfire
redis:
image: redis:7-alpine
ports:
- "6379:6379"
volumes:
- redis_data:/data
networks:
- ecommerce-network
restart: unless-stopped
command: redis-server --appendonly yes
# Main API application
api:
build:
context: .
dockerfile: Dockerfile
environment:
- ConnectionStrings__DefaultConnection=Host=postgres;Database=ecommerce;Username=admin;Password=${DB_PASSWORD}
- ConnectionStrings__Redis=redis:6379
- ConnectionStrings__HangfireConnection=Host=postgres;Database=ecommerce;Username=admin;Password=${DB_PASSWORD}
- ASPNETCORE_ENVIRONMENT=Production
- Hangfire__WorkerCount=4
ports:
- "5000:80"
depends_on:
- postgres
- redis
networks:
- ecommerce-network
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost/health"]
interval: 30s
timeout: 10s
retries: 3
# Hangfire background worker (separate instance)
hangfire-worker:
build:
context: .
dockerfile: Dockerfile
environment:
- ConnectionStrings__DefaultConnection=Host=postgres;Database=ecommerce;Username=admin;Password=${DB_PASSWORD}
- ConnectionStrings__Redis=redis:6379
- ConnectionStrings__HangfireConnection=Host=postgres;Database=ecommerce;Username=admin;Password=${DB_PASSWORD}
- ASPNETCORE_ENVIRONMENT=Production
- Hangfire__WorkerCount=8
- RunBackgroundWorker=true
depends_on:
- postgres
- redis
networks:
- ecommerce-network
restart: unless-stopped
command: ["dotnet", "ECommerce.API.dll", "--urls", "http://*:80"]
# Nginx reverse proxy
nginx:
image: nginx:alpine
ports:
- "80:80"
- "443:443"
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf
- ./ssl:/etc/nginx/ssl
depends_on:
- api
networks:
- ecommerce-network
restart: unless-stopped
volumes:
postgres_data:
redis_data:
networks:
ecommerce-network:
driver: bridge
10.3 Kubernetes Deployment
# kubernetes/deployment.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: ecommerce-api
labels:
app: ecommerce-api
spec:
replicas: 3
selector:
matchLabels:
app: ecommerce-api
template:
metadata:
labels:
app: ecommerce-api
spec:
containers:
- name: ecommerce-api
image: your-registry/ecommerce-api:latest
ports:
- containerPort: 80
env:
- name: ConnectionStrings__DefaultConnection
valueFrom:
secretKeyRef:
name: db-secret
key: connection-string
- name: ConnectionStrings__Redis
value: "redis-service:6379"
- name: ASPNETCORE_ENVIRONMENT
value: "Production"
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
livenessProbe:
httpGet:
path: /health
port: 80
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health/ready
port: 80
initialDelaySeconds: 5
periodSeconds: 5
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hangfire-worker
labels:
app: hangfire-worker
spec:
replicas: 2
selector:
matchLabels:
app: hangfire-worker
template:
metadata:
labels:
app: hangfire-worker
spec:
containers:
- name: hangfire-worker
image: your-registry/ecommerce-api:latest
env:
- name: ConnectionStrings__DefaultConnection
valueFrom:
secretKeyRef:
name: db-secret
key: connection-string
- name: ConnectionStrings__Redis
value: "redis-service:6379"
- name: ASPNETCORE_ENVIRONMENT
value: "Production"
- name: RunBackgroundWorker
value: "true"
resources:
requests:
memory: "512Mi"
cpu: "500m"
limits:
memory: "1Gi"
cpu: "1000m"
---
apiVersion: v1
kind: Service
metadata:
name: ecommerce-service
spec:
selector:
app: ecommerce-api
ports:
- port: 80
targetPort: 80
type: LoadBalancer
10.4 Production Configuration and Best Practices
// Production configuration service
public class ProductionConfiguration
{
public static void ConfigureProductionServices(IServiceCollection services, IConfiguration configuration)
{
// Database configuration
services.AddDbContext<ApplicationDbContext>(options =>
{
options.UseSqlServer(
configuration.GetConnectionString("DefaultConnection"),
sqlOptions =>
{
sqlOptions.EnableRetryOnFailure(
maxRetryCount: 5,
maxRetryDelay: TimeSpan.FromSeconds(30),
errorNumbersToAdd: null);
sqlOptions.CommandTimeout(60);
});
});
// Hangfire production configuration
services.AddHangfire(config =>
{
config.UseSqlServerStorage(
configuration.GetConnectionString("HangfireConnection"),
new SqlServerStorageOptions
{
CommandBatchMaxTimeout = TimeSpan.FromMinutes(5),
SlidingInvisibilityTimeout = TimeSpan.FromMinutes(5),
QueuePollInterval = TimeSpan.FromSeconds(15),
UseRecommendedIsolationLevel = true,
UsePageLocksOnDequeue = true,
DisableGlobalLocks = true,
SchemaName = "Hangfire"
});
});
services.AddHangfireServer(options =>
{
options.WorkerCount = configuration.GetValue<int>("Hangfire:WorkerCount", Environment.ProcessorCount * 2);
options.Queues = new[] { "critical", "default", "emails", "reports", "low" };
options.ServerName = $"{Environment.MachineName}_{Guid.NewGuid():N}";
options.SchedulePollingInterval = TimeSpan.FromSeconds(15);
});
// Health checks
services.AddHealthChecks()
.AddSqlServer(configuration.GetConnectionString("DefaultConnection"), name: "database")
.AddRedis(configuration.GetConnectionString("Redis"), name: "redis")
.AddHangfire(options =>
{
options.MinimumAvailableServers = 1;
options.MaximumJobsFailed = 10;
}, name: "hangfire");
// Caching
services.AddStackExchangeRedisCache(options =>
{
options.Configuration = configuration.GetConnectionString("Redis");
options.InstanceName = "ECommerce_";
});
// HTTP client with retry policies
services.AddHttpClient("ExternalAPI")
.AddPolicyHandler(GetRetryPolicy())
.AddPolicyHandler(GetCircuitBreakerPolicy());
// Application insights for monitoring
services.AddApplicationInsightsTelemetry();
// Background services
services.AddHostedService<DatabaseMaintenanceService>();
services.AddHostedService<HealthCheckService>();
}
private static IAsyncPolicy<HttpResponseMessage> GetRetryPolicy()
{
return HttpPolicyExtensions
.HandleTransientHttpError()
.OrResult(msg => !msg.IsSuccessStatusCode)
.WaitAndRetryAsync(
retryCount: 3,
sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
onRetry: (outcome, timespan, retryCount, context) =>
{
// Log retry attempts
});
}
private static IAsyncPolicy<HttpResponseMessage> GetCircuitBreakerPolicy()
{
return HttpPolicyExtensions
.HandleTransientHttpError()
.CircuitBreakerAsync(
handledEventsAllowedBeforeBreaking: 3,
durationOfBreak: TimeSpan.FromSeconds(30));
}
public static void ConfigureProductionApp(WebApplication app)
{
// Exception handling
app.UseExceptionHandler("/error");
app.UseStatusCodePagesWithReExecute("/error/{0}");
// Security headers
app.UseHsts();
app.UseHttpsRedirection();
// Health checks
app.MapHealthChecks("/health", new HealthCheckOptions
{
ResponseWriter = async (context, report) =>
{
var result = JsonSerializer.Serialize(new
{
status = report.Status.ToString(),
checks = report.Entries.Select(e => new
{
name = e.Key,
status = e.Value.Status.ToString(),
duration = e.Value.Duration.TotalMilliseconds,
exception = e.Value.Exception?.Message
})
});
context.Response.ContentType = "application/json";
await context.Response.WriteAsync(result);
}
});
app.MapHealthChecks("/health/ready", new HealthCheckOptions
{
Predicate = check => check.Tags.Contains("ready")
});
app.MapHealthChecks("/health/live", new HealthCheckOptions
{
Predicate = _ => false
});
// Hangfire dashboard with authorization
app.UseHangfireDashboard("/jobs", new DashboardOptions
{
Authorization = new[] { new HangfireDashboardAuthorizationFilter() },
DashboardTitle = "ECommerce Production Jobs",
StatsPollingInterval = 10000,
AppPath = "/"
});
// Configure recurring jobs for production
if (app.Configuration.GetValue<bool>("EnableRecurringJobs"))
{
ConfigureProductionRecurringJobs();
}
}
private static void ConfigureProductionRecurringJobs()
{
// Production-specific recurring jobs
RecurringJob.AddOrUpdate<DatabaseMaintenanceService>(
"production-database-maintenance",
service => service.RunMaintenanceAsync(),
Cron.Daily(3, 0), // 3 AM daily
TimeZoneInfo.Local);
RecurringJob.AddOrUpdate<PerformanceMonitorService>(
"performance-metrics-cleanup",
service => service.CleanOldMetricsAsync(),
Cron.Weekly(DayOfWeek.Sunday, 2, 0), // Sunday 2 AM
TimeZoneInfo.Local);
RecurringJob.AddOrUpdate<SystemHealthService>(
"system-health-check",
service => service.CheckSystemHealthAsync(),
"*/5 * * * *", // Every 5 minutes
TimeZoneInfo.Local);
}
}
// Production authorization filter for Hangfire
public class HangfireDashboardAuthorizationFilter : IDashboardAuthorizationFilter
{
public bool Authorize(DashboardContext context)
{
var httpContext = context.GetHttpContext();
// Only allow authenticated users with Admin role
return httpContext.User.Identity.IsAuthenticated &&
httpContext.User.IsInRole("Administrator");
}
}
// Production health check service
public class SystemHealthService
{
private readonly IJobMonitor _jobMonitor;
private readonly ILogger<SystemHealthService> _logger;
private readonly INotificationService _notificationService;
public SystemHealthService(
IJobMonitor jobMonitor,
ILogger<SystemHealthService> logger,
INotificationService notificationService)
{
_jobMonitor = jobMonitor;
_logger = logger;
_notificationService = notificationService;
}
public async Task CheckSystemHealthAsync()
{
_logger.LogInformation("Running system health check");
try
{
var healthReport = await _jobMonitor.GetHealthReportAsync();
if (healthReport.OverallHealth == JobHealth.Critical)
{
await _notificationService.NotifyAdminsAsync(
"CRITICAL: System Health Alert",
$"System health is critical. Failed jobs: {healthReport.FailedJobs.Count}");
}
else if (healthReport.OverallHealth == JobHealth.Warning)
{
await _notificationService.NotifyAdminsAsync(
"WARNING: System Health Alert",
$"System health requires attention. Failed jobs: {healthReport.FailedJobs.Count}");
}
// Log health metrics
_logger.LogInformation(
"System health: {HealthStatus}, Servers: {ServerCount}, Failed jobs: {FailedJobCount}",
healthReport.OverallHealth,
healthReport.ServerCount,
healthReport.FailedJobs.Count);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to complete system health check");
await _notificationService.NotifyAdminsAsync(
"ERROR: Health Check Failure",
$"System health check failed: {ex.Message}");
}
}
}
This comprehensive guide covers the complete spectrum of background job processing in ASP.NET Core, from basic hosted services to enterprise-grade distributed job processing. The examples provided are production-ready and can be adapted to various real-world scenarios, ensuring your applications can handle any background processing requirements efficiently and reliably.