41. Testing Background Jobs

Testing Hangfire background jobs: unit testing job logic, mocking context, and verifying job execution

41.1 Why Test Background Jobs?

Background jobs (Chapter 29-31) are code too. They need tests:

Job logic is complex: Often involves retries, rollback, state management
Jobs fail silently: If untested, failures go unnoticed until production
Hard to debug: Background jobs run outside normal request flow
Dependencies matter: Jobs use services, databases, external APIs
Retries can cause issues: Testing retry logic prevents double-processing

What to test:

✓ Job executes successfully with valid input
✓ Job handles errors gracefully
✓ Job retries on failure (Hangfire retry logic)
✓ Job doesn’t process twice (idempotency)
✓ Job updates state correctly
✓ Job calls dependencies correctly

41.2 Setting Up Job Tests

Install test dependencies:

dotnet add package Hangfire.Core
dotnet add package Moq

Job to test:

// Jobs/SendEmailJob.cs
using Hangfire;
using CommandAPI.Models;
using CommandAPI.Services;
using Microsoft.Extensions.Logging;

public class SendEmailJob
{
    private readonly IEmailService _emailService;
    private readonly ILogger<SendEmailJob> _logger;

    public SendEmailJob(IEmailService emailService, ILogger<SendEmailJob> logger)
    {
        _emailService = emailService;
        _logger = logger;
    }

    [Queue("default")]
    [AutomaticRetry(Attempts = 3)]
    public async Task SendCommandNotificationAsync(int commandId, string recipientEmail)
    {
        _logger.LogInformation("Sending notification for command {CommandId} to {Email}", commandId, recipientEmail);

        try
        {
            await _emailService.SendEmailAsync(
                recipientEmail,
                $"Command {commandId} created",
                $"Your new command with ID {commandId} has been created.");

            _logger.LogInformation("Notification sent successfully for command {CommandId}", commandId);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to send notification for command {CommandId}", commandId);
            throw;  // Hangfire will retry
        }
    }
}

// Services/IEmailService.cs
public interface IEmailService
{
    Task SendEmailAsync(string to, string subject, string body);
}

// Services/EmailService.cs
public class EmailService : IEmailService
{
    private readonly ILogger<EmailService> _logger;

    public EmailService(ILogger<EmailService> logger)
    {
        _logger = logger;
    }

    public async Task SendEmailAsync(string to, string subject, string body)
    {
        // In production, send real email via SMTP
        // For testing, we'll mock this
        _logger.LogInformation("Sending email to {To}: {Subject}", to, subject);
        await Task.Delay(100);  // Simulate network delay
    }
}

41.3 Unit Testing Job Logic

Basic job test:

// Tests/JobsTests/SendEmailJobTests.cs
using Xunit;
using Moq;
using Microsoft.Extensions.Logging;
using CommandAPI.Jobs;
using CommandAPI.Services;

public class SendEmailJobTests
{
    private readonly Mock<IEmailService> _mockEmailService;
    private readonly Mock<ILogger<SendEmailJob>> _mockLogger;
    private readonly SendEmailJob _job;

    public SendEmailJobTests()
    {
        _mockEmailService = new Mock<IEmailService>();
        _mockLogger = new Mock<ILogger<SendEmailJob>>();

        _job = new SendEmailJob(_mockEmailService.Object, _mockLogger.Object);
    }

    [Fact]
    public async Task SendCommandNotificationAsync_WithValidInput_SendsEmail()
    {
        // Arrange
        var commandId = 1;
        var recipientEmail = "user@example.com";

        _mockEmailService.Setup(x => x.SendEmailAsync(
            It.IsAny<string>(),
            It.IsAny<string>(),
            It.IsAny<string>()))
            .Returns(Task.CompletedTask);

        // Act
        await _job.SendCommandNotificationAsync(commandId, recipientEmail);

        // Assert
        _mockEmailService.Verify(
            x => x.SendEmailAsync(
                recipientEmail,
                It.Is<string>(s => s.Contains(commandId.ToString())),
                It.IsAny<string>()),
            Times.Once);
    }

    [Fact]
    public async Task SendCommandNotificationAsync_WhenEmailServiceThrows_ThrowsException()
    {
        // Arrange
        var commandId = 1;
        var recipientEmail = "user@example.com";

        _mockEmailService.Setup(x => x.SendEmailAsync(
            It.IsAny<string>(),
            It.IsAny<string>(),
            It.IsAny<string>()))
            .ThrowsAsync(new Exception("SMTP connection failed"));

        // Act & Assert
        await Assert.ThrowsAsync<Exception>(
            () => _job.SendCommandNotificationAsync(commandId, recipientEmail));

        // Verify email service was attempted
        _mockEmailService.Verify(
            x => x.SendEmailAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()),
            Times.Once);
    }

    [Fact]
    public async Task SendCommandNotificationAsync_Logs_BeforeAndAfter()
    {
        // Arrange
        var commandId = 1;
        var recipientEmail = "user@example.com";

        _mockEmailService.Setup(x => x.SendEmailAsync(
            It.IsAny<string>(),
            It.IsAny<string>(),
            It.IsAny<string>()))
            .Returns(Task.CompletedTask);

        // Act
        await _job.SendCommandNotificationAsync(commandId, recipientEmail);

        // Assert - Verify logging occurred
        _mockLogger.Verify(
            x => x.Log(
                LogLevel.Information,
                It.IsAny<EventId>(),
                It.Is<It.IsAnyType>((state, type) => state.ToString().Contains("Sending notification")),
                It.IsAny<Exception>(),
                It.IsAny<Func<It.IsAnyType, Exception, string>>()),
            Times.Once);

        _mockLogger.Verify(
            x => x.Log(
                LogLevel.Information,
                It.IsAny<EventId>(),
                It.Is<It.IsAnyType>((state, type) => state.ToString().Contains("successfully")),
                It.IsAny<Exception>(),
                It.IsAny<Func<It.IsAnyType, Exception, string>>()),
            Times.Once);
    }
}

41.4 Testing Job with Dependencies

Job that uses repository:

// Jobs/GenerateCommandReportJob.cs
using Hangfire;
using CommandAPI.Data;
using CommandAPI.Services;
using Microsoft.Extensions.Logging;

public class GenerateCommandReportJob
{
    private readonly ICommandRepository _repository;
    private readonly IReportService _reportService;
    private readonly ILogger<GenerateCommandReportJob> _logger;

    public GenerateCommandReportJob(
        ICommandRepository repository,
        IReportService reportService,
        ILogger<GenerateCommandReportJob> logger)
    {
        _repository = repository;
        _reportService = reportService;
        _logger = logger;
    }

    [Queue("reports")]
    [AutomaticRetry(Attempts = 2)]
    public async Task GenerateReportAsync(int platformId)
    {
        _logger.LogInformation("Generating report for platform {PlatformId}", platformId);

        try
        {
            var commands = await _repository.GetCommandsByPlatformAsync(platformId);
            
            if (!commands.Any())
            {
                _logger.LogWarning("No commands found for platform {PlatformId}", platformId);
                return;
            }

            var reportData = new ReportData
            {
                PlatformId = platformId,
                CommandCount = commands.Count,
                GeneratedAt = DateTime.UtcNow
            };

            await _reportService.GenerateAndStoreAsync(reportData);
            
            _logger.LogInformation("Report generated successfully for platform {PlatformId}", platformId);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to generate report for platform {PlatformId}", platformId);
            throw;  // Trigger retry
        }
    }
}

// Services/IReportService.cs
public interface IReportService
{
    Task GenerateAndStoreAsync(ReportData data);
}

public class ReportData
{
    public int PlatformId { get; set; }
    public int CommandCount { get; set; }
    public DateTime GeneratedAt { get; set; }
}

// Test
public class GenerateCommandReportJobTests
{
    private readonly Mock<ICommandRepository> _mockRepository;
    private readonly Mock<IReportService> _mockReportService;
    private readonly Mock<ILogger<GenerateCommandReportJob>> _mockLogger;
    private readonly GenerateCommandReportJob _job;

    public GenerateCommandReportJobTests()
    {
        _mockRepository = new Mock<ICommandRepository>();
        _mockReportService = new Mock<IReportService>();
        _mockLogger = new Mock<ILogger<GenerateCommandReportJob>>();

        _job = new GenerateCommandReportJob(
            _mockRepository.Object,
            _mockReportService.Object,
            _mockLogger.Object);
    }

    [Fact]
    public async Task GenerateReportAsync_WithValidPlatform_GeneratesReport()
    {
        // Arrange
        var platformId = 1;
        var commands = new List<Command>
        {
            new Command { Id = 1, HowTo = "Test 1", CommandLine = "cmd1", PlatformId = platformId },
            new Command { Id = 2, HowTo = "Test 2", CommandLine = "cmd2", PlatformId = platformId }
        };

        _mockRepository.Setup(x => x.GetCommandsByPlatformAsync(platformId))
            .ReturnsAsync(commands);
        _mockReportService.Setup(x => x.GenerateAndStoreAsync(It.IsAny<ReportData>()))
            .Returns(Task.CompletedTask);

        // Act
        await _job.GenerateReportAsync(platformId);

        // Assert
        _mockRepository.Verify(x => x.GetCommandsByPlatformAsync(platformId), Times.Once);
        _mockReportService.Verify(
            x => x.GenerateAndStoreAsync(It.Is<ReportData>(r => 
                r.PlatformId == platformId && r.CommandCount == 2)),
            Times.Once);
    }

    [Fact]
    public async Task GenerateReportAsync_WithNoPlatformCommands_DoesNotGenerateReport()
    {
        // Arrange
        var platformId = 999;

        _mockRepository.Setup(x => x.GetCommandsByPlatformAsync(platformId))
            .ReturnsAsync(new List<Command>());

        // Act
        await _job.GenerateReportAsync(platformId);

        // Assert
        _mockRepository.Verify(x => x.GetCommandsByPlatformAsync(platformId), Times.Once);
        _mockReportService.Verify(
            x => x.GenerateAndStoreAsync(It.IsAny<ReportData>()),
            Times.Never);  // Report not generated
    }

    [Fact]
    public async Task GenerateReportAsync_WhenRepositoryThrows_RethrowsForRetry()
    {
        // Arrange
        var platformId = 1;

        _mockRepository.Setup(x => x.GetCommandsByPlatformAsync(platformId))
            .ThrowsAsync(new Exception("Database connection failed"));

        // Act & Assert
        await Assert.ThrowsAsync<Exception>(
            () => _job.GenerateReportAsync(platformId));

        // Hangfire will catch this and retry
    }

    [Fact]
    public async Task GenerateReportAsync_WhenReportServiceThrows_RethrowsForRetry()
    {
        // Arrange
        var platformId = 1;
        var commands = new List<Command>
        {
            new Command { Id = 1, HowTo = "Test", CommandLine = "cmd", PlatformId = platformId }
        };

        _mockRepository.Setup(x => x.GetCommandsByPlatformAsync(platformId))
            .ReturnsAsync(commands);
        _mockReportService.Setup(x => x.GenerateAndStoreAsync(It.IsAny<ReportData>()))
            .ThrowsAsync(new Exception("Failed to store report"));

        // Act & Assert
        await Assert.ThrowsAsync<Exception>(
            () => _job.GenerateReportAsync(platformId));
    }
}

41.5 Testing Idempotency

Jobs must be idempotent—running them multiple times produces the same result:

// Jobs/UpdateCommandStatusJob.cs
using Hangfire;

public class UpdateCommandStatusJob
{
    private readonly ICommandRepository _repository;
    private readonly ILogger<UpdateCommandStatusJob> _logger;

    public UpdateCommandStatusJob(ICommandRepository repository, ILogger<UpdateCommandStatusJob> logger)
    {
        _repository = repository;
        _logger = logger;
    }

    [AutomaticRetry(Attempts = 3)]
    public async Task MarkCommandAsVerifiedAsync(int commandId)
    {
        _logger.LogInformation("Marking command {CommandId} as verified", commandId);

        var command = await _repository.GetCommandByIdAsync(commandId);
        
        if (command == null)
        {
            _logger.LogWarning("Command {CommandId} not found", commandId);
            return;  // Safe - job can be retried
        }

        if (command.Status == CommandStatus.Verified)
        {
            _logger.LogInformation("Command {CommandId} already verified, skipping", commandId);
            return;  // Idempotent - safe to run multiple times
        }

        command.Status = CommandStatus.Verified;
        command.VerifiedAt = DateTime.UtcNow;

        _repository.UpdateCommand(command);
        await _repository.SaveChangesAsync();

        _logger.LogInformation("Command {CommandId} marked as verified", commandId);
    }
}

// Test idempotency
public class UpdateCommandStatusJobTests
{
    [Fact]
    public async Task MarkCommandAsVerifiedAsync_RunTwice_OnlyVerifiesOnce()
    {
        // Arrange
        var commandId = 1;
        var mockRepository = new Mock<ICommandRepository>();
        var mockLogger = new Mock<ILogger<UpdateCommandStatusJob>>();

        var command = new Command
        {
            Id = commandId,
            HowTo = "Test",
            CommandLine = "test",
            PlatformId = 1,
            Status = CommandStatus.Pending
        };

        mockRepository.Setup(x => x.GetCommandByIdAsync(commandId))
            .ReturnsAsync(command);
        mockRepository.Setup(x => x.SaveChangesAsync())
            .Returns(Task.CompletedTask);

        var job = new UpdateCommandStatusJob(mockRepository.Object, mockLogger.Object);

        // Act - Run twice
        await job.MarkCommandAsVerifiedAsync(commandId);
        
        // Command is now verified
        command.Status = CommandStatus.Verified;

        mockRepository.Setup(x => x.GetCommandByIdAsync(commandId))
            .ReturnsAsync(command);

        await job.MarkCommandAsVerifiedAsync(commandId);

        // Assert - SaveChanges called only once
        mockRepository.Verify(x => x.SaveChangesAsync(), Times.Once);
    }
}

public enum CommandStatus
{
    Pending,
    Verified
}

41.6 Testing Retry Behavior

While Hangfire handles retry logic, you can test your retry strategy:

// Jobs/ResilienceAwareJob.cs
using Hangfire;
using Polly;

public class ResilienceAwareJob
{
    private readonly IExternalApiService _apiService;
    private readonly ILogger<ResilienceAwareJob> _logger;

    public ResilienceAwareJob(IExternalApiService apiService, ILogger<ResilienceAwareJob> logger)
    {
        _apiService = apiService;
        _logger = logger;
    }

    [AutomaticRetry(Attempts = 5, DelaysInSeconds = new[] { 10, 60, 300, 900, 3600 })]
    public async Task FetchAndStoreDataAsync()
    {
        _logger.LogInformation("Starting data fetch job");

        // Use Polly retry policy within the job
        var retryPolicy = Policy
            .Handle<HttpRequestException>()
            .WaitAndRetryAsync(
                retryCount: 3,
                sleepDurationProvider: attempt => 
                    TimeSpan.FromSeconds(Math.Pow(2, attempt)),  // Exponential backoff
                onRetry: (exception, duration) =>
                {
                    _logger.LogWarning(exception, "Retrying after {Duration}", duration);
                });

        try
        {
            await retryPolicy.ExecuteAsync(async () =>
            {
                var data = await _apiService.GetDataAsync();
                _logger.LogInformation("Data fetched successfully: {DataCount} items", data.Count);
            });
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Job failed after all retries");
            throw;  // Hangfire will retry with its own delays
        }
    }
}

// Service interface
public interface IExternalApiService
{
    Task<List<DataItem>> GetDataAsync();
}

public class DataItem { }

// Test
public class ResilienceAwareJobTests
{
    [Fact]
    public async Task FetchAndStoreDataAsync_WithTemporaryFailure_RetriesAndSucceeds()
    {
        // Arrange
        var mockApiService = new Mock<IExternalApiService>();
        var mockLogger = new Mock<ILogger<ResilienceAwareJob>>();

        var callCount = 0;

        // First call fails, second succeeds (simulating temporary failure)
        mockApiService.Setup(x => x.GetDataAsync())
            .Returns(() =>
            {
                callCount++;
                if (callCount == 1)
                {
                    return Task.FromException<List<DataItem>>(
                        new HttpRequestException("Service unavailable"));
                }
                return Task.FromResult(new List<DataItem> { new DataItem() });
            });

        var job = new ResilienceAwareJob(mockApiService.Object, mockLogger.Object);

        // Act
        await job.FetchAndStoreDataAsync();

        // Assert
        mockApiService.Verify(x => x.GetDataAsync(), Times.Exactly(2));
    }

    [Fact]
    public async Task FetchAndStoreDataAsync_WithPersistentFailure_EventuallyThrows()
    {
        // Arrange
        var mockApiService = new Mock<IExternalApiService>();
        var mockLogger = new Mock<ILogger<ResilienceAwareJob>>();

        mockApiService.Setup(x => x.GetDataAsync())
            .ThrowsAsync(new HttpRequestException("Service permanently down"));

        var job = new ResilienceAwareJob(mockApiService.Object, mockLogger.Object);

        // Act & Assert
        await Assert.ThrowsAsync<HttpRequestException>(
            () => job.FetchAndStoreDataAsync());
    }
}

41.7 Testing Long-Running Jobs

Jobs that take a long time:

// Jobs/ProcessLargeDatasetJob.cs
using Hangfire;

public class ProcessLargeDatasetJob
{
    private readonly IDataProcessingService _processingService;
    private readonly ILogger<ProcessLargeDatasetJob> _logger;

    public ProcessLargeDatasetJob(
        IDataProcessingService processingService,
        ILogger<ProcessLargeDatasetJob> logger)
    {
        _processingService = processingService;
        _logger = logger;
    }

    [Queue("longrunning")]
    [JobDisplayName("Process {0} for {1}")]
    [AutomaticRetry(Attempts = 1)]  // Don't retry long-running jobs
    public async Task ProcessDatasetAsync(string datasetId, IJobCancellationToken cancellationToken)
    {
        _logger.LogInformation("Processing dataset {DatasetId}", datasetId);

        try
        {
            // Check cancellation periodically
            if (cancellationToken.ShutdownToken.IsCancellationRequested)
            {
                _logger.LogInformation("Job cancelled for dataset {DatasetId}", datasetId);
                return;
            }

            var result = await _processingService.ProcessAsync(datasetId, cancellationToken.ShutdownToken);
            
            _logger.LogInformation("Dataset {DatasetId} processed successfully: {RecordCount} records", 
                datasetId, result.ProcessedCount);
        }
        catch (OperationCanceledException)
        {
            _logger.LogWarning("Job cancelled for dataset {DatasetId}", datasetId);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to process dataset {DatasetId}", datasetId);
            throw;
        }
    }
}

public interface IDataProcessingService
{
    Task<ProcessResult> ProcessAsync(string datasetId, CancellationToken cancellationToken);
}

public class ProcessResult
{
    public int ProcessedCount { get; set; }
}

// Test
public class ProcessLargeDatasetJobTests
{
    [Fact]
    public async Task ProcessDatasetAsync_WithValidDataset_ProcessesSuccessfully()
    {
        // Arrange
        var datasetId = "dataset-123";
        var mockService = new Mock<IDataProcessingService>();
        var mockLogger = new Mock<ILogger<ProcessLargeDatasetJob>>();

        mockService.Setup(x => x.ProcessAsync(datasetId, It.IsAny<CancellationToken>()))
            .ReturnsAsync(new ProcessResult { ProcessedCount = 1000 });

        var job = new ProcessLargeDatasetJob(mockService.Object, mockLogger.Object);
        var cancellationToken = new CancellationToken();

        // Act
        await job.ProcessDatasetAsync(
            datasetId,
            new JobCancellationToken(CancellationToken.None));

        // Assert
        mockService.Verify(
            x => x.ProcessAsync(datasetId, It.IsAny<CancellationToken>()),
            Times.Once);
    }

    [Fact]
    public async Task ProcessDatasetAsync_WhenCancelled_StopsProcessing()
    {
        // Arrange
        var datasetId = "dataset-123";
        var mockService = new Mock<IDataProcessingService>();
        var mockLogger = new Mock<ILogger<ProcessLargeDatasetJob>>();

        var cts = new CancellationTokenSource();

        mockService.Setup(x => x.ProcessAsync(datasetId, It.IsAny<CancellationToken>()))
            .Callback(() => cts.Cancel())
            .ThrowsAsync(new OperationCanceledException());

        var job = new ProcessLargeDatasetJob(mockService.Object, mockLogger.Object);

        // Act & Assert - Should not throw, just log cancellation
        await job.ProcessDatasetAsync(
            datasetId,
            new JobCancellationToken(cts.Token));
    }
}

public class JobCancellationToken : IJobCancellationToken
{
    public CancellationToken ShutdownToken { get; }

    public JobCancellationToken(CancellationToken shutdownToken)
    {
        ShutdownToken = shutdownToken;
    }
}

public interface IJobCancellationToken
{
    CancellationToken ShutdownToken { get; }
}

41.8 Testing with Hangfire Test Server

For more realistic testing, use Hangfire’s test server:

using Hangfire;
using Hangfire.InMemory;

public class HangfireIntegrationTests
{
    [Fact]
    public async Task Job_EnqueuedAndExecuted_Completes()
    {
        // Arrange
        var storage = new InMemoryStorage();
        var jobClient = new BackgroundJobClient(storage);
        var server = new BackgroundJobServer(storage);

        var mockEmailService = new Mock<IEmailService>();
        mockEmailService.Setup(x => x.SendEmailAsync(
            It.IsAny<string>(),
            It.IsAny<string>(),
            It.IsAny<string>()))
            .Returns(Task.CompletedTask);

        // Act - Enqueue job
        var jobId = jobClient.Enqueue(() => Console.WriteLine("Test job"));

        // Assert
        Assert.NotNull(jobId);
    }
}

41.9 Best Practices for Job Testing

1. Always test success and failure paths:

[Fact]
public async Task Job_WithValidInput_Succeeds() { }

[Fact]
public async Task Job_WhenDependencyThrows_Throws() { }

[Fact]
public async Task Job_WhenDependencyThrows_LogsError() { }

2. Verify logging for debugging:

_mockLogger.Verify(
    x => x.Log(
        LogLevel.Information,
        It.IsAny<EventId>(),
        It.Is<It.IsAnyType>((state, type) => state.ToString().Contains("expected message")),
        It.IsAny<Exception>(),
        It.IsAny<Func<It.IsAnyType, Exception, string>>()),
    Times.Once);

3. Test idempotency to prevent double-processing:

// Run job twice, verify idempotent behavior
await job.DoWorkAsync();
await job.DoWorkAsync();
// Verify only one change was made

4. Use meaningful test names:

[Fact]
public async Task SendEmailJob_WithValidEmail_SendsSuccessfully() { }

[Fact]
public async Task SendEmailJob_WhenSmtpThrows_RetriesAndThrows() { }

41.10 Running Job Tests

# Run all job tests
dotnet test --filter "JobTests"

# Run specific job test
dotnet test --filter "SendEmailJobTests"

# Run with verbose output
dotnet test -v d --filter "GenerateCommandReportJobTests"

# Watch mode
dotnet watch test

41.11 Integration Testing Jobs with WebApplicationFactory

Combine job testing with integration testing:

public class JobIntegrationTests : IClassFixture<CommandsApiFactory>
{
    private readonly HttpClient _client;

    public JobIntegrationTests(CommandsApiFactory factory)
    {
        _client = factory.CreateClient();
    }

    [Fact]
    public async Task CreateCommand_EnqueuesNotificationJob()
    {
        // Arrange
        var createDto = new CommandMutateDto
        {
            HowTo = "Test",
            CommandLine = "test",
            PlatformId = 1
        };

        // Act
        var response = await _client.PostAsJsonAsync("/api/commands", createDto);

        // Assert
        Assert.Equal(HttpStatusCode.Created, response.StatusCode);
        // Verify job was enqueued (check Hangfire storage or mock job client)
    }
}

41.12 What’s Next

You now have:

✓ Understanding why background jobs need testing
✓ Unit testing job logic with mocked dependencies
✓ Testing jobs with repositories
✓ Testing idempotency (running jobs multiple times)
✓ Testing retry behavior with Polly
✓ Testing long-running jobs with cancellation
✓ Testing with Hangfire in-memory storage
✓ Best practices for job testing
✓ Integration testing with WebApplicationFactory

Next: Deployment & DevOps—Docker, containerization, and CI/CD pipelines.