diff --git a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs new file mode 100644 index 00000000000..6b10a0e8d9d --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs @@ -0,0 +1,43 @@ +namespace Polly.CircuitBreaker; + +#pragma warning disable CA1815 // Override equals and operator equals on value types + +/// +/// Represents arguments used to generate a dynamic break duration for a circuit breaker. +/// +public readonly struct BreakDurationGeneratorArguments +{ + /// + /// Initializes a new instance of the struct. + /// + /// The failure rate at which the circuit breaker should trip. + /// It represents the ratio of failed actions to the total executed actions. + /// The number of failures that have occurred. + /// This count is used to determine if the failure threshold has been reached. + /// The resilience context providing additional information + /// about the execution state and failures. + public BreakDurationGeneratorArguments( + double failureRate, + int failureCount, + ResilienceContext context) + { + FailureRate = failureRate; + FailureCount = failureCount; + Context = context; + } + + /// + /// Gets the failure rate that represents the ratio of failures to total actions. + /// + public double FailureRate { get; } + + /// + /// Gets the count of failures that have occurred. + /// + public int FailureCount { get; } + + /// + /// Gets the context that provides additional information about the resilience operation. + /// + public ResilienceContext Context { get; } +} diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index ade8eba13a3..8d1bdbdeec1 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -1,3 +1,4 @@ +using System; using System.ComponentModel.DataAnnotations; using System.Diagnostics.CodeAnalysis; @@ -68,6 +69,14 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions [UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Addressed with DynamicDependency on ValidationHelper.Validate method")] public TimeSpan BreakDuration { get; set; } = CircuitBreakerConstants.DefaultBreakDuration; + /// + /// Gets or sets an optional delegate to use to dynamically generate the break duration. + /// + /// + /// The default value is . + /// + public Func>? BreakDurationGenerator { get; set; } + /// /// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker. /// diff --git a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs index 8cfc1cb472d..3b931715771 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs @@ -44,5 +44,7 @@ public override void OnActionFailure(CircuitState currentState, out bool shouldB } public override void OnCircuitClosed() => _metrics.Reset(); + public override int FailureCount => _metrics.GetHealthInfo().FailureCount; + public override double FailureRate => _metrics.GetHealthInfo().FailureRate; } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs index d1e1f2ee710..a585756f73a 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs @@ -10,4 +10,6 @@ internal abstract class CircuitBehavior public abstract void OnActionFailure(CircuitState currentState, out bool shouldBreak); public abstract void OnCircuitClosed(); + public abstract int FailureCount { get; } + public abstract double FailureRate { get; } } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 37c0c70db7f..8e71fb79c7a 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -16,12 +16,14 @@ internal sealed class CircuitStateController : IDisposable private readonly ResilienceStrategyTelemetry _telemetry; private readonly CircuitBehavior _behavior; private readonly TimeSpan _breakDuration; + private readonly Func>? _breakDurationGenerator; private DateTimeOffset _blockedUntil; private CircuitState _circuitState = CircuitState.Closed; private Outcome? _lastOutcome; private BrokenCircuitException _breakingException = new(); private bool _disposed; +#pragma warning disable S107 public CircuitStateController( TimeSpan breakDuration, Func, ValueTask>? onOpened, @@ -29,7 +31,9 @@ public CircuitStateController( Func? onHalfOpen, CircuitBehavior behavior, TimeProvider timeProvider, - ResilienceStrategyTelemetry telemetry) + ResilienceStrategyTelemetry telemetry, + Func>? breakDurationGenerator = null) +#pragma warning restore S107 { _breakDuration = breakDuration; _onOpened = onOpened; @@ -38,6 +42,7 @@ public CircuitStateController( _behavior = behavior; _timeProvider = timeProvider; _telemetry = telemetry; + _breakDurationGenerator = breakDurationGenerator; } public CircuitState CircuitState @@ -314,6 +319,15 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration scheduledTask = null; var utcNow = _timeProvider.GetUtcNow(); + if (_breakDurationGenerator is not null) + { +#pragma warning disable CA2012 +#pragma warning disable S1226 + breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)).GetAwaiter().GetResult(); +#pragma warning restore S1226 +#pragma warning restore CA2012 + } + _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; var transitionedState = _circuitState; diff --git a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs index 0e014526b54..e92e72d8f39 100644 --- a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs +++ b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs @@ -1,15 +1,15 @@ namespace Polly.CircuitBreaker.Health; -internal readonly record struct HealthInfo(int Throughput, double FailureRate) +internal readonly record struct HealthInfo(int Throughput, double FailureRate, int FailureCount) { public static HealthInfo Create(int successes, int failures) { var total = successes + failures; if (total == 0) { - return new HealthInfo(0, 0); + return new HealthInfo(0, 0, failures); } - return new(total, failures / (double)total); + return new(total, failures / (double)total, failures); } } diff --git a/src/Polly.Core/PublicAPI.Unshipped.txt b/src/Polly.Core/PublicAPI.Unshipped.txt index ab058de62d4..a84ad565317 100644 --- a/src/Polly.Core/PublicAPI.Unshipped.txt +++ b/src/Polly.Core/PublicAPI.Unshipped.txt @@ -1 +1,9 @@ #nullable enable +Polly.CircuitBreaker.BreakDurationGeneratorArguments +Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments() -> void +Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments(double failureRate, int failureCount, Polly.ResilienceContext! context) -> void +Polly.CircuitBreaker.BreakDurationGeneratorArguments.Context.get -> Polly.ResilienceContext! +Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureCount.get -> int +Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureRate.get -> double +Polly.CircuitBreaker.CircuitBreakerStrategyOptions.BreakDurationGenerator.get -> System.Func>? +Polly.CircuitBreaker.CircuitBreakerStrategyOptions.BreakDurationGenerator.set -> void diff --git a/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs new file mode 100644 index 00000000000..5afa53fe6ae --- /dev/null +++ b/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs @@ -0,0 +1,43 @@ +using Polly; +using Polly.CircuitBreaker; + +namespace Polly.Core.Tests.CircuitBreaker; + +public class BreakDurationGeneratorArgumentsTests +{ + [Fact] + public void Constructor_ShouldSetFailureRate() + { + double expectedFailureRate = 0.5; + int failureCount = 10; + var context = new ResilienceContext(); + + var args = new BreakDurationGeneratorArguments(expectedFailureRate, failureCount, context); + + args.FailureRate.Should().Be(expectedFailureRate); + } + + [Fact] + public void Constructor_ShouldSetFailureCount() + { + double failureRate = 0.5; + int expectedFailureCount = 10; + var context = new ResilienceContext(); + + var args = new BreakDurationGeneratorArguments(failureRate, expectedFailureCount, context); + + args.FailureCount.Should().Be(expectedFailureCount); + } + + [Fact] + public void Constructor_ShouldSetContext() + { + double failureRate = 0.5; + int failureCount = 10; + var expectedContext = new ResilienceContext(); + + var args = new BreakDurationGeneratorArguments(failureRate, failureCount, expectedContext); + + args.Context.Should().Be(expectedContext); + } +} diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs index def5bd3c73e..c0cc0462207 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs @@ -8,20 +8,21 @@ public class AdvancedCircuitBehaviorTests { private HealthMetrics _metrics = Substitute.For(TimeProvider.System); - [InlineData(10, 10, 0.0, 0.1, false)] - [InlineData(10, 10, 0.1, 0.1, true)] - [InlineData(10, 10, 0.2, 0.1, true)] - [InlineData(11, 10, 0.2, 0.1, true)] - [InlineData(9, 10, 0.1, 0.1, false)] + [InlineData(10, 10, 0.0, 0.1, 0, false)] + [InlineData(10, 10, 0.1, 0.1, 1, true)] + [InlineData(10, 10, 0.2, 0.1, 2, true)] + [InlineData(11, 10, 0.2, 0.1, 3, true)] + [InlineData(9, 10, 0.1, 0.1, 4, false)] [Theory] public void OnActionFailure_WhenClosed_EnsureCorrectBehavior( int throughput, int minimumThruput, double failureRate, double failureThreshold, + int failureCount, bool expectedShouldBreak) { - _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate)); + _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount)); var behavior = new AdvancedCircuitBehavior(failureThreshold, minimumThruput, _metrics); @@ -41,7 +42,6 @@ public void OnActionFailure_State_EnsureCorrectCalls(CircuitState state, bool sh _metrics = Substitute.For(TimeProvider.System); var sut = Create(); - sut.OnActionFailure(state, out var shouldBreak); shouldBreak.Should().BeFalse(); @@ -66,6 +66,27 @@ public void OnCircuitClosed_Ok() _metrics.Received(1).Reset(); } + [Theory] + [InlineData(10, 0.0, 0)] + [InlineData(10, 0.1, 1)] + [InlineData(10, 0.2, 2)] + [InlineData(11, 0.2, 3)] + [InlineData(9, 0.1, 4)] + public void BehaviorProperties_ShouldReflectHealthInfoValues( + int throughput, + double failureRate, + int failureCount) + { + var anyFailureThreshold = 10; + var anyMinimumThruput = 100; + + _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount)); + var behavior = new AdvancedCircuitBehavior(anyFailureThreshold, anyMinimumThruput, _metrics); + + behavior.FailureCount.Should().Be(failureCount, "because the FailureCount should match the HealthInfo"); + behavior.FailureRate.Should().Be(failureRate, "because the FailureRate should match the HealthInfo"); + } + private AdvancedCircuitBehavior Create() { return new(CircuitBreakerConstants.DefaultFailureRatio, CircuitBreakerConstants.DefaultMinimumThroughput, _metrics); diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index dda8bea0618..3a7922c1162 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -305,6 +305,41 @@ public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state, } } + [Fact] + public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() + { + // arrange + using var controller = CreateController(new() + { + FailureRatio = 0, + MinimumThroughput = 0, + SamplingDuration = default, + BreakDuration = TimeSpan.FromMinutes(1), + BreakDurationGenerator = static args => new ValueTask(TimeSpan.FromMinutes(args.FailureCount)), + OnClosed = null, + OnOpened = null, + OnHalfOpened = null, + ManualControl = null, + StateProvider = null + }); + + await TransitionToState(controller, CircuitState.Closed); + + var utcNow = DateTimeOffset.MaxValue; + + _timeProvider.SetUtcNow(utcNow); + _circuitBehavior.FailureCount.Returns(1); + _circuitBehavior.When(v => v.OnActionFailure(CircuitState.Closed, out Arg.Any())) + .Do(x => x[1] = true); + + // act + await controller.OnActionFailureAsync(Outcome.FromResult(99), ResilienceContextPool.Shared.Get()); + + // assert + var blockedTill = GetBlockedTill(controller); + blockedTill.Should().Be(utcNow); + } + [InlineData(true)] [InlineData(false)] [Theory] @@ -470,4 +505,14 @@ private async Task OpenCircuit(CircuitStateController controller, Outcome CreateController(CircuitBreakerStrategyOptions options) => new( + options.BreakDuration, + options.OnOpened, + options.OnClosed, + options.OnHalfOpened, + _circuitBehavior, + _timeProvider, + TestUtilities.CreateResilienceTelemetry(_telemetryListener), + options.BreakDurationGenerator); } diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs index 71628fc2af1..3a53d02515b 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs @@ -17,4 +17,50 @@ public void Create_Ok(int samplingDurationMs, Type expectedType) .Should() .BeOfType(expectedType); } + + [Fact] + public void HealthInfo_WithZeroTotal_ShouldSetValuesCorrectly() + { + // Arrange & Act + var result = HealthInfo.Create(0, 0); + + // Assert + result.Throughput.Should().Be(0); + result.FailureRate.Should().Be(0); + result.FailureCount.Should().Be(0); + } + + [Fact] + public void HealthInfo_ParameterizedConstructor_ShouldSetProperties() + { + // Arrange + int expectedThroughput = 100; + double expectedFailureRate = 0.25; + int expectedFailureCount = 25; + + // Act + var result = new HealthInfo(expectedThroughput, expectedFailureRate, expectedFailureCount); + + // Assert + result.Throughput.Should().Be(expectedThroughput); + result.FailureRate.Should().Be(expectedFailureRate); + result.FailureCount.Should().Be(expectedFailureCount); + } + + [Fact] + public void HealthInfo_Constructor_ShouldSetValuesCorrectly() + { + // Arrange + int throughput = 10; + double failureRate = 0.2; + int failureCount = 2; + + // Act + var result = new HealthInfo(throughput, failureRate, failureCount); + + // Assert + result.Throughput.Should().Be(throughput); + result.FailureRate.Should().Be(failureRate); + result.FailureCount.Should().Be(failureCount); + } } diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs index 2c4d5d8005d..60efc1c3b09 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs @@ -62,11 +62,11 @@ public void GetHealthInfo_EnsureWindowRespected() _timeProvider.Advance(TimeSpan.FromSeconds(2)); health.Add(metrics.GetHealthInfo()); - health[0].Should().Be(new HealthInfo(2, 0.5)); - health[1].Should().Be(new HealthInfo(4, 0.5)); - health[3].Should().Be(new HealthInfo(8, 0.25)); - health[4].Should().Be(new HealthInfo(8, 0.125)); - health[5].Should().Be(new HealthInfo(6, 0.0)); + health[0].Should().Be(new HealthInfo(2, 0.5, 1)); + health[1].Should().Be(new HealthInfo(4, 0.5, 2)); + health[3].Should().Be(new HealthInfo(8, 0.25, 2)); + health[4].Should().Be(new HealthInfo(8, 0.125, 1)); + health[5].Should().Be(new HealthInfo(6, 0.0, 0)); } [Fact] @@ -109,7 +109,7 @@ public void GetHealthInfo_SamplingDurationRespected(bool variance) _timeProvider.Advance(_samplingDuration + (variance ? TimeSpan.FromMilliseconds(1) : TimeSpan.Zero)); - metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0)); + metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0, 0)); } private RollingHealthMetrics Create() => new(_samplingDuration, _windows, _timeProvider);