From bf0aaa596397c8bb94f676e4b8417ebc32d6049c Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 29 Jul 2026 12:07:05 +0100 Subject: [PATCH] make AA configuration API consistent --- docs/ActiveActive.md | 216 +++++++++--- .../Availability/CircuitBreaker.cs | 40 ++- .../Availability/DatabaseExtensions.cs | 40 ++- .../HealthCheck.ConnectedProbe.cs | 23 -- .../Availability/HealthCheck.Execute.cs | 48 ++- .../HealthCheck.HealthCheckProbe.cs | 83 ----- .../HealthCheck.HealthCheckProbeContext.cs | 43 --- .../HealthCheck.HealthCheckProbePolicy.cs | 105 ------ .../Availability/HealthCheck.cs | 225 +++++++----- .../Availability/HealthCheckContext.cs | 30 ++ .../HealthCheckProbe.Connected.cs | 20 ++ .../Availability/HealthCheckProbe.None.cs | 21 ++ ....PingProbe.cs => HealthCheckProbe.Ping.cs} | 17 +- ...Probe.cs => HealthCheckProbe.StringSet.cs} | 17 +- .../Availability/HealthCheckProbe.cs | 59 ++++ .../Availability/HealthCheckProbeContext.cs | 40 +++ .../Availability/HealthCheckProbePolicy.cs | 102 ++++++ .../Availability/HealthCheckResult.cs | 26 ++ .../Availability/MultiGroupMultiplexer.cs | 100 ++++-- .../Availability/MultiGroupOptions.cs | 155 ++++++-- .../Availability/RetryController.cs | 34 +- .../Availability/RetryDatabase.cs | 2 + .../Availability/RetryPolicy.cs | 240 ++++++++++--- .../Availability/RetryResult.cs | 28 ++ .../ConfigurationOptions.cs | 21 +- .../ConnectionMultiplexer.cs | 54 ++- .../Interfaces/IConnectionGroup.cs | 6 + src/StackExchange.Redis/PhysicalConnection.cs | 5 +- .../PublicAPI/PublicAPI.Unshipped.txt | 180 ++++++---- .../StackExchange.Redis.csproj | 3 + .../AvailabilityConfigTests.cs | 333 ++++++++++++++++++ .../StackExchange.Redis.Tests/ConfigTests.cs | 2 +- .../ControllableProbe.cs | 6 +- .../HealthCheckPolicyUnitTests.cs | 2 +- .../MultiGroupTests/BasicMultiGroupTests.cs | 10 +- .../CircuitBreakerRerouteTests.cs | 8 +- .../GroupConfigResolutionTests.cs | 102 ++++++ .../RetryTests/CommandRetryPolicyUnitTests.cs | 26 +- .../RetryTests/RetryEndToEndTests.cs | 30 +- 39 files changed, 1802 insertions(+), 700 deletions(-) delete mode 100644 src/StackExchange.Redis/Availability/HealthCheck.ConnectedProbe.cs delete mode 100644 src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbe.cs delete mode 100644 src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbeContext.cs delete mode 100644 src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbePolicy.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckContext.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckProbe.Connected.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckProbe.None.cs rename src/StackExchange.Redis/Availability/{HealthCheck.PingProbe.cs => HealthCheckProbe.Ping.cs} (50%) rename src/StackExchange.Redis/Availability/{HealthCheck.StringSetProbe.cs => HealthCheckProbe.StringSet.cs} (79%) create mode 100644 src/StackExchange.Redis/Availability/HealthCheckProbe.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckProbeContext.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckResult.cs create mode 100644 src/StackExchange.Redis/Availability/RetryResult.cs create mode 100644 tests/StackExchange.Redis.Tests/AvailabilityConfigTests.cs create mode 100644 tests/StackExchange.Redis.Tests/MultiGroupTests/GroupConfigResolutionTests.cs diff --git a/docs/ActiveActive.md b/docs/ActiveActive.md index 80f30c300..e6328910e 100644 --- a/docs/ActiveActive.md +++ b/docs/ActiveActive.md @@ -15,6 +15,53 @@ The features for Active:Active are available in the `Availability` sub-namespace using StackExchange.Redis; using StackExchange.Redis.Availability; ``` + +### How the configuration types work + +Every configurable piece in this namespace follows the same three-part shape, so once you have learned one you have learned all of them: + +1. The **policy type** (`HealthCheck`, `CircuitBreaker`, `RetryPolicy`) is **immutable** and safe to share between members and connections. It exposes a static `Default` and a static `None`. +2. Each policy has a nested **`Builder`** carrying the mutable knobs. A new `Builder` already starts from the default values, so you only set what you want to change; `Create()` validates the values and returns the policy, and a `Builder` also converts *implicitly* to its policy, so it can be assigned or passed inline. +3. **`MultiGroupOptions`** (itself immutable, with its own `Builder`) holds the group-wide defaults; the matching nullable property on `ConnectionGroupMember` overrides them per member. + +```csharp +// the same pattern, three times; each builder only mentions what differs from the default +HealthCheck healthCheck = new HealthCheck.Builder { ProbeCount = 5 }; +CircuitBreaker breaker = new CircuitBreaker.Builder { FailureRateThreshold = 25 }; +RetryPolicy retry = new RetryPolicy.Builder { MaxAttempts = 5 }; + +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + HealthCheck = healthCheck, + CircuitBreaker = breaker, + RetryPolicy = retry, + HealthCheckInterval = TimeSpan.FromSeconds(2), +}; +``` + +Anything you leave out keeps its default, so a group that only wants a longer failback is just: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder { FailbackDelay = TimeSpan.FromMinutes(2) }; +``` + +Because the policies are immutable, there is no question of whether a change "takes effect" after connecting: to change something, build a new instance. Values are validated once, in `Create()`, which throws `ArgumentOutOfRangeException`/`ArgumentException` naming the offending builder property - so a bad `ProbeCount` or `MaxAttempts` fails at the point you configure it, not later. + +### Where each setting lives + +| Setting | Group-wide default | Per-member override | +|---------|--------------------|---------------------| +| `HealthCheck` | `MultiGroupOptions.HealthCheck` | `ConnectionGroupMember.HealthCheck` | +| `CircuitBreaker` | `MultiGroupOptions.CircuitBreaker` | `ConnectionGroupMember.CircuitBreaker`, else that member's `ConfigurationOptions.CircuitBreaker` | +| `RetryPolicy` | `MultiGroupOptions.RetryPolicy` | *(none; retry is applied per-database via `WithRetry`)* | +| `HealthCheckInterval` | `MultiGroupOptions.HealthCheckInterval` | *(none; it is the group's re-evaluation cadence)* | +| `FailbackDelay` | `MultiGroupOptions.FailbackDelay` | `ConnectionGroupMember.FailbackDelay` | +| `Weight` | *(none)* | `ConnectionGroupMember.Weight` | +| `SkipInitialHealthCheck` | *(none)* | `ConnectionGroupMember.SkipInitialHealthCheck` | + +Resolution is always "member override, else group default". Resolving a group default never writes back into your `ConfigurationOptions` - you can safely reuse one `ConfigurationOptions` for a group member and for an unrelated `Connect` without the group's policies leaking across. + +`ConfigurationOptions` carries only the two availability settings that are meaningful for a **single** connection - `CircuitBreaker` and `RetryPolicy`. `HealthCheck` is a group-only concept and lives on `ConnectionGroupMember`. The library automatically selects the best available endpoint based on: 1. **Availability** - Connected endpoints are always preferred over disconnected ones @@ -213,22 +260,22 @@ The Active:Active feature includes configurable health checking to monitor the h ### Basic Health Check Configuration -Health checks are configured globally for all members using the `MultiGroupOptions` parameter: +Health checks are configured for all members via `MultiGroupOptions`, and can be overridden per member. Every knob is shown here for orientation, with **the value it already defaults to** - in real code you would set only the ones you want to change: ```csharp -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { - Interval = TimeSpan.FromSeconds(5), // How often to check health - ProbeCount = 3, // Maximum number of probe attempts per check - ProbeTimeout = TimeSpan.FromSeconds(3), // Timeout for each probe attempt + ProbeCount = 3, // Maximum number of probe attempts per check + ProbeTimeout = TimeSpan.FromSeconds(3), // Timeout for each probe attempt ProbeInterval = TimeSpan.FromMilliseconds(500), // Delay between failed probes - Probe = HealthCheckProbe.Ping, // Which probe type to use + Probe = HealthCheckProbe.Ping, // Which probe type to use ProbePolicy = HealthCheckProbePolicy.AllSuccess // Evaluation policy }; -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { - HealthCheck = healthCheck + HealthCheck = healthCheck, + HealthCheckInterval = TimeSpan.FromSeconds(5), // How often checks run (a group-level concern) }; ConnectionGroupMember[] members = [ @@ -239,6 +286,8 @@ ConnectionGroupMember[] members = [ await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); ``` +Note that **how often** checks run is `MultiGroupOptions.HealthCheckInterval`, not a property of the check: it is the cadence at which the group re-evaluates *all* members, so a per-member value would be meaningless. + ### Using Default Health Checks If you don't specify a health check, the system uses sensible defaults: @@ -248,39 +297,63 @@ If you don't specify a health check, the system uses sensible defaults: await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); // Equivalent to: -var options = new MultiGroupOptions -{ - HealthCheck = HealthCheck.Default -}; +MultiGroupOptions options = new MultiGroupOptions.Builder { HealthCheck = HealthCheck.Default }; await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); ``` -You can also clone and customize the default: +A new `Builder` starts from the defaults, so customizing means setting only what differs: ```csharp -var customHealthCheck = HealthCheck.Default.Clone(); -customHealthCheck.Interval = TimeSpan.FromSeconds(15); -customHealthCheck.ProbeCount = 5; +HealthCheck customHealthCheck = new HealthCheck.Builder { ProbeCount = 5 }; // everything else defaulted -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { - HealthCheck = customHealthCheck + HealthCheck = customHealthCheck, + HealthCheckInterval = TimeSpan.FromSeconds(15), }; ``` +There is also a `Builder(policy)` overload, for when you want to start from an instance that *isn't* the default - for example, adjusting a policy you were handed: + +```csharp +// take the group's configured check and probe it harder for one member +HealthCheck stricter = new HealthCheck.Builder(conn.Options.HealthCheck) { ProbeCount = 9 }; +``` + ### Health Check Properties -The `HealthCheck` class provides several configurable properties: +The `HealthCheck.Builder` class provides several configurable properties: | Property | Default | Description | |----------|---------|-------------| -| `Interval` | 5 seconds | How frequently health checks are performed | | `ProbeCount` | 3 | Number of probe operations to perform per health check | | `ProbeTimeout` | 3 seconds | Maximum time allowed for an individual probe to complete | | `ProbeInterval` | 500 milliseconds | Delay between consecutive failed probes | | `Probe` | `Ping` | The probe operation to execute | | `ProbePolicy` | `AllSuccess` | Policy for evaluating multiple probe results | +### Per-member health checks, and turning them off + +A member can use its own check - including `HealthCheck.None`, which performs no probes at all and reports `Inconclusive`, leaving that member's selection driven purely by its observed connectivity (and by its circuit-breaker): + +```csharp +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 10 }, + + // a member we only want to reach for on connectivity grounds - never probe it + new("archive.redis.example.com:6379", name: "Archive") { Weight = 1, HealthCheck = HealthCheck.None }, + + // ...and one we want checked much more aggressively than the rest + new("us-west.redis.example.com:6379", name: "US West") + { + Weight = 5, + HealthCheck = new HealthCheck.Builder { ProbeCount = 5, ProbePolicy = HealthCheckProbePolicy.MajoritySuccess }, + }, +]; +``` + +To disable periodic checking for the whole group, set `MultiGroupOptions.HealthCheckInterval` to `TimeSpan.MaxValue`; the group is then only re-evaluated in response to connection events (such as a tripped circuit-breaker). + ### Built-in Probes StackExchange.Redis provides several built-in health check probes: @@ -290,7 +363,7 @@ StackExchange.Redis provides several built-in health check probes: The simplest probe that executes a `PING` command against the server: ```csharp -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { Probe = HealthCheckProbe.Ping }; @@ -303,7 +376,7 @@ This is the default and recommended probe for most scenarios as it's lightweight Checks the connection status without sending any commands: ```csharp -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { Probe = HealthCheckProbe.IsConnected }; @@ -316,7 +389,7 @@ This is even more lightweight than `Ping` but only verifies the socket connectio Performs a write operation to verify read/write capability: ```csharp -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { Probe = HealthCheckProbe.StringSet }; @@ -333,7 +406,7 @@ The probe policy determines how multiple probe results are evaluated to determin The health check passes if **any** probe succeeds. This provides the most lenient evaluation: ```csharp -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { ProbeCount = 3, ProbePolicy = HealthCheckProbePolicy.AnySuccess @@ -346,7 +419,7 @@ var healthCheck = new HealthCheck The health check passes only if **all** probes succeed. This provides the strictest evaluation: ```csharp -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { ProbeCount = 3, ProbePolicy = HealthCheckProbePolicy.AllSuccess @@ -359,7 +432,7 @@ var healthCheck = new HealthCheck The health check passes if a **majority** of probes succeed: ```csharp -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { ProbeCount = 3, ProbePolicy = HealthCheckProbePolicy.MajoritySuccess @@ -396,12 +469,12 @@ its state is scoped to exactly the connection whose health it is measuring; a re ### Configuring a Circuit Breaker for a Group -Circuit breakers are configured globally for all members via `MultiGroupOptions`, alongside the health check. The setting flows into every member connection: +Circuit breakers are configured for all members via `MultiGroupOptions`, alongside the health check. The setting flows into every member connection: ```csharp -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { - CircuitBreaker = CircuitBreaker.Default + CircuitBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 25 } }; ConnectionGroupMember[] members = [ @@ -418,17 +491,19 @@ If you don't specify one, `CircuitBreaker.Default` is used automatically: // these are equivalent await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); -var options = new MultiGroupOptions { CircuitBreaker = CircuitBreaker.Default }; +MultiGroupOptions options = new MultiGroupOptions.Builder { CircuitBreaker = CircuitBreaker.Default }; await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); ``` +A circuit breaker is also useful *without* a group: set `ConfigurationOptions.CircuitBreaker` and any connection will tear itself down when its traffic starts failing. For a group member, the effective breaker is the member's `CircuitBreaker`, else the member's own `ConfigurationOptions.CircuitBreaker`, else the group default. + ### Tuning the Default Circuit Breaker The default circuit breaker uses a rolling time-window: it counts successes and failures over a short window and trips once the failure rate crosses a threshold, provided enough failures have been seen to be statistically meaningful. Use `CircuitBreaker.Builder` to tune it: ```csharp -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { CircuitBreaker = new CircuitBreaker.Builder { @@ -454,7 +529,7 @@ Which faults count against the breaker is decided by *classification*, not by ex Use `CircuitBreaker.None` to opt out entirely: ```csharp -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { CircuitBreaker = CircuitBreaker.None }; @@ -467,8 +542,9 @@ Health checks and circuit breakers keep the *group* pointed at a healthy member; ```csharp await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); -// wrap the database once; reuse the wrapper like any other IDatabaseAsync -IDatabaseAsync db = conn.GetDatabase().WithRetry(new RetryPolicy()); +// wrap the database once; reuse the wrapper like any other IDatabaseAsync. +// the parameterless overload uses the policy configured on the connection (see below) +IDatabaseAsync db = conn.GetDatabase().WithRetry(); // a transient fault (e.g. the active member briefly returning LOADING) is retried // automatically; if the group fails over in the meantime, the retry lands on the new member @@ -479,9 +555,26 @@ var value = await db.StringGetAsync("mykey"); A retrying database can still *create* a transaction: `retryDb.CreateTransaction()` returns an `ITransactionAsync` whose `ExecuteAsync` is retried as a single unit. Each attempt replays the queued operations (and any `WATCH` constraints) against a fresh `MULTI`/`EXEC` - and, in an Active:Active group, onto whichever member is active at the time - so a transaction can ride out a failover just like a single command; the per-operation tasks handed back at build time resolve from the winning attempt. The retry-category gate (below) still applies, using the *most* side-effecting operation in the transaction: a transaction containing an `INCR` is treated as `CommandRetryWriteAccumulating`, so the default policy will not retry it unless you raise `MaxCommandRetryCategory`. +### Configuring the retry policy + +Like the health check and the circuit breaker, `RetryPolicy` can be set as a group-wide default - and the parameterless `WithRetry()` picks it up, so callers don't have to thread a policy through their code: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + RetryPolicy = new RetryPolicy.Builder { MaxAttempts = 5 }, +}; +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + +// uses MultiGroupOptions.RetryPolicy +IDatabaseAsync db = conn.GetDatabase().WithRetry(); +``` + +The same works for a single connection via `ConfigurationOptions.RetryPolicy`. `WithRetry()` resolves in this order: `MultiGroupOptions.RetryPolicy` for a connection group, `ConfigurationOptions.RetryPolicy` for a single connection, else `RetryPolicy.Default`. Pass a policy explicitly - `WithRetry(policy)` - to override that for one database, and use `RetryPolicy.None` to get a wrapper that never retries. + ### RetryPolicy settings -`RetryPolicy` controls how many times, how often, and how far an operation is retried: +`RetryPolicy.Builder` controls how many times, how often, and how far an operation is retried: | Property | Default | Description | |----------|---------|-------------| @@ -493,7 +586,7 @@ A retrying database can still *create* a transaction: `retryDb.CreateTransaction | `MaxCommandRetryCategory` | `CommandRetryWriteLastWins` | The most side-effecting command category that will be retried (see below) | ```csharp -var policy = new RetryPolicy +RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 5, RetryDelay = TimeSpan.FromMilliseconds(200), @@ -570,15 +663,24 @@ An unhealthy member is cleared in one of three ways: ### `FailbackDelay` -`MultiGroupOptions.FailbackDelay` is the interval a member must remain healthy — measured from its *most recent* failure — before it is automatically returned to rotation: +`MultiGroupOptions.FailbackDelay` is the interval a member must remain healthy - measured from its *most recent* failure - before it is automatically returned to rotation: ```csharp -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { FailbackDelay = TimeSpan.FromMinutes(2), // must be healthy for 2 minutes after its last failure }; ``` +Individual members can override this via `ConnectionGroupMember.FailbackDelay`, for example to hold a known-flaky region out of rotation for longer than the rest: + +```csharp +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 10 }, + new("flaky.redis.example.com:6379", name: "Flaky") { Weight = 5, FailbackDelay = TimeSpan.FromMinutes(10) }, +]; +``` + | Value | Behavior | |-------|----------| | `TimeSpan.Zero` (default) | Immediate failback — the member is eligible again as soon as a health check passes | @@ -795,14 +897,16 @@ You can implement custom health check logic by extending `HealthCheckProbe`. Not if the probe involves talking to data via a `RedisKey`, as on "cluster" configurations, it must be ensured that the key used resolves to the correct server; for this purpose, the `server.InventKey` method can be used: +A probe receives a `HealthCheckContext`, carrying the `Server` being probed and the `ProbeTimeout` budget: + ```csharp public abstract class CustomProbe : HealthCheckProbe { - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) + public override Task CheckHealthAsync(HealthCheckContext context) { // create a random key that routes to the correct server, using // the specified prefix - RedisKey key = server.InventKey("health-check/"); + RedisKey key = context.Server.InventKey("health-check/"); // ... } } @@ -814,14 +918,14 @@ Or more conveniently, the key-specific `KeyWriteHealthCheckProbe` encapsulates t public class CustomWriteProbe : KeyWriteHealthCheckProbe { public override async Task CheckHealthAsync( - HealthCheck healthCheck, + HealthCheckContext context, IDatabaseAsync database, RedisKey key) { try { var value = Guid.NewGuid().ToString(); - await database.StringSetAsync(key, value, expiry: healthCheck.ProbeTimeout); + await database.StringSetAsync(key, value, expiry: context.ProbeTimeout); bool isMatch = value == await database.StringGetAsync(key); return isMatch ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; @@ -834,6 +938,8 @@ public class CustomWriteProbe : KeyWriteHealthCheckProbe } ``` +> The context is passed **by value** rather than by `in`, because probes are typically `async`, and async methods cannot take by-ref parameters. The sibling `HealthCheckProbePolicy.Evaluate` is synchronous, so it does take `in HealthCheckProbeContext`. + ### Custom Probe Policies In addition to the inbuilt policies, custom policies can be implemented by extending `HealthCheckProbePolicy`. @@ -859,13 +965,13 @@ public class AtLeastPolicy(int requiredSuccesses) : HealthCheckProbePolicy } // Use the custom policy requiring at least 2 successes -var healthCheck = new HealthCheck +HealthCheck healthCheck = new HealthCheck.Builder { ProbeCount = 5, // Need enough probes to allow for the required successes ProbePolicy = new AtLeastPolicy(2) }; -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { HealthCheck = healthCheck }; @@ -915,7 +1021,7 @@ public sealed class ConsecutiveFailureBreaker(int limit) : CircuitBreaker } } -var options = new MultiGroupOptions +MultiGroupOptions options = new MultiGroupOptions.Builder { CircuitBreaker = new ConsecutiveFailureBreaker(limit: 5) }; @@ -925,24 +1031,36 @@ Keep `ObserveResult` cheap and thread-safe: it runs on the hot path for every co ### Custom Retry Policies -`RetryPolicy` is itself extensible: override `CanRetry(in FaultContext fault)` to make the retry decision yourself. It returns a `RetryPolicyResult` — `None` to give up, or a combination of `SameServer` and `FailoverServer` to indicate where a retry may be attempted. +`RetryPolicy` is itself extensible: override `CanRetry(in FaultContext fault)` to make the retry decision yourself. It returns a `RetryResult` - `None` to give up, or a combination of `SameServer` and `FailoverServer` to indicate where a retry may be attempted. The `FaultContext` gives you the classified `ErrorKind`, the `ConnectionFailureType`, and the command `Flags` (including its retry category) to base the decision on: ```csharp public sealed class ReadOnlyOnlyRetryPolicy : RetryPolicy { - public override RetryPolicyResult CanRetry(in FaultContext fault) + public override RetryResult CanRetry(in FaultContext fault) { // only ever retry pure reads, and only on the same server if (fault.ErrorKind == RedisErrorKind.Loading && (fault.Flags & CommandFlags.CommandRetryReadOnly) != 0) { - return RetryPolicyResult.SameServer; + return RetryResult.SameServer; } // note: base.CanRetry(fault) would apply the default logic - return RetryPolicyResult.None; + return RetryResult.None; } } IDatabaseAsync db = conn.GetDatabase().WithRetry(new ReadOnlyOnlyRetryPolicy()); ``` + +A derived policy inherits the default settings; to derive *and* change the settings, take a `Builder` and pass it to the base constructor: + +```csharp +public sealed class ReadOnlyOnlyRetryPolicy(RetryPolicy.Builder builder) : RetryPolicy(builder) +{ + public override RetryResult CanRetry(in FaultContext fault) => /* ... */; +} + +IDatabaseAsync db = conn.GetDatabase().WithRetry( + new ReadOnlyOnlyRetryPolicy(new RetryPolicy.Builder { MaxAttempts = 5 })); +``` diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index 07e72d3b5..fc876898e 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -15,10 +15,24 @@ namespace StackExchange.Redis.Availability; [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] public abstract class CircuitBreaker { + internal const double DefaultFailureRateThreshold = 10; + internal const int DefaultMinimumNumberOfFailures = 1000; + internal static readonly TimeSpan DefaultMetricsWindowSize = TimeSpan.FromSeconds(2); + + private static readonly CircuitBreaker DefaultInstance = new DefaultCircuitBreaker( +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list +#if NET8_0_OR_GREATER + null, +#endif +#pragma warning restore SA1114 + DefaultFailureRateThreshold, + DefaultMinimumNumberOfFailures, + DefaultMetricsWindowSize); + /// - /// Default circuit-breaker logic. + /// Default circuit-breaker logic: trips when the failure rate over a short rolling window crosses a threshold. /// - public static CircuitBreaker Default => Builder.DefaultInstance; + public static CircuitBreaker Default => DefaultInstance; /// /// No circuit-breaker logic is applied. @@ -28,22 +42,9 @@ public abstract class CircuitBreaker /// /// Allows configuration of the default implementation. /// - public class Builder + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder { - private const double DefaultFailureRateThreshold = 10; - private const int DefaultMinimumNumberOfFailures = 1000; - private static readonly TimeSpan DefaultMetricsWindowSize = TimeSpan.FromSeconds(2); - - internal static CircuitBreaker DefaultInstance = new DefaultCircuitBreaker( -#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list -#if NET8_0_OR_GREATER - null, -#endif -#pragma warning restore SA1114 - DefaultFailureRateThreshold, - DefaultMinimumNumberOfFailures, - DefaultMetricsWindowSize); - /// /// Percentage of failures to trigger circuit breaker. /// @@ -73,6 +74,11 @@ public class Builder /// public CircuitBreaker Create() { + if (FailureRateThreshold is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(FailureRateThreshold), FailureRateThreshold, "A percentage between 0 and 100 is required."); + if (MinimumNumberOfFailures < 1) throw new ArgumentOutOfRangeException(nameof(MinimumNumberOfFailures), MinimumNumberOfFailures, "At least one failure is required; use CircuitBreaker.None to disable circuit-breaking."); + if (MetricsWindowSize <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "A positive window is required."); + if (MetricsWindowSize.TotalSeconds > int.MaxValue) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "The window is too large."); + if ((FailureRateThreshold is DefaultFailureRateThreshold #if NET8_0_OR_GREATER & TimeProvider is null diff --git a/src/StackExchange.Redis/Availability/DatabaseExtensions.cs b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs index 9733bf485..646528884 100644 --- a/src/StackExchange.Redis/Availability/DatabaseExtensions.cs +++ b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs @@ -1,20 +1,36 @@ using System.Diagnostics.CodeAnalysis; using RESPite; -namespace StackExchange.Redis.Availability +namespace StackExchange.Redis.Availability; + +/// +/// Provides availability-related extension methods (such as ) to database instances. +/// +public static class DatabaseExtensions { /// - /// Provides availability-related extension methods (such as ) to database instances. + /// Automatically retry operations when connection failure occurs. This has deep integration with + /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and + /// respect command effect categorization. /// - public static class DatabaseExtensions + /// The database to wrap. + /// + /// The policy to apply; when (the default), the policy configured for the + /// underlying connection is used - for a connection group, + /// for a single connection, else + /// . + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy? retryPolicy = null) + => new RetryDatabase(database, retryPolicy ?? ResolveRetryPolicy(database)); + + // IDatabaseAsync always exposes its multiplexer (via IRedisAsync), so the configured policy is reachable + // without the caller having to thread it through; note IConnectionMultiplexer is a public interface that + // callers may implement or mock, so every step here degrades to the default rather than assuming a type + private static RetryPolicy ResolveRetryPolicy(IDatabaseAsync database) => database.Multiplexer switch { - /// - /// Automatically retry operations when connection failure occurs. This has deep integration with - /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and - /// respect command effect categorization. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy retryPolicy) - => new RetryDatabase(database, retryPolicy); - } + IConnectionGroup group => group.Options.RetryPolicy, + IInternalConnectionMultiplexer muxer => muxer.RawConfig.RetryPolicy ?? RetryPolicy.Default, + _ => RetryPolicy.Default, + }; } diff --git a/src/StackExchange.Redis/Availability/HealthCheck.ConnectedProbe.cs b/src/StackExchange.Redis/Availability/HealthCheck.ConnectedProbe.cs deleted file mode 100644 index 3850aaa61..000000000 --- a/src/StackExchange.Redis/Availability/HealthCheck.ConnectedProbe.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Threading.Tasks; - -namespace StackExchange.Redis.Availability; - -public sealed partial class HealthCheck -{ - public partial class HealthCheckProbe - { - /// - /// Report health using the property, without any additional tests. - /// - public static HealthCheckProbe IsConnected => ConnectedProbe.Instance; - } - - private sealed class ConnectedProbe : HealthCheckProbe - { - public static ConnectedProbe Instance { get; } = new(); - private ConnectedProbe() { } - - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) - => server.IsConnected ? HealthyTask : UnhealthyTask; - } -} diff --git a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs index 968b4e81e..1ee5dda8a 100644 --- a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs @@ -11,7 +11,9 @@ public sealed partial class HealthCheck /// Evaluate the health of the specified multiplexer, by evaluating all endpoints. /// public Task CheckHealthAsync(IConnectionMultiplexer multiplexer) - => multiplexer.IsConnected ? CheckHealthCoreAsync(multiplexer) : HealthCheckProbe.UnhealthyTask; + => !IsEnabled ? HealthCheckProbe.InconclusiveTask + : multiplexer.IsConnected ? CheckHealthCoreAsync(multiplexer) + : HealthCheckProbe.UnhealthyTask; private async Task CheckHealthCoreAsync(IConnectionMultiplexer multiplexer) { @@ -51,21 +53,32 @@ private async Task CheckHealthCoreAsync(IConnectionMultiplexe internal int TotalTimeoutMillis() { - int count = ProbeCount; - if (count <= 0) - { - Debug.Fail("We shouldn't get as far as calculating timeouts with a non-positive probe count."); - return 0; - } + bool valid = TryComputeTotalTimeoutMillis(ProbeCount, ProbeTimeout, ProbeInterval, out int millis); + Debug.Assert(valid, "The probe budget is validated by HealthCheck.Builder.Create, so should always be usable here."); + return millis; + } - TimeSpan probeTimeout = ProbeTimeout, probeInterval = ProbeInterval; + // the total time budget for a full health check, in milliseconds; shared with Builder.Create, which uses + // it to reject a configuration whose budget cannot be expressed (rather than overflowing at check time) + private static bool TryComputeTotalTimeoutMillis(int probeCount, TimeSpan probeTimeout, TimeSpan probeInterval, out int millis) + { + millis = 0; + if (probeCount < 1 || probeTimeout <= TimeSpan.Zero || probeInterval < TimeSpan.Zero) return false; - // the first probe doesn't have an interval before it, the rest do - var totalTicks = probeTimeout.Ticks - + ((probeTimeout.Ticks + probeInterval.Ticks) * (count - 1)); - var millis = (int)TimeSpan.FromTicks(totalTicks).TotalMilliseconds; - Debug.Assert(millis > 0, "Total timeout should be positive"); - return millis; + try + { + // the first probe doesn't have an interval before it, the rest do + long totalTicks = checked(probeTimeout.Ticks + ((probeTimeout.Ticks + probeInterval.Ticks) * (probeCount - 1))); + long totalMillis = totalTicks / TimeSpan.TicksPerMillisecond; + if (totalMillis is <= 0 or > int.MaxValue) return false; + + millis = (int)totalMillis; + return true; + } + catch (OverflowException) + { + return false; + } } // apply timeout and collation logic to a group of probes @@ -130,19 +143,22 @@ internal static void PutReusablePending(ref Task[]? field, re /// Evaluate the health of an endpoint. /// public Task CheckHealthAsync(IServer server) - => server.IsConnected ? CheckHealthCoreAsync(server) : HealthCheckProbe.UnhealthyTask; + => !IsEnabled ? HealthCheckProbe.InconclusiveTask + : server.IsConnected ? CheckHealthCoreAsync(server) + : HealthCheckProbe.UnhealthyTask; private async Task CheckHealthCoreAsync(IServer server) { try { int timeout = (int)ProbeTimeout.TotalMilliseconds, success = 0, failure = 0, remaining = ProbeCount; + HealthCheckContext context = new(server, ProbeTimeout); while (remaining > 0) { HealthCheckResult probeResult; try { - var pendingProbe = Probe.CheckHealthAsync(this, server); + var pendingProbe = Probe.CheckHealthAsync(context); probeResult = await pendingProbe.TimeoutAfter(timeout).ForAwait() ? await pendingProbe.ForAwait() // completed : HealthCheckResult.Unhealthy; // timeout diff --git a/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbe.cs b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbe.cs deleted file mode 100644 index 6789bc891..000000000 --- a/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbe.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Threading.Tasks; -using RESPite; - -namespace StackExchange.Redis.Availability; - -public sealed partial class HealthCheck -{ - /// - /// Describes an operation to perform as part of a health check. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public abstract partial class HealthCheckProbe - { - /// - /// Check the health of the specified endpoint. - /// - public abstract Task CheckHealthAsync(HealthCheck healthCheck, IServer server); - - private static Task? _inconclusive, _healthy, _unhealthy; - - /// - /// Reports a memoized probe that was skipped without being evaluated. - /// - protected internal static Task InconclusiveTask => _inconclusive ??= Task.FromResult(HealthCheckResult.Inconclusive); - - /// - /// Reports a memoized probe that was healthy. - /// - protected internal static Task HealthyTask => _healthy ??= Task.FromResult(HealthCheckResult.Healthy); - - /// - /// Reports a memoized probe that was unhealthy. - /// - protected internal static Task UnhealthyTask => _unhealthy ??= Task.FromResult(HealthCheckResult.Unhealthy); - } - - /// - /// Describes a key-based (write) operation to perform as part of a health check. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public abstract class KeyWriteHealthCheckProbe : HealthCheckProbe - { - /// - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) - { - if (server.IsReplica) return InconclusiveTask; - - RedisKey key = server.InventKey("health-check/"); - if (key.IsNull) return InconclusiveTask; - Debug.Assert(server.Multiplexer.GetServer(key).EndPoint == server.EndPoint, "Key was not routed to the correct endpoint"); - return CheckHealthAsync(healthCheck, server.Multiplexer.GetDatabase(), key); - } - - /// - /// Check the health of the specified database using the provided key. - /// - public abstract Task CheckHealthAsync(HealthCheck healthCheck, IDatabaseAsync database, RedisKey key); - } - - /// - /// Indicates the result of a health check. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public enum HealthCheckResult - { - /// - /// The health check was skipped or could not be determined. - /// - Inconclusive, - - /// - /// The health check was successful. - /// - Healthy, - - /// - /// The health check failed. - /// - Unhealthy, - } -} diff --git a/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbeContext.cs b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbeContext.cs deleted file mode 100644 index 82062e830..000000000 --- a/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbeContext.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; -using RESPite; - -namespace StackExchange.Redis.Availability; - -public sealed partial class HealthCheck -{ - /// - /// Represents the context of a health check probe. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public readonly struct HealthCheckProbeContext(HealthCheckResult result, int success, int failure, int remaining, TimeSpan probeInterval) - { - /// - public override string ToString() => $"Result: {Result}, Success: {Success}, Failure: {Failure}, Remaining: {Remaining}, ProbeInterval: {ProbeInterval}"; - - /// - /// Gets the most recent result. - /// - public HealthCheckResult Result => result; - - /// - /// Gets the number of successful health checks. - /// - public int Success => success; - - /// - /// Gets the number of failed health checks. - /// - public int Failure => failure; - - /// - /// Gets the number of remaining health checks. - /// - public int Remaining => remaining; - - /// - /// Gets the interval to wait before the next probe attempt. - /// - public TimeSpan ProbeInterval => probeInterval; - } -} diff --git a/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbePolicy.cs b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbePolicy.cs deleted file mode 100644 index 5e465dae2..000000000 --- a/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbePolicy.cs +++ /dev/null @@ -1,105 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; -using RESPite; - -namespace StackExchange.Redis.Availability; - -public sealed partial class HealthCheck -{ - /// - /// Attempt to evaluate the outcome of a series of health check operations. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public abstract class HealthCheckProbePolicy - { - /// - /// Attempt to evaluate the policy given the current context. - /// - /// The state of the probes so far. - /// The result of the policy evaluation. - public abstract HealthCheckResult Evaluate(in HealthCheckProbeContext context); - - /// - /// Get the interval to wait before the next probe attempt. - /// - internal TimeSpan GetEffectiveProbeInterval(in HealthCheckProbeContext context) - { - // if we make this public / overrideable, we will need to think about the max delay timeout - - // only delay between failures - return context.Result is HealthCheckResult.Unhealthy ? context.ProbeInterval : TimeSpan.Zero; - } - - /// - /// Require all probes to succeed. - /// - public static HealthCheckProbePolicy AllSuccess => AllSuccessHealthCheckProbePolicy.Instance; - - /// - /// Require at least one probe to succeed. - /// - public static HealthCheckProbePolicy AnySuccess => AnySuccessHealthCheckProbePolicy.Instance; - - /// - /// Require a majority of probes to succeed. - /// - public static HealthCheckProbePolicy MajoritySuccess => MajoritySuccessHealthCheckProbePolicy.Instance; - - private sealed class AllSuccessHealthCheckProbePolicy : HealthCheckProbePolicy - { - public static readonly AllSuccessHealthCheckProbePolicy Instance = new(); - private AllSuccessHealthCheckProbePolicy() { } - - public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) - { - // Fail as soon as we have any failure - if (context.Failure > 0) return HealthCheckResult.Unhealthy; - - // Succeed only when all probes have succeeded (no remaining) - if (context.Remaining == 0) return HealthCheckResult.Healthy; - - // Can't determine yet - return HealthCheckResult.Inconclusive; - } - } - - private sealed class AnySuccessHealthCheckProbePolicy : HealthCheckProbePolicy - { - public static readonly AnySuccessHealthCheckProbePolicy Instance = new(); - private AnySuccessHealthCheckProbePolicy() { } - - public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) - { - // Succeed as soon as we have any success - if (context.Success > 0) return HealthCheckResult.Healthy; - - // Fail only when all probes have failed (no remaining) - if (context.Remaining == 0) return HealthCheckResult.Unhealthy; - - // Can't determine yet - return HealthCheckResult.Inconclusive; - } - } - - private sealed class MajoritySuccessHealthCheckProbePolicy : HealthCheckProbePolicy - { - public static readonly MajoritySuccessHealthCheckProbePolicy Instance = new(); - private MajoritySuccessHealthCheckProbePolicy() { } - - public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) - { - int total = context.Success + context.Failure + context.Remaining; - int majority = (total / 2) + 1; - - // Succeed as soon as we have enough successes for a majority - if (context.Success >= majority) return HealthCheckResult.Healthy; - - // Fail as soon as we have enough failures to make a majority impossible - if (context.Failure >= majority) return HealthCheckResult.Unhealthy; - - // Can't determine yet - return HealthCheckResult.Inconclusive; - } - } - } -} diff --git a/src/StackExchange.Redis/Availability/HealthCheck.cs b/src/StackExchange.Redis/Availability/HealthCheck.cs index 8345ed01b..3419ad85d 100644 --- a/src/StackExchange.Redis/Availability/HealthCheck.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.cs @@ -1,7 +1,5 @@ using System; using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; -using System.Threading; using RESPite; namespace StackExchange.Redis.Availability; @@ -9,117 +7,180 @@ namespace StackExchange.Redis.Availability; /// /// Describes a health check to perform against instances. /// +/// +/// Instances are immutable and safe to share between members; use to configure +/// a custom check. Note that how often checks run is a group-level concern, configured via +/// , not a property of the check itself. +/// [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] -public sealed partial class HealthCheck : ICloneable +public sealed partial class HealthCheck { - private static HealthCheck? _default; + internal const int DefaultProbeCount = 3; + internal static readonly TimeSpan DefaultProbeTimeout = TimeSpan.FromSeconds(3); + internal static readonly TimeSpan DefaultProbeInterval = TimeSpan.FromMilliseconds(500); + + private static readonly HealthCheck DefaultInstance = new( + enabled: true, + DefaultProbeCount, + DefaultProbeTimeout, + DefaultProbeInterval, + HealthCheckProbe.Ping, + HealthCheckProbePolicy.AllSuccess); + + private static readonly HealthCheck DisabledInstance = new( + enabled: false, + DefaultProbeCount, + DefaultProbeTimeout, + DefaultProbeInterval, + HealthCheckProbe.None, + HealthCheckProbePolicy.AllSuccess); /// - /// The default health check options. These options are immutable and cannot be modified; to customize, either - /// use to create a mutable copy, or create a new instance - and customize as needed. + /// The default health check: three probes, all of which must succeed. /// - public static HealthCheck Default => _default ?? CreateDefault(); - - private static HealthCheck CreateDefault() - { - var options = new HealthCheck(); - options.Freeze(); - // memoize, preferring to re-use the existing instance if we're competing (but since frozen: that's fine) - return Interlocked.CompareExchange(ref _default, options, null) ?? options; - } - - internal void Freeze() => _frozen = true; - private bool _frozen; + public static HealthCheck Default => DefaultInstance; /// - /// Create a mutable copy of this health check. + /// No health check is performed; every check reports , leaving + /// member selection driven purely by the observed connectivity of each member (and by any circuit-breaker). /// - public HealthCheck Clone() => new() + public static HealthCheck None => DisabledInstance; + + private HealthCheck( + bool enabled, + int probeCount, + TimeSpan probeTimeout, + TimeSpan probeInterval, + HealthCheckProbe probe, + HealthCheckProbePolicy probePolicy) { - // note: do not copy _frozen - Interval = Interval, - ProbeCount = ProbeCount, - ProbeTimeout = ProbeTimeout, - ProbeInterval = ProbeInterval, - Probe = Probe, - ProbePolicy = ProbePolicy, - }; + IsEnabled = enabled; + ProbeCount = probeCount; + ProbeTimeout = probeTimeout; + ProbeInterval = probeInterval; + Probe = probe; + ProbePolicy = probePolicy; + } - object ICloneable.Clone() => Clone(); + /// + public override string ToString() => IsEnabled + ? $"{Probe.GetType().Name} x{ProbeCount} ({ProbePolicy.GetType().Name})" + : "(disabled)"; /// - /// Create a new health check instance. + /// Whether this health check performs any probes; false only for . /// - public HealthCheck() - { - Interval = TimeSpan.FromSeconds(5); - ProbeCount = 3; - ProbeTimeout = TimeSpan.FromSeconds(3); - ProbeInterval = TimeSpan.FromMilliseconds(500); - Probe = HealthCheckProbe.Ping; - ProbePolicy = HealthCheckProbePolicy.AllSuccess; - } + public bool IsEnabled { get; } /// - /// Gets or sets the interval at which health checks should be performed. + /// Gets the number of probes to perform for this health check. /// - public TimeSpan Interval - { - get; - set => SetField(ref field, value); - } + public int ProbeCount { get; } /// - /// Gets or sets the number of probes to perform for this health check. + /// Gets the time that should be allowed for an individual probe to complete. /// - public int ProbeCount - { - get; - set => SetField(ref field, value); - } + public TimeSpan ProbeTimeout { get; } /// - /// Gets or sets the time that should be allowed for an individual probe to complete. + /// Gets the interval between failed probes. /// - public TimeSpan ProbeTimeout - { - get; - set => SetField(ref field, value); - } + public TimeSpan ProbeInterval { get; } /// - /// Gets or sets the interval between failed probes. + /// Gets the probe to use for this health check. /// - public TimeSpan ProbeInterval - { - get; - set => SetField(ref field, value); - } + public HealthCheckProbe Probe { get; } /// - /// Gets or sets the probe to use for this health check. + /// Gets the policy to use for this health check. /// - public HealthCheckProbe Probe - { - get; - set => SetField(ref field, value); - } + public HealthCheckProbePolicy ProbePolicy { get; } /// - /// Gets or sets the policy to use for this health check. + /// Allows configuration of a . /// - public HealthCheckProbePolicy ProbePolicy + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder { - get; - set => SetField(ref field, value); - } - - // ReSharper disable once RedundantAssignment - private void SetField(ref T field, T value, [CallerMemberName] string caller = "") - { - if (_frozen) Throw(caller); - field = value; - - static void Throw(string caller) => throw new InvalidOperationException($"{nameof(HealthCheck)}.{caller} cannot be modified once the object is in use."); + /// + /// Create a builder pre-populated with the default values. + /// + public Builder() + { + } + + /// + /// Create a builder pre-populated from an existing . + /// + public Builder(HealthCheck healthCheck) + { + ProbeCount = healthCheck.ProbeCount; + ProbeTimeout = healthCheck.ProbeTimeout; + ProbeInterval = healthCheck.ProbeInterval; + Probe = healthCheck.Probe; + ProbePolicy = healthCheck.ProbePolicy; + } + + /// + /// The number of probes to perform for this health check. + /// + public int ProbeCount { get; set; } = DefaultProbeCount; + + /// + /// The time that should be allowed for an individual probe to complete. + /// + public TimeSpan ProbeTimeout { get; set; } = DefaultProbeTimeout; + + /// + /// The interval between failed probes. + /// + public TimeSpan ProbeInterval { get; set; } = DefaultProbeInterval; + + /// + /// The probe to use for this health check. + /// + public HealthCheckProbe Probe { get; set; } = HealthCheckProbe.Ping; + + /// + /// The policy to use for this health check. + /// + public HealthCheckProbePolicy ProbePolicy { get; set; } = HealthCheckProbePolicy.AllSuccess; + + /// + /// Create a new health check instance. + /// + public HealthCheck Create() + { + if (ProbeCount < 1) throw new ArgumentOutOfRangeException(nameof(ProbeCount), ProbeCount, "At least one probe is required; use HealthCheck.None to disable health checks."); + if (ProbeTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(ProbeTimeout), ProbeTimeout, "A positive probe timeout is required."); + if (ProbeInterval < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(ProbeInterval), ProbeInterval, "A non-negative probe interval is required."); + if (Probe is null) throw new ArgumentNullException(nameof(Probe)); + if (ProbePolicy is null) throw new ArgumentNullException(nameof(ProbePolicy)); + + // the total budget is expressed in int milliseconds when the check runs; validate that here, + // rather than letting it overflow into a nonsensical (or negative) timeout later + if (!TryComputeTotalTimeoutMillis(ProbeCount, ProbeTimeout, ProbeInterval, out _)) + { + throw new ArgumentOutOfRangeException(nameof(ProbeTimeout), "The combined probe budget (ProbeCount, ProbeTimeout, ProbeInterval) is too large."); + } + + // prefer the shared default instance when nothing has been customized + if (ProbeCount == DefaultProbeCount + && ProbeTimeout == DefaultProbeTimeout + && ProbeInterval == DefaultProbeInterval + && ReferenceEquals(Probe, HealthCheckProbe.Ping) + && ReferenceEquals(ProbePolicy, HealthCheckProbePolicy.AllSuccess)) + { + return DefaultInstance; + } + + return new HealthCheck(enabled: true, ProbeCount, ProbeTimeout, ProbeInterval, Probe, ProbePolicy); + } + + /// + /// Create a new health check instance. + /// + public static implicit operator HealthCheck(Builder builder) => builder.Create(); } } diff --git a/src/StackExchange.Redis/Availability/HealthCheckContext.cs b/src/StackExchange.Redis/Availability/HealthCheckContext.cs new file mode 100644 index 000000000..8f7a7a13b --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckContext.cs @@ -0,0 +1,30 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Describes the target of a single health-check probe, and the budget available to it. +/// +/// +/// This is passed by value rather than by in, because probe implementations are +/// typically async, and async methods cannot take by-ref parameters. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public readonly struct HealthCheckContext(IServer server, TimeSpan probeTimeout) +{ + /// + public override string ToString() => $"{Server?.EndPoint} (timeout: {ProbeTimeout})"; + + /// + /// Gets the server being probed. + /// + public IServer Server => server; + + /// + /// Gets the time allowed for this probe to complete; probes are not required to enforce this + /// themselves (the caller applies it), but may use it to bound any state they create. + /// + public TimeSpan ProbeTimeout => probeTimeout; +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.Connected.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.Connected.cs new file mode 100644 index 000000000..6d058f3a3 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.Connected.cs @@ -0,0 +1,20 @@ +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Report health using the property, without any additional tests. + /// + public static HealthCheckProbe IsConnected => ConnectedProbe.Instance; + + private sealed class ConnectedProbe : HealthCheckProbe + { + public static ConnectedProbe Instance { get; } = new(); + private ConnectedProbe() { } + + public override Task CheckHealthAsync(HealthCheckContext context) + => context.Server.IsConnected ? HealthyTask : UnhealthyTask; + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.None.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.None.cs new file mode 100644 index 000000000..c114193e9 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.None.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Performs no test at all, always reporting ; this is the + /// probe used by , and leaves member selection driven purely by the + /// observed connectivity of each member. + /// + public static HealthCheckProbe None => NoneProbe.Instance; + + private sealed class NoneProbe : HealthCheckProbe + { + public static NoneProbe Instance { get; } = new(); + private NoneProbe() { } + + public override Task CheckHealthAsync(HealthCheckContext context) => InconclusiveTask; + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheck.PingProbe.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs similarity index 50% rename from src/StackExchange.Redis/Availability/HealthCheck.PingProbe.cs rename to src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs index 594a1bbeb..64b1e882d 100644 --- a/src/StackExchange.Redis/Availability/HealthCheck.PingProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs @@ -2,24 +2,21 @@ namespace StackExchange.Redis.Availability; -public sealed partial class HealthCheck +public abstract partial class HealthCheckProbe { - public partial class HealthCheckProbe - { - /// - /// Verify that the server is responsive by sending a PING command. - /// - public static HealthCheckProbe Ping => PingProbe.Instance; - } + /// + /// Verify that the server is responsive by sending a PING command. + /// + public static HealthCheckProbe Ping => PingProbe.Instance; private sealed class PingProbe : HealthCheckProbe { public static PingProbe Instance { get; } = new(); private PingProbe() { } - public override async Task CheckHealthAsync(HealthCheck healthCheck, IServer server) + public override async Task CheckHealthAsync(HealthCheckContext context) { - await server.PingAsync(); + await context.Server.PingAsync(); return HealthCheckResult.Healthy; } } diff --git a/src/StackExchange.Redis/Availability/HealthCheck.StringSetProbe.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs similarity index 79% rename from src/StackExchange.Redis/Availability/HealthCheck.StringSetProbe.cs rename to src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs index 1a6bd2d2d..6e1524b7a 100644 --- a/src/StackExchange.Redis/Availability/HealthCheck.StringSetProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs @@ -4,15 +4,12 @@ namespace StackExchange.Redis.Availability; -public sealed partial class HealthCheck +public abstract partial class HealthCheckProbe { - public partial class HealthCheckProbe - { - /// - /// Verify that a string can be successfully set and retrieved. - /// - public static HealthCheckProbe StringSet => StringSetProbe.Instance; - } + /// + /// Verify that a string can be successfully set and retrieved. + /// + public static HealthCheckProbe StringSet => StringSetProbe.Instance; internal sealed class StringSetProbe : KeyWriteHealthCheckProbe { @@ -23,7 +20,7 @@ private StringSetProbe() { } private static Random SharedRandom { get; } = new(); #endif - public override async Task CheckHealthAsync(HealthCheck healthCheck, IDatabaseAsync database, RedisKey key) + public override async Task CheckHealthAsync(HealthCheckContext context, IDatabaseAsync database, RedisKey key) { // note we use the lock API here because that can selectively choose between appropriate strategies for // different server versions, including DELEX @@ -41,7 +38,7 @@ public override async Task CheckHealthAsync(HealthCheck healt await database.LockTakeAsync( key: key, value: payload, - expiry: healthCheck.ProbeTimeout, + expiry: context.ProbeTimeout, flags: CommandFlags.FireAndForget).ForAwait(); // release from the db if matches (otherwise, we have no clue what happened, so: leave alone) diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs new file mode 100644 index 000000000..56a6dc9a6 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs @@ -0,0 +1,59 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Describes an operation to perform as part of a health check. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public abstract partial class HealthCheckProbe +{ + /// + /// Check the health of the specified endpoint. + /// + public abstract Task CheckHealthAsync(HealthCheckContext context); + + private static Task? _inconclusive, _healthy, _unhealthy; + + /// + /// Reports a memoized probe that was skipped without being evaluated. + /// + protected internal static Task InconclusiveTask => _inconclusive ??= Task.FromResult(HealthCheckResult.Inconclusive); + + /// + /// Reports a memoized probe that was healthy. + /// + protected internal static Task HealthyTask => _healthy ??= Task.FromResult(HealthCheckResult.Healthy); + + /// + /// Reports a memoized probe that was unhealthy. + /// + protected internal static Task UnhealthyTask => _unhealthy ??= Task.FromResult(HealthCheckResult.Unhealthy); +} + +/// +/// Describes a key-based (write) operation to perform as part of a health check. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public abstract class KeyWriteHealthCheckProbe : HealthCheckProbe +{ + /// + public sealed override Task CheckHealthAsync(HealthCheckContext context) + { + var server = context.Server; + if (server.IsReplica) return InconclusiveTask; + + RedisKey key = server.InventKey("health-check/"); + if (key.IsNull) return InconclusiveTask; + Debug.Assert(server.Multiplexer.GetServer(key).EndPoint == server.EndPoint, "Key was not routed to the correct endpoint"); + return CheckHealthAsync(context, server.Multiplexer.GetDatabase(), key); + } + + /// + /// Check the health of the specified database using the provided key. + /// + public abstract Task CheckHealthAsync(HealthCheckContext context, IDatabaseAsync database, RedisKey key); +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbeContext.cs b/src/StackExchange.Redis/Availability/HealthCheckProbeContext.cs new file mode 100644 index 000000000..3feb5d8b9 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbeContext.cs @@ -0,0 +1,40 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Represents the context of a health check probe. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public readonly struct HealthCheckProbeContext(HealthCheckResult result, int success, int failure, int remaining, TimeSpan probeInterval) +{ + /// + public override string ToString() => $"Result: {Result}, Success: {Success}, Failure: {Failure}, Remaining: {Remaining}, ProbeInterval: {ProbeInterval}"; + + /// + /// Gets the most recent result. + /// + public HealthCheckResult Result => result; + + /// + /// Gets the number of successful health checks. + /// + public int Success => success; + + /// + /// Gets the number of failed health checks. + /// + public int Failure => failure; + + /// + /// Gets the number of remaining health checks. + /// + public int Remaining => remaining; + + /// + /// Gets the interval to wait before the next probe attempt. + /// + public TimeSpan ProbeInterval => probeInterval; +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs new file mode 100644 index 000000000..4eb695475 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs @@ -0,0 +1,102 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Attempt to evaluate the outcome of a series of health check operations. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public abstract class HealthCheckProbePolicy +{ + /// + /// Attempt to evaluate the policy given the current context. + /// + /// The state of the probes so far. + /// The result of the policy evaluation. + public abstract HealthCheckResult Evaluate(in HealthCheckProbeContext context); + + /// + /// Get the interval to wait before the next probe attempt. + /// + internal TimeSpan GetEffectiveProbeInterval(in HealthCheckProbeContext context) + { + // if we make this public / overrideable, we will need to think about the max delay timeout + + // only delay between failures + return context.Result is HealthCheckResult.Unhealthy ? context.ProbeInterval : TimeSpan.Zero; + } + + /// + /// Require all probes to succeed. + /// + public static HealthCheckProbePolicy AllSuccess => AllSuccessHealthCheckProbePolicy.Instance; + + /// + /// Require at least one probe to succeed. + /// + public static HealthCheckProbePolicy AnySuccess => AnySuccessHealthCheckProbePolicy.Instance; + + /// + /// Require a majority of probes to succeed. + /// + public static HealthCheckProbePolicy MajoritySuccess => MajoritySuccessHealthCheckProbePolicy.Instance; + + private sealed class AllSuccessHealthCheckProbePolicy : HealthCheckProbePolicy + { + public static readonly AllSuccessHealthCheckProbePolicy Instance = new(); + private AllSuccessHealthCheckProbePolicy() { } + + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + // Fail as soon as we have any failure + if (context.Failure > 0) return HealthCheckResult.Unhealthy; + + // Succeed only when all probes have succeeded (no remaining) + if (context.Remaining == 0) return HealthCheckResult.Healthy; + + // Can't determine yet + return HealthCheckResult.Inconclusive; + } + } + + private sealed class AnySuccessHealthCheckProbePolicy : HealthCheckProbePolicy + { + public static readonly AnySuccessHealthCheckProbePolicy Instance = new(); + private AnySuccessHealthCheckProbePolicy() { } + + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + // Succeed as soon as we have any success + if (context.Success > 0) return HealthCheckResult.Healthy; + + // Fail only when all probes have failed (no remaining) + if (context.Remaining == 0) return HealthCheckResult.Unhealthy; + + // Can't determine yet + return HealthCheckResult.Inconclusive; + } + } + + private sealed class MajoritySuccessHealthCheckProbePolicy : HealthCheckProbePolicy + { + public static readonly MajoritySuccessHealthCheckProbePolicy Instance = new(); + private MajoritySuccessHealthCheckProbePolicy() { } + + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + int total = context.Success + context.Failure + context.Remaining; + int majority = (total / 2) + 1; + + // Succeed as soon as we have enough successes for a majority + if (context.Success >= majority) return HealthCheckResult.Healthy; + + // Fail as soon as we have enough failures to make a majority impossible + if (context.Failure >= majority) return HealthCheckResult.Unhealthy; + + // Can't determine yet + return HealthCheckResult.Inconclusive; + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckResult.cs b/src/StackExchange.Redis/Availability/HealthCheckResult.cs new file mode 100644 index 000000000..c8632507c --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckResult.cs @@ -0,0 +1,26 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Indicates the result of a health check. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public enum HealthCheckResult +{ + /// + /// The health check was skipped or could not be determined. + /// + Inconclusive, + + /// + /// The health check was successful. + /// + Healthy, + + /// + /// The health check failed. + /// + Unhealthy, +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs index 4ff4eb872..9505d5521 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -33,9 +33,7 @@ public static Task ConnectGroupAsync( { // create a defensive copy of the array; we don't want callers being able to radically swap things! members = (ConnectionGroupMember[])members.Clone(); - options ??= MultiGroupOptions.Default; - options.Freeze(); - return MultiGroupMultiplexer.ConnectAsync(members, options, log); + return MultiGroupMultiplexer.ConnectAsync(members, options ?? MultiGroupOptions.Default, log); } /// @@ -55,9 +53,7 @@ public static Task ConnectGroupAsync( TextWriter? log = null) #pragma warning restore RS0026 { - options ??= MultiGroupOptions.Default; - options.Freeze(); - return MultiGroupMultiplexer.ConnectAsync([member0, member1], options, log); + return MultiGroupMultiplexer.ConnectAsync([member0, member1], options ?? MultiGroupOptions.Default, log); } } @@ -206,6 +202,42 @@ public bool SkipInitialHealthCheck /// public string Name { get; private set; } = name; + // ---- per-member overrides of the group-wide MultiGroupOptions defaults ---- + // every value on MultiGroupOptions that can vary per member appears here as a nullable + // counterpart; null means "use the group default". These are read when the member is added + // to a group (for the circuit-breaker, which is fixed at connection construction) or on each + // health-check pass (for the rest), so changing them later is only meaningful for the latter. + + /// + /// The health-check to use for this member; when , + /// is used. Use + /// to leave this member's selection driven purely by its observed connectivity. + /// + public HealthCheck? HealthCheck { get; set; } + + /// + /// The circuit-breaker to use for this member; when , the member's own + /// is used, else + /// . This is applied when the member connects, so + /// setting it after the member has been added to a group has no effect on existing connections. + /// + public CircuitBreaker? CircuitBreaker { get; set; } + + /// + /// How long this member must remain healthy, following its most recent failure, before it is + /// eligible to be selected again; when , + /// is used. + /// + public TimeSpan? FailbackDelay { get; set; } + + // the breaker to hand to this member's connections: member override, else its own config, else the group + internal CircuitBreaker? ResolveCircuitBreaker(MultiGroupOptions options) + => CircuitBreaker ?? Configuration.CircuitBreaker ?? options.CircuitBreaker; + + internal HealthCheck ResolveHealthCheck(MultiGroupOptions options) => HealthCheck ?? options.HealthCheck; + + internal TimeSpan ResolveFailbackDelay(MultiGroupOptions options) => FailbackDelay ?? options.FailbackDelay; + /// /// The relative weight of this group member; higher is preferred. /// @@ -282,12 +314,12 @@ internal static uint ToLatencyTicks(TimeSpan latency) return x; } - internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheck.HealthCheckResult result, long failbackFailureCutoffTicks) + internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheckResult result, long failbackFailureCutoffTicks) { bool isConnected; if (_muxer is { IsConnected: true } muxer) { - isConnected = result is not HealthCheck.HealthCheckResult.Unhealthy; + isConnected = result is not HealthCheckResult.Unhealthy; SetLatency(muxer.UpdateLatency()); } else @@ -444,8 +476,10 @@ internal static async Task ConnectAsync( var config = members[i].Configuration; config.AbortOnConnectFail = false; config.HeartbeatConsistencyChecks = true; - config.CircuitBreaker ??= options.CircuitBreaker; // AA options flow into the children - pending[i] = ConnectionMultiplexer.ConnectAsync(config, log); + + // note the resolved circuit-breaker is passed *alongside* the configuration rather than + // written into it; see ConnectionMultiplexer.GroupCircuitBreaker + pending[i] = ConnectionMultiplexer.ConnectGroupMemberAsync(config, log, members[i].ResolveCircuitBreaker(options)); } for (int i = 0; i < pending.Length; i++) @@ -463,6 +497,8 @@ internal static async Task ConnectAsync( private readonly MultiGroupOptions _options; + public MultiGroupOptions Options => _options; + private MultiGroupMultiplexer(ConnectionGroupMember[] members, MultiGroupOptions options) { _options = options; @@ -538,10 +574,11 @@ static async Task PollAsync(WeakReference weakRef, CancellationToken cancellatio static bool TryGetHealthCheck(object? target, out TimeSpan interval) { - if (target is MultiGroupMultiplexer typed - && typed._options.HealthCheck is { } healthCheck) + if (target is MultiGroupMultiplexer typed) { - interval = healthCheck.Interval; + // note the interval is a group-level concern (how often we re-evaluate the active + // member), not a property of any individual health-check + interval = typed._options.HealthCheckInterval; return interval > TimeSpan.Zero & interval != TimeSpan.MaxValue; } @@ -552,28 +589,32 @@ static bool TryGetHealthCheck(object? target, out TimeSpan interval) internal bool IsDisposed => _disposed; - private Task[]? _reusableHealthCheckBuffer; + private Task[]? _reusableHealthCheckBuffer; private int _healthCheckGate; // 0 = idle, 1 = a check/select pass is in flight (see TryHealthCheckAndSelectPreferredGroupAsync) internal async Task RunHealthCheckAsync() { if (_disposed) return; - var healthCheck = _options.HealthCheck; var members = _members; + if (members.Length == 0) return; // nothing to check (and no budget to compute) + var pending = HealthCheck.GetReusablePending(ref _reusableHealthCheckBuffer, members.Length); + + // members can use different health-checks (see ConnectionGroupMember.HealthCheck), so the + // budget for the whole pass is the largest individual budget + int totalTimeoutMillis = 0; for (int i = 0; i < members.Length; i++) { - // per-member health-check overrides the group default when specified (see the group/muxer - // split on circuit-breakers); left null, we fall back to the shared group health-check var muxer = members[i].Multiplexer; - pending[i] = (muxer.RawConfig.HealthCheck ?? healthCheck).CheckHealthAsync(muxer); + var healthCheck = members[i].ResolveHealthCheck(_options); + totalTimeoutMillis = Math.Max(totalTimeoutMillis, healthCheck.TotalTimeoutMillis()); + pending[i] = healthCheck.CheckHealthAsync(muxer); } - await Task.WhenAll(pending).TimeoutAfter(healthCheck.TotalTimeoutMillis()).ForAwait(); - var failbackFailureCutoff = GetFailbackFailureCutoff(); + await Task.WhenAll(pending).TimeoutAfter(totalTimeoutMillis).ForAwait(); for (int i = 0; i < pending.Length; i++) { - HealthCheck.HealthCheckResult result; + HealthCheckResult result; if (pending[i].IsCompletedSuccessfully) { result = await pending[i].ForAwait(); @@ -581,10 +622,10 @@ internal async Task RunHealthCheckAsync() else { _ = pending[i].ObserveErrors(); - result = HealthCheck.HealthCheckResult.Unhealthy; + result = HealthCheckResult.Unhealthy; } - var delta = members[i].UpdateState(result, failbackFailureCutoff); + var delta = members[i].UpdateState(result, GetFailbackFailureCutoff(members[i])); if (delta != GroupConnectionChangedEventArgs.ChangeType.Unknown) { OnConnectionChanged(delta, members[i]); @@ -594,13 +635,13 @@ internal async Task RunHealthCheckAsync() HealthCheck.PutReusablePending(ref _reusableHealthCheckBuffer, ref pending); } - private long GetFailbackFailureCutoff() + private long GetFailbackFailureCutoff(ConnectionGroupMember member) { // the minimum last-observed unhealthy time (as UTC ticks) that we'll allow for reconnect; // for example, if the FailbackDelay is 2 minutes, and the time is 14:32:55, then the last // failure must have happened at 14:30:55 or earlier. Pure long tick math on the wall clock: // DateTime.Ticks and TimeSpan.Ticks are the same 100ns unit, so the subtraction is valid - var delay = _options.FailbackDelay; + var delay = member.ResolveFailbackDelay(_options); if (delay == TimeSpan.MaxValue) return long.MinValue; // manual mode: never auto-reset return DateTime.UtcNow.Ticks - delay.Ticks; @@ -1241,8 +1282,8 @@ public async Task AddAsync(ConnectionGroupMember member, TextWriter? log = null) member.Init(_members.Length); member.Configuration.AbortOnConnectFail = false; // members are gated by health-checks, not connect-fail member.Configuration.HeartbeatConsistencyChecks = true; - member.Configuration.CircuitBreaker ??= _options.CircuitBreaker; // AA options flow into the children - var muxer = await ConnectionMultiplexer.ConnectAsync(member.Configuration, log).ConfigureAwait(false); + var muxer = await ConnectionMultiplexer.ConnectGroupMemberAsync( + member.Configuration, log, member.ResolveCircuitBreaker(_options)).ConfigureAwait(false); member.SetMultiplexer(muxer); // unless told otherwise, run an initial health-check so a healthy member can be selected immediately; @@ -1250,9 +1291,8 @@ public async Task AddAsync(ConnectionGroupMember member, TextWriter? log = null) // passes - this is the only way to add a member that is not yet healthy if (!member.SkipInitialHealthCheck) { - var health = await (muxer.RawConfig.HealthCheck ?? _options.HealthCheck) - .CheckHealthAsync(muxer).ConfigureAwait(false); - member.UpdateState(health, GetFailbackFailureCutoff()); + var health = await member.ResolveHealthCheck(_options).CheckHealthAsync(muxer).ConfigureAwait(false); + member.UpdateState(health, GetFailbackFailureCutoff(member)); } // apply any shared hooks diff --git a/src/StackExchange.Redis/Availability/MultiGroupOptions.cs b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs index 779a1b6fd..f6260cb4f 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupOptions.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs @@ -1,7 +1,5 @@ -using System; +using System; using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; -using System.Threading; using RESPite; namespace StackExchange.Redis.Availability; @@ -9,41 +7,64 @@ namespace StackExchange.Redis.Availability; /// /// Configuration options for controlling connections to multiple groups. /// +/// +/// Instances are immutable; use to configure. Every value here is a group-wide +/// default, and can be overridden per-member by the matching property on +/// (where one exists); the effective value is "member override, else group default". +/// [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] -public sealed class MultiGroupOptions() +public sealed class MultiGroupOptions { - private static MultiGroupOptions? _default; - private bool _frozen; + internal static readonly TimeSpan DefaultHealthCheckInterval = TimeSpan.FromSeconds(5); + internal static readonly TimeSpan DefaultFailbackDelay = TimeSpan.Zero; + + private static readonly MultiGroupOptions DefaultInstance = new( + HealthCheck.Default, CircuitBreaker.Default, RetryPolicy.Default, DefaultHealthCheckInterval, DefaultFailbackDelay); /// /// Default shared options. /// - public static MultiGroupOptions Default => _default ??= CreateDefault(); + public static MultiGroupOptions Default => DefaultInstance; - private static MultiGroupOptions CreateDefault() + private MultiGroupOptions( + HealthCheck healthCheck, + CircuitBreaker circuitBreaker, + RetryPolicy retryPolicy, + TimeSpan healthCheckInterval, + TimeSpan failbackDelay) { - var options = new MultiGroupOptions(); - options.Freeze(); - return Interlocked.CompareExchange(ref _default, options, null) ?? options; + HealthCheck = healthCheck; + CircuitBreaker = circuitBreaker; + RetryPolicy = retryPolicy; + HealthCheckInterval = healthCheckInterval; + FailbackDelay = failbackDelay; } + /// + public override string ToString() => $"health-check: {HealthCheck} every {HealthCheckInterval}; failback: {FailbackDelay}"; + /// /// The health check to use for members of the group when no per-member health check is specified. /// - public HealthCheck HealthCheck - { - get => field ?? HealthCheck.Default; - set => SetField(ref field, value); - } + public HealthCheck HealthCheck { get; } /// /// The circuit-breaker to use for members of the group when no per-member circuit-breaker is specified. /// - public CircuitBreaker CircuitBreaker - { - get => field ?? CircuitBreaker.Default; - set => SetField(ref field, value); - } + public CircuitBreaker CircuitBreaker { get; } + + /// + /// The retry policy used by for databases + /// obtained from this group. + /// + public RetryPolicy RetryPolicy { get; } + + /// + /// How frequently health checks are performed, and therefore how frequently the active member is + /// re-evaluated. disables periodic checking entirely (the group is then + /// only re-evaluated in response to connection events such as a tripped circuit-breaker). + /// + public TimeSpan HealthCheckInterval { get; } /// /// If a member has been marked unhealthy by a failing health-check or circuit-breaker, it will not be @@ -52,20 +73,88 @@ public CircuitBreaker CircuitBreaker /// , failback is not automatic and requires explicit use of /// or . /// - public TimeSpan FailbackDelay - { - get => field; - set => SetField(ref field, value); - } + public TimeSpan FailbackDelay { get; } - // ReSharper disable once RedundantAssignment - private void SetField(ref T field, T value, [CallerMemberName] string caller = "") + /// + /// Allows configuration of . + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder { - if (_frozen) Throw(caller); - field = value; + /// + /// Create a builder pre-populated with the default values. + /// + public Builder() + { + } - static void Throw(string caller) => throw new InvalidOperationException($"{nameof(MultiGroupOptions)}.{caller} cannot be modified once the object is in use."); - } + /// + /// Create a builder pre-populated from existing options. + /// + public Builder(MultiGroupOptions options) + { + HealthCheck = options.HealthCheck; + CircuitBreaker = options.CircuitBreaker; + RetryPolicy = options.RetryPolicy; + HealthCheckInterval = options.HealthCheckInterval; + FailbackDelay = options.FailbackDelay; + } + + /// + /// The health check to use for members of the group when no per-member health check is specified. + /// + public HealthCheck HealthCheck { get; set; } = HealthCheck.Default; + + /// + /// The circuit-breaker to use for members of the group when no per-member circuit-breaker is specified. + /// + public CircuitBreaker CircuitBreaker { get; set; } = CircuitBreaker.Default; + + /// + /// The retry policy used by for databases + /// obtained from this group. + /// + public RetryPolicy RetryPolicy { get; set; } = RetryPolicy.Default; - internal void Freeze() => _frozen = true; + /// + /// How frequently health checks are performed, and therefore how frequently the active member is + /// re-evaluated; disables periodic checking. + /// + public TimeSpan HealthCheckInterval { get; set; } = DefaultHealthCheckInterval; + + /// + /// How long a member must remain healthy, following its most recent failure, before it is eligible to + /// be selected again; requires explicit intervention. + /// + public TimeSpan FailbackDelay { get; set; } = DefaultFailbackDelay; + + /// + /// Create a new options instance. + /// + public MultiGroupOptions Create() + { + if (HealthCheck is null) throw new ArgumentNullException(nameof(HealthCheck)); + if (CircuitBreaker is null) throw new ArgumentNullException(nameof(CircuitBreaker)); + if (RetryPolicy is null) throw new ArgumentNullException(nameof(RetryPolicy)); + if (HealthCheckInterval <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(HealthCheckInterval), HealthCheckInterval, "A positive interval is required; use TimeSpan.MaxValue to disable periodic health checks."); + if (FailbackDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(FailbackDelay), FailbackDelay, "A non-negative delay is required."); + + // prefer the shared default instance when nothing has been customized + if (ReferenceEquals(HealthCheck, HealthCheck.Default) + && ReferenceEquals(CircuitBreaker, CircuitBreaker.Default) + && ReferenceEquals(RetryPolicy, RetryPolicy.Default) + && HealthCheckInterval == DefaultHealthCheckInterval + && FailbackDelay == DefaultFailbackDelay) + { + return DefaultInstance; + } + + return new MultiGroupOptions(HealthCheck, CircuitBreaker, RetryPolicy, HealthCheckInterval, FailbackDelay); + } + + /// + /// Create a new options instance. + /// + public static implicit operator MultiGroupOptions(Builder builder) => builder.Create(); + } } diff --git a/src/StackExchange.Redis/Availability/RetryController.cs b/src/StackExchange.Redis/Availability/RetryController.cs index a4b9fea6a..72bbf8c33 100644 --- a/src/StackExchange.Redis/Availability/RetryController.cs +++ b/src/StackExchange.Redis/Availability/RetryController.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using StackExchange.Redis.Interfaces; @@ -19,22 +20,27 @@ public RetryController(RetryPolicy policy, DatabaseFeatureFlags features) { _policy = policy; - // capture config locally rather than constant cross-object lookups (plus: mutability) + // capture config locally rather than constant cross-object lookups; a RetryPolicy is immutable and + // is validated by RetryPolicy.Builder, so no range checks are needed here _maxBeforeFailover = (features & DatabaseFeatureFlags.Failover) == 0 ? int.MaxValue : policy.MaxAttemptsBeforeFailover; _maxAttempts = policy.MaxAttempts; if (_maxBeforeFailover == _maxAttempts) _maxBeforeFailover = int.MaxValue; // then we'll never look - // guard the failover threshold: values < 1 can never be hit by the loop counter (which starts at 1), - // so they would *silently* disable failover rather than erroring; validate the raw policy value - if (policy.MaxAttemptsBeforeFailover < 1) throw new ArgumentOutOfRangeException(nameof(policy.MaxAttemptsBeforeFailover)); - _delayMillis = policy.DelayMilliseconds; - _failoverMillis = policy.FailoverMilliseconds; - _jitterMillis = policy.JitterMilliseconds; - if (_delayMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.RetryDelay)); - if (_jitterMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.JitterMax)); - if (_failoverMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.FailoverDelay)); + _delayMillis = ToMilliseconds(policy.RetryDelay); + _failoverMillis = ToMilliseconds(policy.FailoverDelay); + _jitterMillis = ToMilliseconds(policy.JitterMax); + + Debug.Assert(_maxAttempts >= 1 && _maxBeforeFailover >= 1, "attempt counts should be validated by RetryPolicy"); + Debug.Assert(_delayMillis >= 0 && _jitterMillis >= 0 && _failoverMillis >= 0, "delays should be validated by RetryPolicy"); + + static int ToMilliseconds(TimeSpan value) => (int)(value.Ticks / TimeSpan.TicksPerMillisecond); } + /// + /// The policy this controller is applying; exposed for tests, which assert how a policy was resolved. + /// + public RetryPolicy Policy => _policy; + /// /// Whether it is ever worth capturing the next-failover token: only when there is more than one /// attempt and the failover threshold sits below the attempt cap. @@ -57,14 +63,14 @@ public bool CanRetry( // ask the retry policy for advice, and mask off the bits we know about FaultContext ctx = new(fault); var policy = _policy.CanRetry(ctx) & - (RetryPolicy.RetryPolicyResult.FailoverServer | RetryPolicy.RetryPolicyResult.SameServer); + (RetryResult.FailoverServer | RetryResult.SameServer); if (policy is 0) { // retry policy says: nope return false; } - if (policy is RetryPolicy.RetryPolicyResult.FailoverServer) + if (policy is RetryResult.FailoverServer) { // we can *only* retry on a different server; is failover available? delay = failover; @@ -77,11 +83,11 @@ public bool CanRetry( // by count, we should really switch over to the failover now; is failover available *and* are we allowed? delay = failover; failover = CancellationToken.None; // only failover once - return delay.CanBeCanceled & (policy & RetryPolicy.RetryPolicyResult.FailoverServer) != 0; + return delay.CanBeCanceled & (policy & RetryResult.FailoverServer) != 0; } // can we pause and retry on the same server? - return (policy & RetryPolicy.RetryPolicyResult.SameServer) != 0; + return (policy & RetryResult.SameServer) != 0; } public Task FailoverOrDelayAsync(CancellationToken delay) diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index dbb6064dd..958299db2 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -22,6 +22,8 @@ DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) private readonly IDatabaseAsync _inner; private readonly RetryController _controller; + internal RetryPolicy Policy => _controller.Policy; + public CancellationToken GetNextFailover() => _controller.TracksFailover ? _inner.GetNextFailover() : CancellationToken.None; diff --git a/src/StackExchange.Redis/Availability/RetryPolicy.cs b/src/StackExchange.Redis/Availability/RetryPolicy.cs index 447424b2f..ffc2a3142 100644 --- a/src/StackExchange.Redis/Availability/RetryPolicy.cs +++ b/src/StackExchange.Redis/Availability/RetryPolicy.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using RESPite; @@ -9,78 +8,116 @@ namespace StackExchange.Redis.Availability; /// Configures how messages can be retried due to connection / transient faults. Other faults (such as invalid /// usage) are not retried. /// +/// +/// Instances are immutable and safe to share; use to configure the standard policy, or +/// derive from this type and override to make the decision yourself. +/// [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] public class RetryPolicy { + internal const int DefaultMaxAttempts = 3; + internal const int DefaultMaxAttemptsBeforeFailover = 1; + internal const CommandFlags DefaultMaxCommandRetryCategory = CommandFlags.CommandRetryWriteLastWins; + internal static readonly TimeSpan DefaultRetryDelay = TimeSpan.FromSeconds(1); + internal static readonly TimeSpan DefaultJitterMax = TimeSpan.FromMilliseconds(500); + internal static readonly TimeSpan DefaultFailoverDelay = TimeSpan.FromSeconds(5); + + private static readonly RetryPolicy DefaultInstance = new(); + + /// + /// The default retry policy; retries transient faults up to times, for commands + /// at or below . + /// + public static RetryPolicy Default => DefaultInstance; + + /// + /// Never retries anything; useful to disable retries without restructuring calling code. + /// + public static RetryPolicy None => NoRetryPolicy.Instance; + + /// + /// Create a policy using the default settings; intended for use by derived types that override + /// - use to obtain the standard policy. + /// + protected RetryPolicy() + : this(DefaultMaxAttempts, DefaultMaxAttemptsBeforeFailover, DefaultRetryDelay, DefaultJitterMax, DefaultFailoverDelay, DefaultMaxCommandRetryCategory) + { + } + + /// + /// Create a policy using the settings from the supplied ; intended for use by + /// derived types that override . + /// + protected RetryPolicy(Builder builder) + : this( + Validate(builder).MaxAttempts, + builder.MaxAttemptsBeforeFailover, + builder.RetryDelay, + builder.JitterMax, + builder.FailoverDelay, + builder.MaxCommandRetryCategory) + { + } + + private RetryPolicy( + int maxAttempts, + int maxAttemptsBeforeFailover, + TimeSpan retryDelay, + TimeSpan jitterMax, + TimeSpan failoverDelay, + CommandFlags maxCommandRetryCategory) + { + MaxAttempts = maxAttempts; + MaxAttemptsBeforeFailover = maxAttemptsBeforeFailover; + RetryDelay = retryDelay; + JitterMax = jitterMax; + FailoverDelay = failoverDelay; + MaxCommandRetryCategory = maxCommandRetryCategory; + } + + /// + public override string ToString() => $"{GetType().Name}: {MaxAttempts} attempt(s), up to {MaxCommandRetryCategory}"; + /// /// The maximum number of times an operation can be attempted. Defaults to 3. /// - public int MaxAttempts { get; set; } = 3; + public int MaxAttempts { get; } /// /// The maximum number of times to retry an operation before waiting for failover; this only currently /// applies to multi-group connections created via ConnectionMultiplexer.ConnectGroupAsync. /// Defaults to 1. /// - public int MaxAttemptsBeforeFailover { get; set; } = 1; - - private int _delayMillis = 1000, _jitterMillis = 500, _failoverMillis = 5000; + public int MaxAttemptsBeforeFailover { get; } /// /// Gets the time to wait between retries that are *not* dependent on a failover happening. Defaults to 1 second. /// - public TimeSpan RetryDelay - { - get => TimeSpan.FromMilliseconds(_delayMillis); - set => _delayMillis = checked((int)value.TotalMilliseconds); - } + public TimeSpan RetryDelay { get; } /// /// Gets the time to wait for a failover, after attempts. Only one /// failover attempt is awaited. When this applies, is ignored, /// but is still respected. Defaults to 5 seconds. /// - public TimeSpan FailoverDelay - { - get => TimeSpan.FromMilliseconds(_failoverMillis); - set => _failoverMillis = checked((int)value.TotalMilliseconds); - } + public TimeSpan FailoverDelay { get; } /// - /// Gets or sets the upper bound for jitter - additional random delay between retries to prevent stampedes. + /// Gets the upper bound for jitter - additional random delay between retries to prevent stampedes. /// Defaults to 0.5 seconds, meaning between 0 and 0.5 *additional* seconds on top of . /// - public TimeSpan JitterMax - { - get => TimeSpan.FromMilliseconds(_jitterMillis); - set => _jitterMillis = checked((int)value.TotalMilliseconds); - } - - internal int DelayMilliseconds => _delayMillis; - internal int JitterMilliseconds => _jitterMillis; - internal int FailoverMilliseconds => _failoverMillis; + public TimeSpan JitterMax { get; } /// - /// Gets or sets the max side-effect category that will be retried; defaults to . + /// Gets the max side-effect category that will be retried; defaults to . /// - public CommandFlags MaxCommandRetryCategory - { - get => _maxCommandRetryCategory; - set - { - if ((value & Message.MaskRetryCategory) is 0 | (value & ~Message.MaskRetryCategory) is not 0) - throw new InvalidOperationException("Valid CommandRetry* flags should be specified"); - _maxCommandRetryCategory = value; - } - } - - private CommandFlags _maxCommandRetryCategory = CommandFlags.CommandRetryWriteLastWins; + public CommandFlags MaxCommandRetryCategory { get; } /// /// Controls which operations can be repeated, optionally indicating that this should progress to /// a new server. /// - public virtual RetryPolicyResult CanRetry(in FaultContext fault) + public virtual RetryResult CanRetry(in FaultContext fault) { var actual = fault.Flags & Message.MaskRetryCategory; if (actual is 0) actual = CommandFlags.CommandRetryWriteAccumulating; // if not set, assume similar to INCR @@ -88,47 +125,144 @@ public virtual RetryPolicyResult CanRetry(in FaultContext fault) if (actual is CommandFlags.CommandRetryNever) { // explicitly disabled at command level - return RetryPolicyResult.None; + return RetryResult.None; } if (actual > MaxCommandRetryCategory) // note this also covers CommandRetryAlways { // side-effects are beyond what the policy allows - return RetryPolicyResult.None; + return RetryResult.None; } if (CircuitBreaker.DefaultIsFailure(in fault)) { // assume we can send it everywhere - var result = RetryPolicyResult.SameServer | RetryPolicyResult.FailoverServer; + var result = RetryResult.SameServer | RetryResult.FailoverServer; if ((fault.Flags & Message.CommandServerSpecific) != 0) - result &= ~RetryPolicyResult.FailoverServer; + result &= ~RetryResult.FailoverServer; return result; } // do not retry - return RetryPolicyResult.None; + return RetryResult.None; + } + + private static Builder Validate(Builder builder) + { + if (builder is null) throw new ArgumentNullException(nameof(builder)); + if (builder.MaxAttempts < 1) throw new ArgumentOutOfRangeException(nameof(builder.MaxAttempts), builder.MaxAttempts, "At least one attempt is required; use RetryPolicy.None to disable retries."); + + // values < 1 can never be hit by the attempt counter (which starts at 1), so they would *silently* + // disable failover rather than erroring + if (builder.MaxAttemptsBeforeFailover < 1) throw new ArgumentOutOfRangeException(nameof(builder.MaxAttemptsBeforeFailover), builder.MaxAttemptsBeforeFailover, "At least one attempt is required before failover."); + if (builder.RetryDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(builder.RetryDelay), builder.RetryDelay, "A non-negative retry delay is required."); + if (builder.JitterMax < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(builder.JitterMax), builder.JitterMax, "A non-negative jitter bound is required."); + if (builder.FailoverDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(builder.FailoverDelay), builder.FailoverDelay, "A non-negative failover delay is required."); + + // the retry loop expresses all three delays in int milliseconds + if (!IsExpressibleAsMilliseconds(builder.RetryDelay)) throw new ArgumentOutOfRangeException(nameof(builder.RetryDelay), builder.RetryDelay, "The retry delay is too large."); + if (!IsExpressibleAsMilliseconds(builder.JitterMax)) throw new ArgumentOutOfRangeException(nameof(builder.JitterMax), builder.JitterMax, "The jitter bound is too large."); + if (!IsExpressibleAsMilliseconds(builder.FailoverDelay)) throw new ArgumentOutOfRangeException(nameof(builder.FailoverDelay), builder.FailoverDelay, "The failover delay is too large."); + + var category = builder.MaxCommandRetryCategory; + if ((category & Message.MaskRetryCategory) is 0 | (category & ~Message.MaskRetryCategory) is not 0) + { + throw new ArgumentException("A single valid CommandRetry* flag should be specified.", nameof(builder.MaxCommandRetryCategory)); + } + + return builder; + + static bool IsExpressibleAsMilliseconds(TimeSpan value) => value.Ticks / TimeSpan.TicksPerMillisecond <= int.MaxValue; } /// - /// Indicates the result of a query. + /// Allows configuration of the standard implementation. /// - [Flags] - public enum RetryPolicyResult + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder { /// - /// None; the operation should not be retried. + /// Create a builder pre-populated with the default values. + /// + public Builder() + { + } + + /// + /// Create a builder pre-populated from an existing . + /// + public Builder(RetryPolicy policy) + { + MaxAttempts = policy.MaxAttempts; + MaxAttemptsBeforeFailover = policy.MaxAttemptsBeforeFailover; + RetryDelay = policy.RetryDelay; + JitterMax = policy.JitterMax; + FailoverDelay = policy.FailoverDelay; + MaxCommandRetryCategory = policy.MaxCommandRetryCategory; + } + + /// + /// The maximum number of times an operation can be attempted. /// - None = 0, + public int MaxAttempts { get; set; } = DefaultMaxAttempts; /// - /// The operation can be retried on the same server. + /// The maximum number of times to retry an operation before waiting for failover. /// - SameServer = 1, + public int MaxAttemptsBeforeFailover { get; set; } = DefaultMaxAttemptsBeforeFailover; /// - /// The operation can be retried on a different server after a failover operation. + /// The time to wait between retries that are *not* dependent on a failover happening. /// - FailoverServer = 2, + public TimeSpan RetryDelay { get; set; } = DefaultRetryDelay; + + /// + /// The upper bound for jitter - additional random delay between retries to prevent stampedes. + /// + public TimeSpan JitterMax { get; set; } = DefaultJitterMax; + + /// + /// The time to wait for a failover, after attempts. + /// + public TimeSpan FailoverDelay { get; set; } = DefaultFailoverDelay; + + /// + /// The max side-effect category that will be retried. + /// + public CommandFlags MaxCommandRetryCategory { get; set; } = DefaultMaxCommandRetryCategory; + + /// + /// Create a new retry policy instance. + /// + public RetryPolicy Create() + { + Validate(this); + + // prefer the shared default instance when nothing has been customized + if (MaxAttempts == DefaultMaxAttempts + && MaxAttemptsBeforeFailover == DefaultMaxAttemptsBeforeFailover + && RetryDelay == DefaultRetryDelay + && JitterMax == DefaultJitterMax + && FailoverDelay == DefaultFailoverDelay + && MaxCommandRetryCategory == DefaultMaxCommandRetryCategory) + { + return DefaultInstance; + } + + return new RetryPolicy(MaxAttempts, MaxAttemptsBeforeFailover, RetryDelay, JitterMax, FailoverDelay, MaxCommandRetryCategory); + } + + /// + /// Create a new retry policy instance. + /// + public static implicit operator RetryPolicy(Builder builder) => builder.Create(); + } + + private sealed class NoRetryPolicy : RetryPolicy + { + public static readonly NoRetryPolicy Instance = new(); + private NoRetryPolicy() { } + + public override RetryResult CanRetry(in FaultContext fault) => RetryResult.None; } } diff --git a/src/StackExchange.Redis/Availability/RetryResult.cs b/src/StackExchange.Redis/Availability/RetryResult.cs new file mode 100644 index 000000000..404955212 --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryResult.cs @@ -0,0 +1,28 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Indicates the result of a query. +/// +[Flags] +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public enum RetryResult +{ + /// + /// None; the operation should not be retried. + /// + None = 0, + + /// + /// The operation can be retried on the same server. + /// + SameServer = 1, + + /// + /// The operation can be retried on a different server after a failover operation. + /// + FailoverServer = 2, +} diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index ee6cd6f32..90e90933e 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -980,7 +980,7 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow heartbeatInterval = heartbeatInterval, WriteMode = WriteMode, CircuitBreaker = CircuitBreaker, - HealthCheck = HealthCheck, + RetryPolicy = RetryPolicy, #if DEBUG OutputLog = OutputLog, #endif @@ -1185,7 +1185,7 @@ private void Clear() _protocol = default; WriteMode = default; CircuitBreaker = null; - HealthCheck = null; + RetryPolicy = null; #if DEBUG OutputLog = null; #endif @@ -1386,18 +1386,25 @@ public RedisProtocol? Protocol internal BufferedStreamWriter.WriteMode WriteMode { get; set; } /// - /// The circuit-breaker to apply to physical connections; when null, no breaker is used. When - /// connecting a group, this flows in from if not set explicitly. + /// The circuit-breaker to apply to physical connections; when null, no breaker is used. /// + /// + /// For a member of a connection group, the effective breaker is + /// , else this, else + /// . + /// [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] public CircuitBreaker? CircuitBreaker { get; set; } /// - /// The health-check to apply when this connection is a member of a group; when null, the - /// group-level is used. + /// The retry policy used by for databases + /// obtained from this connection; when null, is used. /// + /// + /// For a member of a connection group, applies instead. + /// [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public HealthCheck? HealthCheck { get; set; } + public RetryPolicy? RetryPolicy { get; set; } internal bool AllowSimulateConnectionFailure { diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 86fd7783d..44b2957e6 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -47,6 +47,21 @@ public sealed partial class ConnectionMultiplexer : IInternalConnectionMultiplex internal CommandMap CommandMap { get; } internal EndPointCollection EndPoints { get; } internal ConfigurationOptions RawConfig { get; } + + /// + /// When this multiplexer is a member of a connection group, the group resolves the effective + /// circuit-breaker (member override, else this member's own configuration, else the group default) + /// and supplies it here. This deliberately does *not* write back into : callers + /// may legitimately reuse a single across multiple connections, + /// and a group default must not leak into an unrelated one. + /// + internal Availability.CircuitBreaker? GroupCircuitBreaker { get; private set; } + + /// + /// The circuit-breaker that physical connections for this multiplexer should use, if any. + /// + internal Availability.CircuitBreaker? EffectiveCircuitBreaker => GroupCircuitBreaker ?? RawConfig.CircuitBreaker; + internal ServerSelectionStrategy ServerSelectionStrategy { get; } ServerSelectionStrategy IInternalConnectionMultiplexer.ServerSelectionStrategy => ServerSelectionStrategy; ConnectionMultiplexer IInternalConnectionMultiplexer.UnderlyingMultiplexer => this; @@ -125,10 +140,11 @@ static ConnectionMultiplexer() SetAutodetectFeatureFlags(); } - private ConnectionMultiplexer(ConfigurationOptions configuration, ServerType? serverType = null, EndPointCollection? endpoints = null) + private ConnectionMultiplexer(ConfigurationOptions configuration, ServerType? serverType = null, EndPointCollection? endpoints = null, Availability.CircuitBreaker? groupCircuitBreaker = null) { Interlocked.Increment(ref s_MuxerCreateCount); + GroupCircuitBreaker = groupCircuitBreaker; RawConfig = configuration ?? throw new ArgumentNullException(nameof(configuration)); EndPoints = endpoints ?? RawConfig.EndPoints.Clone(); EndPoints.SetDefaultPorts(serverType, ssl: RawConfig.Ssl); @@ -156,9 +172,9 @@ private ConnectionMultiplexer(ConfigurationOptions configuration, ServerType? se lastHeartbeatTicks = Environment.TickCount; } - private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions configuration, ILogger? log, ServerType? serverType, out EventHandler? connectHandler, EndPointCollection? endpoints = null) + private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions configuration, ILogger? log, ServerType? serverType, out EventHandler? connectHandler, EndPointCollection? endpoints = null, Availability.CircuitBreaker? groupCircuitBreaker = null) { - var muxer = new ConnectionMultiplexer(configuration, serverType, endpoints); + var muxer = new ConnectionMultiplexer(configuration, serverType, endpoints, groupCircuitBreaker); connectHandler = null; if (log is not null) { @@ -575,7 +591,35 @@ public static Task ConnectAsync(ConfigurationOptions conf : ConnectImplAsync(configuration, log); } - private static async Task ConnectImplAsync(ConfigurationOptions configuration, TextWriter? writer = null, ServerType? serverType = null) + /// + /// Connect a multiplexer that is a member of a connection group, applying the group's resolved + /// circuit-breaker without writing it back into the caller's + /// (which the caller may legitimately reuse for other connections). + /// + internal static Task ConnectGroupMemberAsync(ConfigurationOptions configuration, TextWriter? log, Availability.CircuitBreaker? groupCircuitBreaker) + { + Dependencies.Assert(); + Validate(configuration); + + if (configuration.IsSentinel) + { + // the sentinel path builds the primary connection internally, so we cannot pass the breaker + // down into construction; apply it afterwards - it is picked up by subsequent physical + // connections, and an explicit ConfigurationOptions.CircuitBreaker still applies throughout + return ApplyAfterConnectAsync(SentinelPrimaryConnectAsync(configuration, log), groupCircuitBreaker); + } + + return ConnectImplAsync(configuration, log, groupCircuitBreaker: groupCircuitBreaker); + + static async Task ApplyAfterConnectAsync(Task pending, Availability.CircuitBreaker? groupCircuitBreaker) + { + var muxer = await pending.ForAwait(); + muxer.GroupCircuitBreaker = groupCircuitBreaker; + return muxer; + } + } + + private static async Task ConnectImplAsync(ConfigurationOptions configuration, TextWriter? writer = null, ServerType? serverType = null, Availability.CircuitBreaker? groupCircuitBreaker = null) { IDisposable? killMe = null; EventHandler? connectHandler = null; @@ -587,7 +631,7 @@ private static async Task ConnectImplAsync(ConfigurationO var sw = ValueStopwatch.StartNew(); log?.LogInformationConnectingAsync(RuntimeInformation.FrameworkDescription, Utils.GetLibVersion()); - muxer = CreateMultiplexer(configuration, log, serverType, out connectHandler); + muxer = CreateMultiplexer(configuration, log, serverType, out connectHandler, groupCircuitBreaker: groupCircuitBreaker); killMe = muxer; Interlocked.Increment(ref muxer._connectAttemptCount); bool configured = await muxer.ReconfigureAsync(first: true, reconfigureAll: false, log, null, "connect").ObserveErrors().ForAwait(); diff --git a/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs b/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs index e15f951af..1d401d742 100644 --- a/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs +++ b/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs @@ -50,6 +50,12 @@ public interface IConnectionGroup : IConnectionMultiplexer /// Gets the currently active member. /// ConnectionGroupMember? ActiveMember { get; } + + /// + /// Gets the group-wide options this group was created with; these are immutable, and are the defaults + /// against which each 's overrides are resolved. + /// + MultiGroupOptions Options { get; } } /// diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index bbf4b6811..3bfb8faba 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -118,8 +118,9 @@ public PhysicalConnection(PhysicalBridge bridge, BufferedStreamWriter.WriteMode _inputCancel = new(); _outputCancel = new(); } - // grab a per-connection accumulator from the configured breaker (null when none is configured) - circuitBreaker = bridge.Multiplexer.RawConfig.CircuitBreaker?.CreateAccumulator(); + // grab a per-connection accumulator from the configured breaker (null when none is configured); + // for a connection-group member this resolves the group's default too - see EffectiveCircuitBreaker + circuitBreaker = bridge.Multiplexer.EffectiveCircuitBreaker?.CreateAccumulator(); OnCreateEcho(); } diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 174c7f37a..1e48387d6 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable StackExchange.Redis.IDatabase.ListMove(StackExchange.Redis.RedisKey sourceKey, StackExchange.Redis.RedisKey destinationKey, StackExchange.Redis.ListSide sourceSide, StackExchange.Redis.ListSide destinationSide, long count, StackExchange.Redis.ListMoveCount mode = StackExchange.Redis.ListMoveCount.UpTo, StackExchange.Redis.ListMoveOrder order = StackExchange.Redis.ListMoveOrder.Bulk, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue[]? StackExchange.Redis.IDatabase.SetCombineLength(StackExchange.Redis.SetOperation operation, StackExchange.Redis.RedisKey[]! keys, long limit = 0, bool approximate = false, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long StackExchange.Redis.IDatabase.StreamRead(StackExchange.Redis.StreamPosition[]! streamPositions, int? countPerStream = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisStream[]! @@ -24,21 +24,15 @@ StackExchange.Redis.RedisFeatures.HashImport.get -> bool StackExchange.Redis.IDatabaseAsync.Database.get -> int [SER007]StackExchange.Redis.Availability.RetryPolicy [SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void [SER007]StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.RetryPolicy.JitterMax.set -> void [SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int -[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.set -> void [SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int -[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void [SER007]StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags -[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.set -> void [SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void [SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey StackExchange.Redis.Availability.DatabaseExtensions -[SER007]static StackExchange.Redis.Availability.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! +[SER007]static StackExchange.Redis.Availability.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy? retryPolicy = null) -> StackExchange.Redis.IDatabaseAsync! [SER007]StackExchange.Redis.IDatabaseAsync.CreateTransaction(object? asyncState = null) -> StackExchange.Redis.ITransactionAsync! StackExchange.Redis.ITransactionAsync StackExchange.Redis.ITransactionAsync.AddCondition(StackExchange.Redis.Condition! condition) -> StackExchange.Redis.ConditionResult! @@ -79,40 +73,9 @@ StackExchange.Redis.ITransactionAsync.ExecuteAsync(StackExchange.Redis.CommandFl [SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.PreviousGroup.get -> StackExchange.Redis.Availability.ConnectionGroupMember? [SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.Type.get -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType [SER007]StackExchange.Redis.Availability.HealthCheck -[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IConnectionMultiplexer! multiplexer) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.Availability.HealthCheck.Clone() -> StackExchange.Redis.Availability.HealthCheck! -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheck() -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.HealthCheckProbe() -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Failure.get -> int -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.HealthCheckProbeContext() -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.HealthCheckProbeContext(StackExchange.Redis.Availability.HealthCheck.HealthCheckResult result, int success, int failure, int remaining, System.TimeSpan probeInterval) -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ProbeInterval.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Remaining.get -> int -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Result.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Success.get -> int -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.HealthCheckProbePolicy() -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult.Healthy = 1 -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult.Inconclusive = 0 -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult.Unhealthy = 2 -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.Availability.HealthCheck.Interval.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.HealthCheck.Interval.set -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe -[SER007]StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.KeyWriteHealthCheckProbe() -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.Probe.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]StackExchange.Redis.Availability.HealthCheck.Probe.set -> void [SER007]StackExchange.Redis.Availability.HealthCheck.ProbeCount.get -> int -[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeCount.set -> void [SER007]StackExchange.Redis.Availability.HealthCheck.ProbeInterval.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeInterval.set -> void -[SER007]StackExchange.Redis.Availability.HealthCheck.ProbePolicy.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! -[SER007]StackExchange.Redis.Availability.HealthCheck.ProbePolicy.set -> void [SER007]StackExchange.Redis.Availability.HealthCheck.ProbeTimeout.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeTimeout.set -> void [SER007]StackExchange.Redis.Availability.IConnectionGroup [SER007]StackExchange.Redis.Availability.IConnectionGroup.ActiveMember.get -> StackExchange.Redis.Availability.ConnectionGroupMember? [SER007]StackExchange.Redis.Availability.IConnectionGroup.AddAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! @@ -122,41 +85,21 @@ StackExchange.Redis.ITransactionAsync.ExecuteAsync(StackExchange.Redis.CommandFl [SER007]StackExchange.Redis.Availability.IConnectionGroup.TryFailoverTo(StackExchange.Redis.Availability.ConnectionGroupMember? member) -> bool [SER007]StackExchange.Redis.Availability.MultiGroupOptions [SER007]StackExchange.Redis.Availability.MultiGroupOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker! -[SER007]StackExchange.Redis.Availability.MultiGroupOptions.CircuitBreaker.set -> void [SER007]StackExchange.Redis.Availability.MultiGroupOptions.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck! -[SER007]StackExchange.Redis.Availability.MultiGroupOptions.HealthCheck.set -> void -[SER007]StackExchange.Redis.Availability.MultiGroupOptions.MultiGroupOptions() -> void [SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member0, StackExchange.Redis.Availability.ConnectionGroupMember! member1, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! [SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember![]! members, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! [SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? [SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.set -> void -[SER007]StackExchange.Redis.ConfigurationOptions.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? -[SER007]StackExchange.Redis.ConfigurationOptions.HealthCheck.set -> void [SER007]StackExchange.Redis.ConnectionFailureType.CircuitBreaker = 11 -> StackExchange.Redis.ConnectionFailureType [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.IsUnhealthy.get -> bool [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ResetIsUnhealthy() -> void [SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.set -> void [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! [SER007]static StackExchange.Redis.Availability.CircuitBreaker.Default.get -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]static StackExchange.Redis.Availability.CircuitBreaker.None.get -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.Ping.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.StringSet.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! -[SER007]override StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]override StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ToString() -> string! [SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! [SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! [SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! @@ -197,11 +140,6 @@ StackExchange.Redis.ITransactionAsync.ExecuteAsync(StackExchange.Redis.CommandFl [SER007]StackExchange.Redis.RedisErrorKind.WrongType = 22 -> StackExchange.Redis.RedisErrorKind [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.FaultContext fault) -> void [SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsFailure(in StackExchange.Redis.Availability.FaultContext fault) -> bool -[SER007]virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault) -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.None = 0 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.SameServer = 1 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.FailoverServer = 2 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult [SER007]StackExchange.Redis.CommandFlags.CommandRetryAlways = 8192 -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryConnection = 32768 -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryReadOnly = 65536 -> StackExchange.Redis.CommandFlags @@ -217,3 +155,117 @@ StackExchange.Redis.RedisServerException.Kind.get -> StackExchange.Redis.RedisEr StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, StackExchange.Redis.CommandFlags flags, string! message) -> void StackExchange.Redis.RedisTimeoutException.Flags.get -> StackExchange.Redis.CommandFlags StackExchange.Redis.RedisTimeoutException.RedisTimeoutException(StackExchange.Redis.CommandFlags flags, string! message, StackExchange.Redis.CommandStatus commandStatus) -> void +[SER007]abstract StackExchange.Redis.Availability.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheckContext context) -> System.Threading.Tasks.Task! +[SER007]abstract StackExchange.Redis.Availability.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]abstract StackExchange.Redis.Availability.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheckContext context, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! +[SER007]override sealed StackExchange.Redis.Availability.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheckContext context) -> System.Threading.Tasks.Task! +[SER007]override StackExchange.Redis.Availability.HealthCheckContext.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.HealthCheckProbeContext.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.HealthCheck.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.MultiGroupOptions.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.RetryPolicy.ToString() -> string! +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.CircuitBreaker.set -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.FailbackDelay.get -> System.TimeSpan? +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.FailbackDelay.set -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.HealthCheck.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Builder(StackExchange.Redis.Availability.HealthCheck! healthCheck) -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Create() -> StackExchange.Redis.Availability.HealthCheck! +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeCount.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeCount.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Probe.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeInterval.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbePolicy.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbePolicy.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Probe.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeTimeout.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeTimeout.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IConnectionMultiplexer! multiplexer) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.HealthCheckContext +[SER007]StackExchange.Redis.Availability.HealthCheckContext.HealthCheckContext() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckContext.HealthCheckContext(StackExchange.Redis.IServer! server, System.TimeSpan probeTimeout) -> void +[SER007]StackExchange.Redis.Availability.HealthCheckContext.ProbeTimeout.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheckContext.Server.get -> StackExchange.Redis.IServer! +[SER007]StackExchange.Redis.Availability.HealthCheck.IsEnabled.get -> bool +[SER007]StackExchange.Redis.Availability.HealthCheckProbe +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Failure.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.HealthCheckProbeContext() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.HealthCheckProbeContext(StackExchange.Redis.Availability.HealthCheckResult result, int success, int failure, int remaining, System.TimeSpan probeInterval) -> void +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.ProbeInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Remaining.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Result.get -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Success.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.Probe.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]StackExchange.Redis.Availability.HealthCheckProbe.HealthCheckProbe() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckProbePolicy +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbePolicy.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]StackExchange.Redis.Availability.HealthCheckProbePolicy.HealthCheckProbePolicy() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckResult.Healthy = 1 -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckResult.Inconclusive = 0 -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckResult.Unhealthy = 2 -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.IConnectionGroup.Options.get -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]StackExchange.Redis.Availability.KeyWriteHealthCheckProbe +[SER007]StackExchange.Redis.Availability.KeyWriteHealthCheckProbe.KeyWriteHealthCheckProbe() -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.Builder(StackExchange.Redis.Availability.MultiGroupOptions! options) -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.CircuitBreaker.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.Create() -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackDelay.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheckInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheckInterval.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheck.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.RetryPolicy.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.RetryPolicy.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.HealthCheckInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.RetryPolicy.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.Builder(StackExchange.Redis.Availability.RetryPolicy! policy) -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.Create() -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.FailoverDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.FailoverDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.JitterMax.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.JitterMax.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttemptsBeforeFailover.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttemptsBeforeFailover.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttempts.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttempts.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxCommandRetryCategory.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.RetryDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.RetryDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicy(StackExchange.Redis.Availability.RetryPolicy.Builder! builder) -> void +[SER007]StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.Availability.RetryResult.FailoverServer = 2 -> StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.Availability.RetryResult.None = 0 -> StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.Availability.RetryResult.SameServer = 1 -> StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.ConfigurationOptions.RetryPolicy.get -> StackExchange.Redis.Availability.RetryPolicy? +[SER007]StackExchange.Redis.ConfigurationOptions.RetryPolicy.set -> void +[SER007]static StackExchange.Redis.Availability.HealthCheck.Builder.implicit operator StackExchange.Redis.Availability.HealthCheck!(StackExchange.Redis.Availability.HealthCheck.Builder! builder) -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.Availability.HealthCheck.None.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.None.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.Ping.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.StringSet.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Builder.implicit operator StackExchange.Redis.Availability.MultiGroupOptions!(StackExchange.Redis.Availability.MultiGroupOptions.Builder! builder) -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]static StackExchange.Redis.Availability.RetryPolicy.Builder.implicit operator StackExchange.Redis.Availability.RetryPolicy!(StackExchange.Redis.Availability.RetryPolicy.Builder! builder) -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]static StackExchange.Redis.Availability.RetryPolicy.Default.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]static StackExchange.Redis.Availability.RetryPolicy.None.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault) -> StackExchange.Redis.Availability.RetryResult diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 8d5f585ba..51d4ca902 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -68,6 +68,9 @@ HealthCheck.cs + + HealthCheckProbe.cs + MultiGroupSubscriber.cs diff --git a/tests/StackExchange.Redis.Tests/AvailabilityConfigTests.cs b/tests/StackExchange.Redis.Tests/AvailabilityConfigTests.cs new file mode 100644 index 000000000..4328fcc58 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/AvailabilityConfigTests.cs @@ -0,0 +1,333 @@ +using System; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Covers the shape shared by every Availability configuration type: an immutable policy with static +/// Default/None, configured through a nested Builder that validates in Create() and collapses onto the +/// shared default when nothing was customized. +/// +public class AvailabilityConfigTests +{ + // ---- HealthCheck ---- + [Fact] + public void HealthCheck_UntouchedBuilder_CollapsesOntoDefault() + { + Assert.Same(HealthCheck.Default, new HealthCheck.Builder().Create()); + Assert.Same(HealthCheck.Default, new HealthCheck.Builder(HealthCheck.Default).Create()); + } + + [Fact] + public void HealthCheck_BuilderRoundTripsExistingInstance() + { + HealthCheck original = new HealthCheck.Builder + { + ProbeCount = 7, + ProbeTimeout = TimeSpan.FromSeconds(11), + ProbeInterval = TimeSpan.FromMilliseconds(250), + Probe = HealthCheckProbe.IsConnected, + ProbePolicy = HealthCheckProbePolicy.MajoritySuccess, + }; + + // the copy constructor is the replacement for the old Clone() + var copy = new HealthCheck.Builder(original).Create(); + + Assert.NotSame(original, copy); + Assert.Equal(original.ProbeCount, copy.ProbeCount); + Assert.Equal(original.ProbeTimeout, copy.ProbeTimeout); + Assert.Equal(original.ProbeInterval, copy.ProbeInterval); + Assert.Same(original.Probe, copy.Probe); + Assert.Same(original.ProbePolicy, copy.ProbePolicy); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void HealthCheck_RejectsNonPositiveProbeCount(int probeCount) + { + var builder = new HealthCheck.Builder { ProbeCount = probeCount }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(HealthCheck.Builder.ProbeCount), ex.ParamName); + } + + [Fact] + public void HealthCheck_RejectsNonPositiveProbeTimeout() + { + var builder = new HealthCheck.Builder { ProbeTimeout = TimeSpan.Zero }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(HealthCheck.Builder.ProbeTimeout), ex.ParamName); + } + + [Fact] + public void HealthCheck_RejectsNegativeProbeInterval() + { + var builder = new HealthCheck.Builder { ProbeInterval = TimeSpan.FromMilliseconds(-1) }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(HealthCheck.Builder.ProbeInterval), ex.ParamName); + } + + [Fact] + public void HealthCheck_RejectsUnrepresentableTotalBudget() + { + // ProbeCount x ProbeTimeout has to fit in int milliseconds; this used to overflow silently + var builder = new HealthCheck.Builder { ProbeCount = 1000, ProbeTimeout = TimeSpan.FromDays(30) }; + Assert.Throws(() => builder.Create()); + } + + [Fact] + public void HealthCheck_None_IsDisabledAndStable() + { + Assert.Same(HealthCheck.None, HealthCheck.None); + Assert.NotSame(HealthCheck.None, HealthCheck.Default); + Assert.False(HealthCheck.None.IsEnabled); + Assert.True(HealthCheck.Default.IsEnabled); + } + + [Fact] + public async Task HealthCheck_None_ReportsInconclusiveWithoutProbing() + { + // a null server would throw if the probe were actually invoked + Assert.Equal(HealthCheckResult.Inconclusive, await HealthCheck.None.CheckHealthAsync(server: null!)); + } + + // ---- RetryPolicy ---- + [Fact] + public void RetryPolicy_UntouchedBuilder_CollapsesOntoDefault() + { + Assert.Same(RetryPolicy.Default, new RetryPolicy.Builder().Create()); + Assert.Same(RetryPolicy.Default, new RetryPolicy.Builder(RetryPolicy.Default).Create()); + } + + [Fact] + public void RetryPolicy_BuilderRoundTripsExistingInstance() + { + RetryPolicy original = new RetryPolicy.Builder + { + MaxAttempts = 9, + MaxAttemptsBeforeFailover = 4, + RetryDelay = TimeSpan.FromMilliseconds(123), + JitterMax = TimeSpan.FromMilliseconds(45), + FailoverDelay = TimeSpan.FromSeconds(6), + MaxCommandRetryCategory = CommandFlags.CommandRetryWriteAccumulating, + }; + + var copy = new RetryPolicy.Builder(original).Create(); + + Assert.NotSame(original, copy); + Assert.Equal(original.MaxAttempts, copy.MaxAttempts); + Assert.Equal(original.MaxAttemptsBeforeFailover, copy.MaxAttemptsBeforeFailover); + Assert.Equal(original.RetryDelay, copy.RetryDelay); + Assert.Equal(original.JitterMax, copy.JitterMax); + Assert.Equal(original.FailoverDelay, copy.FailoverDelay); + Assert.Equal(original.MaxCommandRetryCategory, copy.MaxCommandRetryCategory); + } + + [Fact] + public void RetryPolicy_RejectsZeroAttempts() + { + var builder = new RetryPolicy.Builder { MaxAttempts = 0 }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(RetryPolicy.Builder.MaxAttempts), ex.ParamName); + } + + [Fact] + public void RetryPolicy_RejectsZeroAttemptsBeforeFailover() + { + // previously this silently disabled failover, and only threw later, from WithRetry + var builder = new RetryPolicy.Builder { MaxAttemptsBeforeFailover = 0 }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(RetryPolicy.Builder.MaxAttemptsBeforeFailover), ex.ParamName); + } + + [Fact] + public void RetryPolicy_RejectsNegativeDelays() + { + Assert.Equal( + nameof(RetryPolicy.Builder.RetryDelay), + Assert.Throws(() => new RetryPolicy.Builder { RetryDelay = TimeSpan.FromTicks(-1) }.Create()).ParamName); + Assert.Equal( + nameof(RetryPolicy.Builder.JitterMax), + Assert.Throws(() => new RetryPolicy.Builder { JitterMax = TimeSpan.FromTicks(-1) }.Create()).ParamName); + Assert.Equal( + nameof(RetryPolicy.Builder.FailoverDelay), + Assert.Throws(() => new RetryPolicy.Builder { FailoverDelay = TimeSpan.FromTicks(-1) }.Create()).ParamName); + } + + [Theory] + [InlineData(CommandFlags.None)] // no category at all + [InlineData(CommandFlags.FireAndForget)] // not a category + [InlineData(CommandFlags.CommandRetryReadOnly | CommandFlags.PreferReplica)] // category plus noise + public void RetryPolicy_RejectsInvalidRetryCategory(CommandFlags flags) + { + var builder = new RetryPolicy.Builder { MaxCommandRetryCategory = flags }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(RetryPolicy.Builder.MaxCommandRetryCategory), ex.ParamName); + } + + [Fact] + public void RetryPolicy_None_NeverRetries() + { + Assert.Same(RetryPolicy.None, RetryPolicy.None); + Assert.NotSame(RetryPolicy.None, RetryPolicy.Default); + + // a transient, retryable fault on a read-only command: the default policy retries, None does not + var fault = new FaultContext(new RedisConnectionException(ConnectionFailureType.SocketFailure, CommandFlags.None, "boom")); + Assert.Equal(RetryResult.None, RetryPolicy.None.CanRetry(in fault)); + } + + // ---- CircuitBreaker ---- + [Fact] + public void CircuitBreaker_RejectsOutOfRangeThreshold() + { + Assert.Equal( + nameof(CircuitBreaker.Builder.FailureRateThreshold), + Assert.Throws(() => new CircuitBreaker.Builder { FailureRateThreshold = 101 }.Create()).ParamName); + Assert.Equal( + nameof(CircuitBreaker.Builder.FailureRateThreshold), + Assert.Throws(() => new CircuitBreaker.Builder { FailureRateThreshold = -1 }.Create()).ParamName); + } + + [Fact] + public void CircuitBreaker_RejectsInvalidWindowAndMinimum() + { + Assert.Equal( + nameof(CircuitBreaker.Builder.MinimumNumberOfFailures), + Assert.Throws(() => new CircuitBreaker.Builder { MinimumNumberOfFailures = 0 }.Create()).ParamName); + Assert.Equal( + nameof(CircuitBreaker.Builder.MetricsWindowSize), + Assert.Throws(() => new CircuitBreaker.Builder { MetricsWindowSize = TimeSpan.Zero }.Create()).ParamName); + } + + // ---- MultiGroupOptions ---- + [Fact] + public void MultiGroupOptions_UntouchedBuilder_CollapsesOntoDefault() + { + Assert.Same(MultiGroupOptions.Default, new MultiGroupOptions.Builder().Create()); + Assert.Same(MultiGroupOptions.Default, new MultiGroupOptions.Builder(MultiGroupOptions.Default).Create()); + } + + [Fact] + public void MultiGroupOptions_DefaultsAreTheSharedPolicyDefaults() + { + var options = MultiGroupOptions.Default; + Assert.Same(HealthCheck.Default, options.HealthCheck); + Assert.Same(CircuitBreaker.Default, options.CircuitBreaker); + Assert.Same(RetryPolicy.Default, options.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(5), options.HealthCheckInterval); + Assert.Equal(TimeSpan.Zero, options.FailbackDelay); + } + + [Fact] + public void MultiGroupOptions_RejectsInvalidIntervals() + { + Assert.Equal( + nameof(MultiGroupOptions.Builder.HealthCheckInterval), + Assert.Throws(() => new MultiGroupOptions.Builder { HealthCheckInterval = TimeSpan.Zero }.Create()).ParamName); + Assert.Equal( + nameof(MultiGroupOptions.Builder.FailbackDelay), + Assert.Throws(() => new MultiGroupOptions.Builder { FailbackDelay = TimeSpan.FromTicks(-1) }.Create()).ParamName); + + // MaxValue is the documented "never" sentinel for both, and must remain legal + MultiGroupOptions ok = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.MaxValue, + FailbackDelay = TimeSpan.MaxValue, + }; + Assert.Equal(TimeSpan.MaxValue, ok.HealthCheckInterval); + Assert.Equal(TimeSpan.MaxValue, ok.FailbackDelay); + } + + [Fact] + public void MultiGroupOptions_BuilderConvertsImplicitly() + { + // every Builder in the namespace supports this, so options can be written inline at the call-site + MultiGroupOptions options = new MultiGroupOptions.Builder { FailbackDelay = TimeSpan.FromMinutes(2) }; + Assert.Equal(TimeSpan.FromMinutes(2), options.FailbackDelay); + } + + // ---- per-member override resolution ---- + [Fact] + public void Member_ResolvesGroupDefaultsWhenNoOverride() + { + var member = new ConnectionGroupMember("localhost:6379"); + var options = MultiGroupOptions.Default; + + Assert.Same(options.HealthCheck, member.ResolveHealthCheck(options)); + Assert.Same(options.CircuitBreaker, member.ResolveCircuitBreaker(options)); + Assert.Equal(options.FailbackDelay, member.ResolveFailbackDelay(options)); + } + + [Fact] + public void Member_OverridesBeatGroupDefaults() + { + HealthCheck memberCheck = new HealthCheck.Builder { ProbeCount = 1 }; + CircuitBreaker memberBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 42 }; + var member = new ConnectionGroupMember("localhost:6379") + { + HealthCheck = memberCheck, + CircuitBreaker = memberBreaker, + FailbackDelay = TimeSpan.FromMinutes(3), + }; + + var options = MultiGroupOptions.Default; + Assert.Same(memberCheck, member.ResolveHealthCheck(options)); + Assert.Same(memberBreaker, member.ResolveCircuitBreaker(options)); + Assert.Equal(TimeSpan.FromMinutes(3), member.ResolveFailbackDelay(options)); + } + + [Fact] + public void Member_CircuitBreakerFallsBackToItsOwnConfigurationBeforeTheGroup() + { + // precedence is: member override, then the member's own ConfigurationOptions, then the group default + CircuitBreaker fromConfig = new CircuitBreaker.Builder { FailureRateThreshold = 42 }; + var config = ConfigurationOptions.Parse("localhost:6379"); + config.CircuitBreaker = fromConfig; + + var member = new ConnectionGroupMember(config); + Assert.Same(fromConfig, member.ResolveCircuitBreaker(MultiGroupOptions.Default)); + + CircuitBreaker fromMember = new CircuitBreaker.Builder { FailureRateThreshold = 13 }; + member.CircuitBreaker = fromMember; + Assert.Same(fromMember, member.ResolveCircuitBreaker(MultiGroupOptions.Default)); + } + + [Fact] + public void GroupDefaultsAreNotWrittenBackIntoCallerConfiguration() + { + // callers may legitimately reuse a ConfigurationOptions across connections, so resolving a group + // default must not mutate it (this used to be a `config.CircuitBreaker ??= options.CircuitBreaker`) + var config = ConfigurationOptions.Parse("localhost:6379"); + var member = new ConnectionGroupMember(config); + + Assert.Same(MultiGroupOptions.Default.CircuitBreaker, member.ResolveCircuitBreaker(MultiGroupOptions.Default)); + Assert.Null(config.CircuitBreaker); + } + + // ---- WithRetry() policy resolution ---- + [Fact] + public async Task WithRetry_UsesConfiguredPolicyForASingleConnection() + { + RetryPolicy configured = new RetryPolicy.Builder { MaxAttempts = 7 }; + var config = ConfigurationOptions.Parse("localhost:6379"); + config.RetryPolicy = configured; + config.AbortOnConnectFail = false; + + await using var muxer = await ConnectionMultiplexer.ConnectAsync(config); + var retrying = Assert.IsType(muxer.GetDatabase().WithRetry()); + Assert.Same(configured, retrying.Policy); + } + + [Fact] + public async Task WithRetry_FallsBackToDefaultWhenNoneConfigured() + { + var config = ConfigurationOptions.Parse("localhost:6379"); + config.AbortOnConnectFail = false; + + await using var muxer = await ConnectionMultiplexer.ConnectAsync(config); + var retrying = Assert.IsType(muxer.GetDatabase().WithRetry()); + Assert.Same(RetryPolicy.Default, retrying.Policy); + } +} diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 83c02bbcb..d2f5c3c0b 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -80,7 +80,6 @@ orderby name "defaultOptions", "defaultVersion", "EndPoints", - "HealthCheck", "heartbeatInterval", "keepAlive", "LibraryName", @@ -95,6 +94,7 @@ orderby name "RequestBufferPool", "ResponseBufferPool", "responseTimeout", + "RetryPolicy", "ServiceName", "SocketManager", #if !NETFRAMEWORK diff --git a/tests/StackExchange.Redis.Tests/ControllableProbe.cs b/tests/StackExchange.Redis.Tests/ControllableProbe.cs index 4d8e47dc7..4d5c5ae23 100644 --- a/tests/StackExchange.Redis.Tests/ControllableProbe.cs +++ b/tests/StackExchange.Redis.Tests/ControllableProbe.cs @@ -7,12 +7,12 @@ namespace StackExchange.Redis.Tests; // A health-check probe whose verdict is driven by the test: nominated endpoints report unhealthy on // demand, everything else is healthy. This keeps a specific member deselected deterministically, even // after its physical connection reconnects underneath us. -internal sealed class ControllableProbe : HealthCheck.HealthCheckProbe +internal sealed class ControllableProbe : HealthCheckProbe { private volatile EndPoint? _down; public void MarkDown(EndPoint endpoint) => _down = endpoint; - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) - => Equals(server.EndPoint, _down) ? UnhealthyTask : HealthyTask; + public override Task CheckHealthAsync(HealthCheckContext context) + => Equals(context.Server.EndPoint, _down) ? UnhealthyTask : HealthyTask; } diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs index 1bbcebb4e..c8a30030e 100644 --- a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -1,6 +1,6 @@ using System; +using StackExchange.Redis.Availability; using Xunit; -using static StackExchange.Redis.Availability.HealthCheck; namespace StackExchange.Redis.Tests; diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs index a99fcd724..23eaaa7bd 100644 --- a/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs @@ -108,13 +108,13 @@ public enum InbuiltProbe [InlineData(InbuiltProbe.StringSet, ServerType.Cluster)] public async Task SelectByWeight(InbuiltProbe probe, ServerType serverType) { - var healthCheck = new HealthCheck + HealthCheck healthCheck = new HealthCheck.Builder { Probe = probe switch { - InbuiltProbe.IsConnected => HealthCheck.HealthCheckProbe.IsConnected, - InbuiltProbe.Ping => HealthCheck.HealthCheckProbe.Ping, - InbuiltProbe.StringSet => HealthCheck.HealthCheckProbe.StringSet, + InbuiltProbe.IsConnected => HealthCheckProbe.IsConnected, + InbuiltProbe.Ping => HealthCheckProbe.Ping, + InbuiltProbe.StringSet => HealthCheckProbe.StringSet, _ => throw new ArgumentOutOfRangeException(nameof(probe)), }, }; @@ -132,7 +132,7 @@ public async Task SelectByWeight(InbuiltProbe probe, ServerType serverType) new(server1.GetClientConfig()) { Weight = 9 }, new(server2.GetClientConfig()) { Weight = 3 }, ]; - var options = new MultiGroupOptions { HealthCheck = healthCheck }; + MultiGroupOptions options = new MultiGroupOptions.Builder { HealthCheck = healthCheck }; await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); Assert.True(conn.IsConnected); var typed = Assert.IsType(conn); diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs index abd442e60..86d3dc01f 100644 --- a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs @@ -42,13 +42,13 @@ public async Task CircuitBreakerTrip_ReroutesAwayFromMember() new(serverC.GetClientConfig(), "C") { Weight = 1 }, ]; - var options = new MultiGroupOptions + MultiGroupOptions options = new MultiGroupOptions.Builder { - HealthCheck = new HealthCheck + // enormous, so the poll loop cannot be what reroutes us during the test + HealthCheckInterval = TimeSpan.FromMinutes(30), + HealthCheck = new HealthCheck.Builder { Probe = probe, - // enormous, so the poll loop cannot be what reroutes us during the test - Interval = TimeSpan.FromMinutes(30), ProbeCount = 1, ProbeTimeout = TimeSpan.FromSeconds(5), }, diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/GroupConfigResolutionTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/GroupConfigResolutionTests.cs new file mode 100644 index 000000000..2aca640ba --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/GroupConfigResolutionTests.cs @@ -0,0 +1,102 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +/// +/// Verifies how a live group resolves its configuration: that group defaults reach the members, that a +/// per-member override wins, and that none of this mutates the caller's . +/// +public class GroupConfigResolutionTests(ITestOutputHelper log) +{ + [Fact] + public async Task GroupExposesItsOptions() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + FailbackDelay = TimeSpan.FromMinutes(4), + RetryPolicy = new RetryPolicy.Builder { MaxAttempts = 6 }, + }; + + ConnectionGroupMember[] members = [new(server0.GetClientConfig()), new(server1.GetClientConfig())]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + + Assert.Same(options, conn.Options); + Assert.Equal(TimeSpan.FromMinutes(4), conn.Options.FailbackDelay); + } + + [Fact] + public async Task WithRetryUsesTheGroupPolicy() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + RetryPolicy groupPolicy = new RetryPolicy.Builder { MaxAttempts = 6, RetryDelay = TimeSpan.Zero }; + MultiGroupOptions options = new MultiGroupOptions.Builder { RetryPolicy = groupPolicy }; + + ConnectionGroupMember[] members = [new(server0.GetClientConfig()), new(server1.GetClientConfig())]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + + // the parameterless overload resolves the policy from the group it is attached to + var retrying = Assert.IsType(conn.GetDatabase().WithRetry()); + Assert.Same(groupPolicy, retrying.Policy); + + // ...and an explicit policy still wins + RetryPolicy explicitPolicy = new RetryPolicy.Builder { MaxAttempts = 2 }; + var explicitlyRetrying = Assert.IsType(conn.GetDatabase().WithRetry(explicitPolicy)); + Assert.Same(explicitPolicy, explicitlyRetrying.Policy); + } + + [Fact] + public async Task GroupCircuitBreakerReachesMembersWithoutMutatingCallerConfig() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + CircuitBreaker groupBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 42 }; + CircuitBreaker memberBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 13 }; + + var config0 = server0.GetClientConfig(); + var config1 = server1.GetClientConfig(); + + MultiGroupOptions options = new MultiGroupOptions.Builder { CircuitBreaker = groupBreaker }; + ConnectionGroupMember[] members = [new(config0), new(config1) { CircuitBreaker = memberBreaker }]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + + // the group default reached the first member's connection, and the override reached the second + Assert.Same(groupBreaker, AsMultiplexer(members[0]).EffectiveCircuitBreaker); + Assert.Same(memberBreaker, AsMultiplexer(members[1]).EffectiveCircuitBreaker); + + // ...and neither was written back into the caller's configuration, which remains reusable + Assert.Null(config0.CircuitBreaker); + Assert.Null(config1.CircuitBreaker); + + static ConnectionMultiplexer AsMultiplexer(ConnectionGroupMember member) => member.Multiplexer; + } + + [Fact] + public async Task DisabledHealthCheckLeavesMemberSelectableOnConnectivityAlone() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + // HealthCheck.None performs no probes and reports Inconclusive, which is not Unhealthy - so a + // connected member stays eligible, and the higher weight still wins + MultiGroupOptions options = new MultiGroupOptions.Builder { HealthCheck = HealthCheck.None }; + ConnectionGroupMember[] members = [ + new(server0.GetClientConfig(), "alpha") { Weight = 1 }, + new(server1.GetClientConfig(), "beta") { Weight = 9 }, + ]; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.True(conn.IsConnected); + Assert.Equal("beta", conn.ActiveMember?.Name); + Assert.All(members, member => Assert.False(member.IsUnhealthy)); + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs index 0a7dbe657..45720df2e 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs @@ -11,11 +11,11 @@ public class CommandRetryPolicyUnitTests // Builds a FaultContext for a spoofed server error of the given kind, carrying the given // command-flags, and asks the policy whether it may be retried. - private static RetryPolicy.RetryPolicyResult CanRetry(RedisErrorKind kind, CommandFlags flags, RetryPolicy? policy = null) + private static RetryResult CanRetry(RedisErrorKind kind, CommandFlags flags, RetryPolicy? policy = null) { // the exception carries both the Kind and the command-flags; FaultContext reads them back var fault = new FaultContext(new RedisServerException(kind, flags, kind.ToString())); - return (policy ?? new RetryPolicy()).CanRetry(in fault); + return (policy ?? RetryPolicy.Default).CanRetry(in fault); } // The command's retry-category is checked against the policy's max category: the default max is @@ -35,7 +35,7 @@ private static RetryPolicy.RetryPolicyResult CanRetry(RedisErrorKind kind, Comma public void CanRetry_CategoryVersusDefaultMax(CommandFlags category, bool expectRetry) { var result = CanRetry(RedisErrorKind.Loading, category); - Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + Assert.Equal(expectRetry, result != RetryResult.None); } // With an in-range category (== default max), the outcome is decided purely by whether the error @@ -48,7 +48,7 @@ public void CanRetry_CategoryVersusDefaultMax(CommandFlags category, bool expect public void CanRetry_ErrorKindGatesRetry_WhenInRange(RedisErrorKind kind, bool expectRetry) { var result = CanRetry(kind, CommandFlags.CommandRetryWriteLastWins); - Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + Assert.Equal(expectRetry, result != RetryResult.None); } // "never" and "always" adjust only the category range - they do not override the error-kind check: @@ -62,15 +62,15 @@ public void CanRetry_ErrorKindGatesRetry_WhenInRange(RedisErrorKind kind, bool e public void CanRetry_NeverAndAlwaysAffectRangeNotErrorKind(CommandFlags category, RedisErrorKind kind, bool expectRetry) { var result = CanRetry(kind, category); - Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + Assert.Equal(expectRetry, result != RetryResult.None); } // When a retry is permitted, it normally offers both the same server and a failover server; but a // "server specific" (sticky) command must not move endpoints, so only the same-server option remains. [Theory] - [InlineData(CommandFlags.None, RetryPolicy.RetryPolicyResult.SameServer | RetryPolicy.RetryPolicyResult.FailoverServer)] - [InlineData(Message.CommandServerSpecific, RetryPolicy.RetryPolicyResult.SameServer)] - public void CanRetry_ServerSpecificRestrictsToSameServer(CommandFlags extra, RetryPolicy.RetryPolicyResult expected) + [InlineData(CommandFlags.None, RetryResult.SameServer | RetryResult.FailoverServer)] + [InlineData(Message.CommandServerSpecific, RetryResult.SameServer)] + public void CanRetry_ServerSpecificRestrictsToSameServer(CommandFlags extra, RetryResult expected) { // in-range category (== default max) + transient error => a retry is offered; the sticky flag // only changes *where* the retry may go, not *whether* it happens. @@ -89,8 +89,8 @@ public void CanRetry_ServerSpecificDoesNotAffectRange(CommandFlags category, boo var withoutFlag = CanRetry(RedisErrorKind.Loading, category); var withFlag = CanRetry(RedisErrorKind.Loading, category | Message.CommandServerSpecific); - Assert.Equal(expectRetry, withoutFlag != RetryPolicy.RetryPolicyResult.None); - Assert.Equal(expectRetry, withFlag != RetryPolicy.RetryPolicyResult.None); + Assert.Equal(expectRetry, withoutFlag != RetryResult.None); + Assert.Equal(expectRetry, withFlag != RetryResult.None); } // --- RetryDatabase.CanRetry: attempt accounting ---------------------------------- @@ -107,7 +107,7 @@ public void CanRetry_ServerSpecificDoesNotAffectRange(CommandFlags category, boo [InlineData(3, false)] public void RetryDatabase_CanRetry_MaxAttempts_NoFailover(int attempt, bool expected) { - var policy = new RetryPolicy { MaxAttempts = 3, MaxAttemptsBeforeFailover = 3 }; + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, MaxAttemptsBeforeFailover = 3 }; var controller = new RetryController(policy, DatabaseFeatureFlags.None); // CanRetry never touches any database using var cts = new CancellationTokenSource(); @@ -130,7 +130,7 @@ public void RetryDatabase_CanRetry_MaxAttempts_NoFailover(int attempt, bool expe [Fact] public void RetryDatabase_CanRetry_FailoverAtThreshold() { - var policy = new RetryPolicy { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; // failover is only armed when the inner database advertises the feature; supply it explicitly var controller = new RetryController(policy, DatabaseFeatureFlags.Failover); @@ -169,7 +169,7 @@ public void RetryDatabase_CanRetry_FailoverAtThreshold() [Fact] public void RetryDatabase_CanRetry_ServerSpecific_CannotFailover() { - var policy = new RetryPolicy { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; var controller = new RetryController(policy, DatabaseFeatureFlags.Failover); using var cts = new CancellationTokenSource(); diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index b7b2202de..414823af0 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -30,7 +30,7 @@ public async Task WithRetry_RidesOutTransientLoading() server.LoadingOps = 2; // zero delay/jitter so the test isn't paying the default ~1s retry backoff between attempts - var policy = new RetryPolicy + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, @@ -86,12 +86,12 @@ public async Task WithRetry_FailsOverBetweenGroupsOnLoading() new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target ]; - var options = new MultiGroupOptions + MultiGroupOptions options = new MultiGroupOptions.Builder { - HealthCheck = new HealthCheck + HealthCheckInterval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us + HealthCheck = new HealthCheck.Builder { Probe = probe, - Interval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us ProbeCount = 1, ProbeTimeout = TimeSpan.FromSeconds(5), }, @@ -102,7 +102,7 @@ public async Task WithRetry_FailsOverBetweenGroupsOnLoading() Assert.Same(members[0], conn.ActiveMember); // A is active (highest weight) // failover enabled, plenty of attempts, no artificial delay between them - var policy = new RetryPolicy + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 20, MaxAttemptsBeforeFailover = 1, @@ -143,7 +143,7 @@ public async Task WithRetry_Transaction_RidesOutTransientExec() server.FailExecOps = 1; // fail the first EXEC; the second should commit - var policy = new RetryPolicy + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, @@ -180,7 +180,7 @@ public async Task WithRetry_Transaction_AccumulatingOp_RespectsCategoryGate() RedisKey key = "retry:tran:incr"; // default cap = write-last-wins; an INCR makes the aggregate accumulating -> NOT retried - var conservative = new RetryPolicy { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + RetryPolicy conservative = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; server.FailExecOps = 1; var tran1 = db.WithRetry(conservative).CreateTransaction(); var incr1 = tran1.StringIncrementAsync(key); @@ -190,7 +190,7 @@ public async Task WithRetry_Transaction_AccumulatingOp_RespectsCategoryGate() Assert.Equal(0, server.FailExecOps); // raise the cap to allow accumulating writes: the same transaction now rides out the transient failure - var permissive = new RetryPolicy + RetryPolicy permissive = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, @@ -222,7 +222,7 @@ public async Task WithRetry_Transaction_SatisfiedCondition_RidesOutTransientExec server.FailExecOps = 1; // fail the first EXEC; the condition must still hold on the replay - var policy = new RetryPolicy { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; var tran = db.WithRetry(policy).CreateTransaction(); var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); // satisfied on both attempts var setTask = tran.StringSetAsync(key, "committed"); @@ -251,7 +251,7 @@ public async Task WithRetry_Transaction_UnsatisfiedCondition_AbortsWithoutRetry( server.FailExecOps = 0; // no transient fault; the condition itself aborts the transaction - var policy = new RetryPolicy { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; var tran = db.WithRetry(policy).CreateTransaction(); var cond = tran.AddCondition(Condition.StringEqual(key, "different")); // NOT satisfied var setTask = tran.StringSetAsync(key, "committed"); @@ -282,7 +282,7 @@ public async Task WithRetry_Transaction_PerOpError_FaultsOnlyThatProxy() server.FailExecOps = 0; // EXEC commits; one queued op errors at execution time // allow accumulating so the INCR doesn't gate retries - though nothing here retries anyway (EXEC commits) - var policy = new RetryPolicy + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, @@ -331,12 +331,12 @@ public async Task WithRetry_Transaction_FailsOverBetweenGroups() new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target ]; - var options = new MultiGroupOptions + MultiGroupOptions options = new MultiGroupOptions.Builder { - HealthCheck = new HealthCheck + HealthCheckInterval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us + HealthCheck = new HealthCheck.Builder { Probe = probe, - Interval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us ProbeCount = 1, ProbeTimeout = TimeSpan.FromSeconds(5), }, @@ -346,7 +346,7 @@ public async Task WithRetry_Transaction_FailsOverBetweenGroups() Assert.True(conn.IsConnected); Assert.Same(members[0], conn.ActiveMember); // A is active (highest weight) - var policy = new RetryPolicy + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 20, MaxAttemptsBeforeFailover = 1,