Best-effort teardown ownership release + 'resources check' detects missing schema (closes #3123)

src/Persistence/Wolverine.RDBMS/MessageDatabase.Admin.cs

@@ -181,6 +181,27 @@ public async Task CheckConnectivityAsync(CancellationToken token)
         await conn.CloseAsync();
     }
 
+    public async Task AssertStorageExistsAsync(CancellationToken token)
+    {
+        await using var conn = await DataSource.OpenConnectionAsync(token);
+        try
+        {
+            // A reachable database is not enough — the schema/tables can be missing (e.g. never
+            // provisioned, or dropped). Reuse the same Weasel diff the migration path uses so that
+            // 'resources check' reports an un-provisioned schema as unhealthy.
+            var migration = await SchemaMigration.DetermineAsync(conn, token, Objects);
+            if (migration.Difference != SchemaPatchDifference.None)
+            {
+                throw new InvalidOperationException(
+                    $"The Wolverine message storage for database '{Name}' is missing or out of date (schema difference: {migration.Difference}). Run 'resources setup' or enable auto-provisioning.");
+            }
+        }
+        finally
+        {
+            await conn.CloseAsync();
+        }
+    }
+
     private async Task migrateAsync(DbConnection conn)
     {
         var migration = await SchemaMigration.DetermineAsync(conn, _cancellation, Objects);

src/Testing/CoreTests/Persistence/release_all_ownership_is_best_effort.cs

@@ -0,0 +1,46 @@
+using CoreTests.Runtime;
+using NSubstitute;
+using Shouldly;
+using Wolverine.Persistence;
+using Wolverine.Persistence.Durability;
+using Xunit;
+
+namespace CoreTests.Persistence;
+
+// Regression for #3124's sibling teardown bug #3123: when teardown runs after a failed/partial
+// startup, an ancillary store whose schema was never created throws (e.g. PostgreSQL 42P01) from
+// ReleaseAllOwnershipAsync. That must not abort releasing the OTHER stores, nor surface as an
+// unhandled teardown exception that masks the original startup failure.
+public class release_all_ownership_is_best_effort
+{
+    private static IMessageStore StoreFor(string uri, MessageStoreRole role, IMessageStoreAdmin admin)
+    {
+        var store = Substitute.For<IMessageStore>();
+        store.Uri.Returns(new Uri(uri));
+        store.Role.Returns(role);
+        store.Admin.Returns(admin);
+        return store;
+    }
+
+    [Fact]
+    public async Task one_failing_store_does_not_stop_the_others_or_throw()
+    {
+        var failingAdmin = Substitute.For<IMessageStoreAdmin>();
+        failingAdmin.ReleaseAllOwnershipAsync(Arg.Any<int>())
+            .Returns(Task.FromException(new InvalidOperationException("42P01: relation does not exist")));
+
+        var healthyAdmin = Substitute.For<IMessageStoreAdmin>();
+        healthyAdmin.ReleaseAllOwnershipAsync(Arg.Any<int>()).Returns(Task.CompletedTask);
+
+        var failing = StoreFor("wolverinedb://fake/ancillary-without-schema", MessageStoreRole.Ancillary, failingAdmin);
+        var healthy = StoreFor("wolverinedb://fake/main", MessageStoreRole.Ancillary, healthyAdmin);
+
+        var collection = new MessageStoreCollection(new MockWolverineRuntime(), [failing, healthy], []);
+
+        // Must not throw even though one store's release fails
+        await Should.NotThrowAsync(() => collection.ReleaseAllOwnershipAsync(5));
+
+        // And the healthy store must still have had its ownership released
+        await healthyAdmin.Received().ReleaseAllOwnershipAsync(5);
+    }
+}

src/Wolverine/Persistence/Durability/IMessageStoreAdmin.cs

@@ -59,6 +59,14 @@ public interface IMessageStoreAdmin
     /// <returns></returns>
     public Task CheckConnectivityAsync(CancellationToken token);
 
+    /// <summary>
+    ///     Verify that the durable envelope storage objects (schema/tables) actually exist and match the
+    ///     configured schema, throwing when they do not. Used by <c>resources check</c> so a missing or
+    ///     un-provisioned schema is reported as unhealthy rather than passing on connectivity alone.
+    ///     The default is a no-op for stores that don't support schema introspection.
+    /// </summary>
+    Task AssertStorageExistsAsync(CancellationToken token) => Task.CompletedTask;
+
     /// <summary>
     ///     Apply any necessary database migrations to bring the underlying envelope
     ///     storage to the configured requirements of the Wolverine system

src/Wolverine/Persistence/Durability/MessageStoreResource.cs

@@ -18,9 +18,13 @@ public MessageStoreResource(WolverineOptions options, IMessageStore persistence)
     public Uri SubjectUri { get; }
     public Uri ResourceUri { get; }
 
-    public Task Check(CancellationToken token)
+    public async Task Check(CancellationToken token)
     {
-        return _persistence.Admin.CheckConnectivityAsync(token);
+        await _persistence.Admin.CheckConnectivityAsync(token);
+
+        // Connectivity alone isn't enough for "resources check" — verify the schema/tables actually
+        // exist so a missing or un-provisioned storage schema is reported as unhealthy.
+        await _persistence.Admin.AssertStorageExistsAsync(token);
     }
 
     public Task ClearState(CancellationToken token)

src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs

@@ -474,6 +474,11 @@ Task IMessageStoreAdmin.CheckConnectivityAsync(CancellationToken token)
         return executeOnAllAsync(d => d.Admin.CheckConnectivityAsync(token));
     }
 
+    Task IMessageStoreAdmin.AssertStorageExistsAsync(CancellationToken token)
+    {
+        return executeOnAllAsync(d => d.Admin.AssertStorageExistsAsync(token));
+    }
+
     async Task IMessageStoreAdmin.MigrateAsync()
     {
         if (!_initialized)

src/Wolverine/Persistence/MessageStoreCollection.cs

@@ -2,6 +2,7 @@
 using JasperFx.Core;
 using JasperFx.Core.Reflection;
 using JasperFx.Descriptors;
+using Microsoft.Extensions.Logging;
 using Wolverine.Persistence.Durability;
 using Wolverine.Persistence.Durability.DeadLetterManagement;
 using Wolverine.Runtime;
@@ -319,9 +320,25 @@ public bool TryFindMultiTenantedForMainStore(IMessageStore store, out MultiTenan
 
     public async Task ReleaseAllOwnershipAsync(int nodeNumber)
     {
-        foreach (var store in  _services.Enumerate().Select(x => x.Value))
+        // Best-effort, per store. This runs during teardown — including after a FAILED or partial
+        // startup, where an ancillary store's schema (e.g. wolverine_incoming_envelopes) may never
+        // have been created and the release UPDATE throws (PostgreSQL 42P01, etc.). Releasing
+        // ownership is itself optional: any envelopes left as owner_id = nodeNumber are reclaimed by
+        // the durability agent's recovery polling on the next live node. So a single failing store
+        // must not abort releasing the others, nor surface as an unhandled teardown error that masks
+        // the real startup failure. See GH-3123.
+        foreach (var store in _services.Enumerate().Select(x => x.Value))
         {
-            await store.Admin.ReleaseAllOwnershipAsync(nodeNumber);
+            try
+            {
+                await store.Admin.ReleaseAllOwnershipAsync(nodeNumber);
+            }
+            catch (Exception e)
+            {
+                _runtime.Logger.LogDebug(e,
+                    "Error while releasing node ownership for message store {Store} during teardown. This is safe to ignore; ownership is reclaimed by recovery polling.",
+                    store.Name);
+            }
         }
     }