| | | 1 | | using Elsa.Common; |
| | | 2 | | using Elsa.Hosting.Management.Notifications; |
| | | 3 | | using Elsa.Hosting.Management.Options; |
| | | 4 | | using Elsa.KeyValues.Contracts; |
| | | 5 | | using Elsa.KeyValues.Models; |
| | | 6 | | using Elsa.Mediator.Contracts; |
| | | 7 | | using JetBrains.Annotations; |
| | | 8 | | using Medallion.Threading; |
| | | 9 | | using Microsoft.Extensions.DependencyInjection; |
| | | 10 | | using Microsoft.Extensions.Hosting; |
| | | 11 | | using Microsoft.Extensions.Options; |
| | | 12 | | |
| | | 13 | | namespace Elsa.Hosting.Management.HostedServices; |
| | | 14 | | |
| | | 15 | | /// <summary> |
| | | 16 | | /// Service to check the heartbeats of all running instances and determine whether instances have stopped working. |
| | | 17 | | /// </summary> |
| | | 18 | | [UsedImplicitly] |
| | | 19 | | public class InstanceHeartbeatMonitorService : IHostedService, IDisposable |
| | | 20 | | { |
| | | 21 | | private readonly IServiceProvider _serviceProvider; |
| | | 22 | | private readonly HeartbeatOptions _heartbeatOptions; |
| | | 23 | | private Timer? _timer; |
| | | 24 | | |
| | | 25 | | /// <summary> |
| | | 26 | | /// Creates a new instance of the <see cref="InstanceHeartbeatMonitorService"/> |
| | | 27 | | /// </summary> |
| | 0 | 28 | | public InstanceHeartbeatMonitorService(IServiceProvider serviceProvider, IOptions<HeartbeatOptions> heartbeatOptions |
| | | 29 | | { |
| | 0 | 30 | | _serviceProvider = serviceProvider; |
| | 0 | 31 | | _heartbeatOptions = heartbeatOptions.Value; |
| | 0 | 32 | | } |
| | | 33 | | |
| | | 34 | | /// <inheritdoc /> |
| | | 35 | | public Task StartAsync(CancellationToken cancellationToken) |
| | | 36 | | { |
| | 0 | 37 | | _timer = new Timer(MonitorHeartbeats, null, TimeSpan.Zero, _heartbeatOptions.Interval); |
| | 0 | 38 | | return Task.CompletedTask; |
| | | 39 | | } |
| | | 40 | | |
| | | 41 | | /// <inheritdoc /> |
| | | 42 | | public Task StopAsync(CancellationToken cancellationToken) |
| | | 43 | | { |
| | 0 | 44 | | _timer?.Change(Timeout.Infinite, Timeout.Infinite); |
| | 0 | 45 | | return Task.CompletedTask; |
| | | 46 | | } |
| | | 47 | | |
| | | 48 | | /// <inheritdoc /> |
| | | 49 | | public void Dispose() |
| | | 50 | | { |
| | 0 | 51 | | _timer?.Dispose(); |
| | 0 | 52 | | } |
| | | 53 | | |
| | | 54 | | private void MonitorHeartbeats(object? state) |
| | | 55 | | { |
| | 0 | 56 | | _ = Task.Run(async () => await MonitorHeartbeatsAsync()); |
| | 0 | 57 | | } |
| | | 58 | | |
| | | 59 | | private async Task MonitorHeartbeatsAsync() |
| | | 60 | | { |
| | 0 | 61 | | using var scope = _serviceProvider.CreateScope(); |
| | 0 | 62 | | var lockProvider = scope.ServiceProvider.GetRequiredService<IDistributedLockProvider>(); |
| | 0 | 63 | | var store = scope.ServiceProvider.GetRequiredService<IKeyValueStore>(); |
| | 0 | 64 | | var notificationSender = scope.ServiceProvider.GetRequiredService<INotificationSender>(); |
| | 0 | 65 | | var systemClock = scope.ServiceProvider.GetRequiredService<ISystemClock>(); |
| | | 66 | | |
| | | 67 | | const string lockKey = "InstanceHeartbeatMonitorService"; |
| | 0 | 68 | | await using var monitorLock = await lockProvider.TryAcquireLockAsync(lockKey, TimeSpan.Zero); |
| | 0 | 69 | | if (monitorLock == null) |
| | | 70 | | return; |
| | | 71 | | |
| | 0 | 72 | | var filter = new KeyValueFilter |
| | 0 | 73 | | { |
| | 0 | 74 | | StartsWith = true, |
| | 0 | 75 | | Key = InstanceHeartbeatService.HeartbeatKeyPrefix |
| | 0 | 76 | | }; |
| | 0 | 77 | | var heartbeats = await store.FindManyAsync(filter, default); |
| | | 78 | | |
| | 0 | 79 | | foreach (var heartbeat in heartbeats) |
| | | 80 | | { |
| | 0 | 81 | | var lastHeartbeat = DateTimeOffset.Parse(heartbeat.SerializedValue).UtcDateTime; |
| | | 82 | | |
| | 0 | 83 | | if (systemClock.UtcNow - lastHeartbeat <= _heartbeatOptions.Timeout) |
| | | 84 | | continue; |
| | | 85 | | |
| | 0 | 86 | | var instanceName = heartbeat.Key[InstanceHeartbeatService.HeartbeatKeyPrefix.Length..]; |
| | 0 | 87 | | await notificationSender.SendAsync(new HeartbeatTimedOut(instanceName)); |
| | 0 | 88 | | await store.DeleteAsync(heartbeat.Key, default); |
| | 0 | 89 | | } |
| | 0 | 90 | | } |
| | | 91 | | } |