| | | 1 | | using Elsa.Common.Entities; |
| | | 2 | | using Elsa.Common.Models; |
| | | 3 | | using Elsa.Extensions; |
| | | 4 | | using Elsa.Common; |
| | | 5 | | using Elsa.Workflows.Runtime.Entities; |
| | | 6 | | using Elsa.Workflows.Runtime.Messages; |
| | | 7 | | using Elsa.Workflows.Runtime.Options; |
| | | 8 | | using Elsa.Workflows.Runtime.OrderDefinitions; |
| | | 9 | | using Microsoft.Extensions.Logging; |
| | | 10 | | using Microsoft.Extensions.Options; |
| | | 11 | | |
| | | 12 | | namespace Elsa.Workflows.Runtime; |
| | | 13 | | |
| | 335 | 14 | | public class BookmarkQueueProcessor( |
| | 335 | 15 | | IBookmarkQueueStore store, |
| | 335 | 16 | | IBookmarkQueueDeadLetterManager deadLetterManager, |
| | 335 | 17 | | IWorkflowResumer workflowResumer, |
| | 335 | 18 | | ISystemClock systemClock, |
| | 335 | 19 | | IOptions<BookmarkQueuePurgeOptions> options, |
| | 335 | 20 | | ILogger<BookmarkQueueProcessor> logger) : IBookmarkQueueProcessor |
| | | 21 | | { |
| | | 22 | | public async Task ProcessAsync(CancellationToken cancellationToken = default) |
| | | 23 | | { |
| | 336 | 24 | | var batchSize = 50; |
| | 336 | 25 | | var offset = 0; |
| | | 26 | | |
| | 337 | 27 | | while (!cancellationToken.IsCancellationRequested) |
| | | 28 | | { |
| | 336 | 29 | | var pageArgs = PageArgs.FromRange(offset, batchSize); |
| | 336 | 30 | | var page = await store.PageAsync(pageArgs, new BookmarkQueueItemOrder<DateTimeOffset>(x => x.CreatedAt, Orde |
| | 336 | 31 | | var retainedCount = await ProcessPageAsync(page, cancellationToken); |
| | | 32 | | |
| | 335 | 33 | | if (page.Items.Count < batchSize) |
| | | 34 | | break; |
| | | 35 | | |
| | 1 | 36 | | offset += retainedCount; |
| | 1 | 37 | | } |
| | 335 | 38 | | } |
| | | 39 | | |
| | | 40 | | private async Task<int> ProcessPageAsync(Page<BookmarkQueueItem> page, CancellationToken cancellationToken = default |
| | | 41 | | { |
| | 336 | 42 | | var retainedCount = 0; |
| | | 43 | | |
| | 13633 | 44 | | foreach (var bookmarkQueueItem in page.Items) |
| | | 45 | | { |
| | 6481 | 46 | | if (await ProcessItemAsync(bookmarkQueueItem, cancellationToken)) |
| | 6381 | 47 | | retainedCount++; |
| | | 48 | | } |
| | | 49 | | |
| | 335 | 50 | | return retainedCount; |
| | 335 | 51 | | } |
| | | 52 | | |
| | | 53 | | private async Task<bool> ProcessItemAsync(BookmarkQueueItem item, CancellationToken cancellationToken = default) |
| | | 54 | | { |
| | 6481 | 55 | | var filter = item.CreateBookmarkFilter(); |
| | 6481 | 56 | | var resumeOptions = item.Options; |
| | | 57 | | |
| | 6481 | 58 | | logger.LogDebug("Processing bookmark queue item {BookmarkQueueItemId} for workflow instance {WorkflowInstanceId} |
| | | 59 | | |
| | | 60 | | List<RunWorkflowInstanceResponse> responses; |
| | | 61 | | |
| | | 62 | | try |
| | | 63 | | { |
| | 6481 | 64 | | responses = (await workflowResumer.ResumeAsync(filter, resumeOptions, cancellationToken)).ToList(); |
| | 6477 | 65 | | } |
| | 1 | 66 | | catch (OperationCanceledException) |
| | | 67 | | { |
| | 1 | 68 | | throw; |
| | | 69 | | } |
| | 3 | 70 | | catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) |
| | | 71 | | { |
| | 3 | 72 | | return await HandleFailureAsync(item, ex, cancellationToken); |
| | | 73 | | } |
| | | 74 | | |
| | 6477 | 75 | | if (responses.Count > 0) |
| | | 76 | | { |
| | 97 | 77 | | logger.LogDebug("Successfully resumed {WorkflowCount} workflow instances using stimulus {StimulusHash} for a |
| | 97 | 78 | | await store.DeleteAsync(item.Id, cancellationToken); |
| | 97 | 79 | | return false; |
| | | 80 | | } |
| | | 81 | | |
| | 6380 | 82 | | logger.LogDebug("No matching bookmarks found for bookmark queue item {BookmarkQueueItemId} for workflow instance |
| | 6380 | 83 | | return true; |
| | 6480 | 84 | | } |
| | | 85 | | |
| | | 86 | | private async Task<bool> HandleFailureAsync(BookmarkQueueItem item, Exception exception, CancellationToken cancellat |
| | | 87 | | { |
| | 3 | 88 | | item.DeliveryAttempts++; |
| | 3 | 89 | | item.LastAttemptedAt = systemClock.UtcNow; |
| | 3 | 90 | | item.LastErrorType = exception.GetType().FullName; |
| | 3 | 91 | | item.LastErrorMessage = exception.Message; |
| | | 92 | | |
| | 3 | 93 | | if (item.DeliveryAttempts < options.Value.MaxDeliveryAttempts) |
| | | 94 | | { |
| | 1 | 95 | | logger.LogWarning( |
| | 1 | 96 | | exception, |
| | 1 | 97 | | "Failed to process bookmark queue item {BookmarkQueueItemId}. Attempt {DeliveryAttempt} of {MaxDeliveryA |
| | 1 | 98 | | item.Id, |
| | 1 | 99 | | item.DeliveryAttempts, |
| | 1 | 100 | | options.Value.MaxDeliveryAttempts); |
| | | 101 | | |
| | 1 | 102 | | await store.SaveAsync(item, cancellationToken); |
| | 1 | 103 | | return true; |
| | | 104 | | } |
| | | 105 | | |
| | 2 | 106 | | logger.LogError( |
| | 2 | 107 | | exception, |
| | 2 | 108 | | "Moving bookmark queue item {BookmarkQueueItemId} to dead letter after {DeliveryAttempt} failed delivery att |
| | 2 | 109 | | item.Id, |
| | 2 | 110 | | item.DeliveryAttempts); |
| | | 111 | | |
| | 2 | 112 | | await deadLetterManager.DeadLetterAsync(item, "Failed", exception, cancellationToken); |
| | 2 | 113 | | await store.DeleteAsync(item.Id, cancellationToken); |
| | 2 | 114 | | return false; |
| | 3 | 115 | | } |
| | | 116 | | } |