| | | 1 | | using System.Diagnostics; |
| | | 2 | | using System.Diagnostics.Metrics; |
| | | 3 | | |
| | | 4 | | using DiagnosticsActivity = System.Diagnostics.Activity; |
| | | 5 | | using DiagnosticsActivityKind = System.Diagnostics.ActivityKind; |
| | | 6 | | using WorkflowActivityStatus = Elsa.Workflows.ActivityStatus; |
| | | 7 | | using WorkflowInstanceStatus = Elsa.Workflows.WorkflowStatus; |
| | | 8 | | using WorkflowInstanceSubStatus = Elsa.Workflows.WorkflowSubStatus; |
| | | 9 | | |
| | | 10 | | namespace Elsa.Workflows.Telemetry; |
| | | 11 | | |
| | | 12 | | /// <summary> |
| | | 13 | | /// Provides tracing and metrics instrumentation for workflow and activity execution. |
| | | 14 | | /// </summary> |
| | | 15 | | public static class WorkflowInstrumentation |
| | | 16 | | { |
| | | 17 | | public const string ActivitySourceName = "Elsa.Workflows"; |
| | | 18 | | public const string MeterName = "Elsa.Workflows"; |
| | | 19 | | |
| | | 20 | | public const string WorkflowSystem = "workflow.system"; |
| | | 21 | | public const string WorkflowOperationName = "workflow.operation.name"; |
| | | 22 | | public const string WorkflowName = "workflow.name"; |
| | | 23 | | public const string WorkflowInstanceId = "workflow.instance.id"; |
| | | 24 | | public const string WorkflowDefinitionId = "workflow.definition.id"; |
| | | 25 | | public const string WorkflowDefinitionVersion = "workflow.definition.version"; |
| | | 26 | | public const string WorkflowDefinitionVersionId = "workflow.definition.version.id"; |
| | | 27 | | public const string WorkflowStatus = "workflow.status"; |
| | | 28 | | public const string WorkflowSubStatus = "workflow.substatus"; |
| | | 29 | | public const string WorkflowFaulted = "workflow.faulted"; |
| | | 30 | | public const string WorkflowParentInstanceId = "workflow.parent.instance.id"; |
| | | 31 | | public const string WorkflowCorrelationId = "workflow.correlation.id"; |
| | | 32 | | |
| | | 33 | | public const string ActivityOperationName = "workflow.activity.operation.name"; |
| | | 34 | | public const string ActivityId = "workflow.activity.id"; |
| | | 35 | | public const string ActivityName = "workflow.activity.name"; |
| | | 36 | | public const string ActivityType = "workflow.activity.type"; |
| | | 37 | | public const string ActivityVersion = "workflow.activity.version"; |
| | | 38 | | public const string ActivityExecutionId = "workflow.activity.execution.id"; |
| | | 39 | | public const string ActivityStatus = "workflow.activity.status"; |
| | | 40 | | public const string ActivityOutcome = "workflow.activity.outcome"; |
| | | 41 | | public const string ActivityParentExecutionId = "workflow.activity.parent.execution.id"; |
| | | 42 | | public const string ActivityScheduledByExecutionId = "workflow.activity.scheduled.by.execution.id"; |
| | | 43 | | public const string ActivityFaulted = "workflow.activity.faulted"; |
| | | 44 | | |
| | | 45 | | public const string TenantId = "elsa.tenant.id"; |
| | | 46 | | public const string ExceptionType = "exception.type"; |
| | | 47 | | |
| | | 48 | | [Obsolete("Use ExceptionType. Workflow spans follow OpenTelemetry exception semantic conventions.")] |
| | | 49 | | public const string ErrorType = ExceptionType; |
| | | 50 | | |
| | | 51 | | private const string SystemName = "elsa"; |
| | | 52 | | private const string WorkflowExecuteOperation = "workflow.execute"; |
| | | 53 | | private const string ActivityExecuteOperation = "activity.execute"; |
| | | 54 | | |
| | 4 | 55 | | private static readonly string? Version = typeof(WorkflowInstrumentation).Assembly.GetName().Version?.ToString(); |
| | 4 | 56 | | private static readonly ActivitySource Source = new(ActivitySourceName, Version); |
| | 4 | 57 | | private static readonly Meter Meter = new(MeterName, Version); |
| | 4 | 58 | | private static readonly Counter<long> WorkflowStartedCounter = Meter.CreateCounter<long>("elsa.workflow.started", de |
| | 4 | 59 | | private static readonly Counter<long> WorkflowCompletedCounter = Meter.CreateCounter<long>("elsa.workflow.completed" |
| | 4 | 60 | | private static readonly Counter<long> WorkflowFaultedCounter = Meter.CreateCounter<long>("elsa.workflow.faulted", de |
| | 4 | 61 | | private static readonly Histogram<double> ActivityDuration = Meter.CreateHistogram<double>("elsa.activity.duration", |
| | | 62 | | |
| | | 63 | | internal static WorkflowInstrumentationScope StartWorkflow(WorkflowExecutionContext context, bool? isStarting = null |
| | | 64 | | { |
| | 509 | 65 | | var shouldRecordStarted = isStarting ?? context.SubStatus == WorkflowInstanceSubStatus.Pending; |
| | 509 | 66 | | var activity = Source.StartActivity(WorkflowExecuteOperation, DiagnosticsActivityKind.Internal); |
| | | 67 | | |
| | 509 | 68 | | if (activity != null) |
| | | 69 | | { |
| | 8 | 70 | | SetWorkflowTags(activity, context); |
| | 8 | 71 | | activity.SetTag(WorkflowOperationName, WorkflowExecuteOperation); |
| | | 72 | | } |
| | | 73 | | |
| | 509 | 74 | | if (shouldRecordStarted && WorkflowStartedCounter.Enabled) |
| | 7 | 75 | | WorkflowStartedCounter.Add(1, CreateWorkflowTags(context, false)); |
| | | 76 | | |
| | 509 | 77 | | return new WorkflowInstrumentationScope(activity); |
| | | 78 | | } |
| | | 79 | | |
| | | 80 | | internal static ActivityInstrumentationScope StartActivity(ActivityExecutionContext context) |
| | | 81 | | { |
| | 3306 | 82 | | var activity = Source.StartActivity(ActivityExecuteOperation, DiagnosticsActivityKind.Internal); |
| | | 83 | | |
| | 3306 | 84 | | if (activity != null) |
| | | 85 | | { |
| | 6 | 86 | | SetWorkflowTags(activity, context.WorkflowExecutionContext); |
| | 6 | 87 | | SetActivityTags(activity, context); |
| | 6 | 88 | | activity.SetTag(ActivityOperationName, ActivityExecuteOperation); |
| | | 89 | | } |
| | | 90 | | |
| | 3306 | 91 | | return new ActivityInstrumentationScope(activity, Stopwatch.GetTimestamp()); |
| | | 92 | | } |
| | | 93 | | |
| | | 94 | | internal static void StopWorkflow(WorkflowInstrumentationScope scope, WorkflowExecutionContext context, Exception? e |
| | | 95 | | { |
| | 509 | 96 | | var activity = scope.Activity; |
| | 509 | 97 | | var workflowException = exception ?? (context.SubStatus == WorkflowInstanceSubStatus.Faulted ? context.Exception |
| | 509 | 98 | | var cancelled = exception is OperationCanceledException || (exception == null && context.SubStatus == WorkflowIn |
| | 509 | 99 | | var faulted = !cancelled && (workflowException != null || context.SubStatus == WorkflowInstanceSubStatus.Faulted |
| | 509 | 100 | | var workflowSubStatus = cancelled ? WorkflowInstanceSubStatus.Cancelled : faulted ? WorkflowInstanceSubStatus.Fa |
| | | 101 | | |
| | 509 | 102 | | if (activity != null) |
| | | 103 | | { |
| | 8 | 104 | | SetWorkflowTags(activity, context, workflowSubStatus); |
| | 8 | 105 | | activity.SetTag(WorkflowFaulted, faulted); |
| | 8 | 106 | | SetError(activity, workflowException, faulted); |
| | 8 | 107 | | activity.Dispose(); |
| | | 108 | | } |
| | | 109 | | |
| | 509 | 110 | | if (faulted && WorkflowFaultedCounter.Enabled) |
| | 3 | 111 | | WorkflowFaultedCounter.Add(1, CreateWorkflowTags(context, workflowSubStatusOverride: workflowSubStatus)); |
| | 506 | 112 | | else if (context.SubStatus == WorkflowInstanceSubStatus.Finished && WorkflowCompletedCounter.Enabled) |
| | 1 | 113 | | WorkflowCompletedCounter.Add(1, CreateWorkflowTags(context)); |
| | 506 | 114 | | } |
| | | 115 | | |
| | | 116 | | internal static void StopActivity(ActivityInstrumentationScope scope, ActivityExecutionContext context, Exception? e |
| | | 117 | | { |
| | 3306 | 118 | | var activity = scope.Activity; |
| | 3306 | 119 | | var cancelled = exception is OperationCanceledException || (exception == null && context.Status == WorkflowActiv |
| | 3306 | 120 | | var faulted = !cancelled && (exception != null || context.Status == WorkflowActivityStatus.Faulted); |
| | 3306 | 121 | | var activityStatus = cancelled ? WorkflowActivityStatus.Canceled : context.Status; |
| | 3306 | 122 | | var duration = Stopwatch.GetElapsedTime(scope.StartTimestamp).TotalSeconds; |
| | | 123 | | |
| | 3306 | 124 | | if (ActivityDuration.Enabled) |
| | 6 | 125 | | ActivityDuration.Record(duration, CreateActivityTags(context, activityStatus, faulted)); |
| | | 126 | | |
| | 3306 | 127 | | if (activity != null) |
| | | 128 | | { |
| | 6 | 129 | | SetActivityTags(activity, context, activityStatus); |
| | 6 | 130 | | activity.SetTag(ActivityFaulted, faulted); |
| | 6 | 131 | | if (faulted) |
| | 3 | 132 | | activity.SetTag(ActivityStatus, WorkflowActivityStatus.Faulted.ToString()); |
| | 6 | 133 | | SetActivityOutcome(activity, context); |
| | 6 | 134 | | SetError(activity, exception ?? context.Exception, faulted); |
| | 6 | 135 | | activity.Dispose(); |
| | | 136 | | } |
| | 3306 | 137 | | } |
| | | 138 | | |
| | | 139 | | private static void SetWorkflowTags(DiagnosticsActivity activity, WorkflowExecutionContext context, WorkflowInstance |
| | | 140 | | { |
| | 22 | 141 | | var workflow = context.Workflow; |
| | 22 | 142 | | var identity = workflow.Identity; |
| | 22 | 143 | | var workflowSubStatus = workflowSubStatusOverride ?? context.SubStatus; |
| | 22 | 144 | | var workflowStatus = GetWorkflowStatus(context, workflowSubStatusOverride); |
| | | 145 | | |
| | 22 | 146 | | activity.SetTag(WorkflowSystem, SystemName); |
| | 22 | 147 | | activity.SetTag(WorkflowInstanceId, context.Id); |
| | 22 | 148 | | activity.SetTag(WorkflowDefinitionId, identity.DefinitionId); |
| | 22 | 149 | | activity.SetTag(WorkflowDefinitionVersion, identity.Version); |
| | 22 | 150 | | activity.SetTag(WorkflowDefinitionVersionId, identity.Id); |
| | 22 | 151 | | activity.SetTag(WorkflowStatus, workflowStatus.ToString()); |
| | 22 | 152 | | activity.SetTag(WorkflowSubStatus, workflowSubStatus.ToString()); |
| | 22 | 153 | | AddIfNotNull(activity, WorkflowName, workflow.WorkflowMetadata.Name ?? workflow.Name); |
| | 22 | 154 | | AddIfNotNull(activity, WorkflowParentInstanceId, context.ParentWorkflowInstanceId); |
| | 22 | 155 | | AddIfNotNull(activity, WorkflowCorrelationId, context.CorrelationId); |
| | 22 | 156 | | AddIfNotNull(activity, TenantId, identity.TenantId); |
| | 22 | 157 | | } |
| | | 158 | | |
| | | 159 | | private static void SetActivityTags(DiagnosticsActivity activity, ActivityExecutionContext context, WorkflowActivity |
| | | 160 | | { |
| | 12 | 161 | | var currentActivity = context.Activity; |
| | 12 | 162 | | var activityStatus = activityStatusOverride ?? context.Status; |
| | | 163 | | |
| | 12 | 164 | | activity.SetTag(ActivityId, currentActivity.Id); |
| | 12 | 165 | | activity.SetTag(ActivityType, currentActivity.Type); |
| | 12 | 166 | | activity.SetTag(ActivityVersion, currentActivity.Version); |
| | 12 | 167 | | activity.SetTag(ActivityExecutionId, context.Id); |
| | 12 | 168 | | activity.SetTag(ActivityStatus, activityStatus.ToString()); |
| | 12 | 169 | | AddIfNotNull(activity, ActivityName, currentActivity.Name ?? context.ActivityDescriptor.DisplayName ?? context.A |
| | 12 | 170 | | AddIfNotNull(activity, ActivityParentExecutionId, context.ParentActivityExecutionContext?.Id); |
| | 12 | 171 | | AddIfNotNull(activity, ActivityScheduledByExecutionId, context.SchedulingActivityExecutionId); |
| | 12 | 172 | | } |
| | | 173 | | |
| | | 174 | | private static void SetActivityOutcome(DiagnosticsActivity activity, ActivityExecutionContext context) |
| | | 175 | | { |
| | 6 | 176 | | if (!context.JournalData.TryGetValue("Outcomes", out var outcomes)) |
| | 5 | 177 | | return; |
| | | 178 | | |
| | 1 | 179 | | var outcome = outcomes switch |
| | 1 | 180 | | { |
| | 0 | 181 | | IEnumerable<string> names => string.Join(",", names), |
| | 1 | 182 | | _ => outcomes?.ToString() |
| | 1 | 183 | | }; |
| | | 184 | | |
| | 1 | 185 | | AddIfNotNull(activity, ActivityOutcome, outcome); |
| | 1 | 186 | | } |
| | | 187 | | |
| | | 188 | | private static void SetError(DiagnosticsActivity activity, Exception? exception, bool faulted) |
| | | 189 | | { |
| | 14 | 190 | | if (!faulted) |
| | | 191 | | { |
| | 8 | 192 | | activity.SetStatus(ActivityStatusCode.Ok); |
| | 8 | 193 | | return; |
| | | 194 | | } |
| | | 195 | | |
| | 6 | 196 | | activity.SetStatus(ActivityStatusCode.Error, "Faulted"); |
| | | 197 | | |
| | 6 | 198 | | if (exception != null) |
| | | 199 | | { |
| | 5 | 200 | | var exceptionType = exception.GetType().FullName; |
| | 5 | 201 | | activity.SetTag(ExceptionType, exceptionType); |
| | 5 | 202 | | activity.AddEvent(new("exception", tags: new ActivityTagsCollection |
| | 5 | 203 | | { |
| | 5 | 204 | | { ExceptionType, exceptionType } |
| | 5 | 205 | | })); |
| | | 206 | | } |
| | 6 | 207 | | } |
| | | 208 | | |
| | | 209 | | private static TagList CreateWorkflowTags(WorkflowExecutionContext context, bool includeExecutionStatus = true, Work |
| | | 210 | | { |
| | 11 | 211 | | var workflow = context.Workflow; |
| | 11 | 212 | | var identity = workflow.Identity; |
| | 11 | 213 | | var workflowSubStatus = workflowSubStatusOverride ?? context.SubStatus; |
| | 11 | 214 | | var workflowStatus = GetWorkflowStatus(context, workflowSubStatusOverride); |
| | 11 | 215 | | var tags = new TagList |
| | 11 | 216 | | { |
| | 11 | 217 | | { WorkflowSystem, SystemName }, |
| | 11 | 218 | | { WorkflowDefinitionId, identity.DefinitionId }, |
| | 11 | 219 | | { WorkflowDefinitionVersion, identity.Version } |
| | 11 | 220 | | }; |
| | | 221 | | |
| | 11 | 222 | | if (includeExecutionStatus) |
| | | 223 | | { |
| | 4 | 224 | | tags.Add(WorkflowStatus, workflowStatus.ToString()); |
| | 4 | 225 | | tags.Add(WorkflowSubStatus, workflowSubStatus.ToString()); |
| | | 226 | | } |
| | | 227 | | |
| | 11 | 228 | | AddIfNotNull(ref tags, WorkflowName, workflow.WorkflowMetadata.Name ?? workflow.Name); |
| | 11 | 229 | | AddIfNotNull(ref tags, TenantId, identity.TenantId); |
| | 11 | 230 | | return tags; |
| | | 231 | | } |
| | | 232 | | |
| | | 233 | | private static TagList CreateActivityTags(ActivityExecutionContext context, WorkflowActivityStatus activityStatus, b |
| | | 234 | | { |
| | 6 | 235 | | var currentActivity = context.Activity; |
| | 6 | 236 | | var tags = new TagList |
| | 6 | 237 | | { |
| | 6 | 238 | | { WorkflowSystem, SystemName }, |
| | 6 | 239 | | { WorkflowDefinitionId, context.WorkflowExecutionContext.Workflow.Identity.DefinitionId }, |
| | 6 | 240 | | { ActivityType, currentActivity.Type }, |
| | 6 | 241 | | { ActivityVersion, currentActivity.Version }, |
| | 6 | 242 | | { ActivityStatus, faulted ? WorkflowActivityStatus.Faulted.ToString() : activityStatus.ToString() }, |
| | 6 | 243 | | { ActivityFaulted, faulted } |
| | 6 | 244 | | }; |
| | | 245 | | |
| | 6 | 246 | | AddIfNotNull(ref tags, TenantId, context.WorkflowExecutionContext.Workflow.Identity.TenantId); |
| | 6 | 247 | | return tags; |
| | | 248 | | } |
| | | 249 | | |
| | | 250 | | private static WorkflowInstanceStatus GetWorkflowStatus(WorkflowExecutionContext context, WorkflowInstanceSubStatus? |
| | | 251 | | { |
| | 33 | 252 | | if (workflowSubStatusOverride == null) |
| | 23 | 253 | | return context.Status; |
| | | 254 | | |
| | 10 | 255 | | return workflowSubStatusOverride.Value switch |
| | 10 | 256 | | { |
| | 10 | 257 | | WorkflowInstanceSubStatus.Cancelled or WorkflowInstanceSubStatus.Faulted or WorkflowInstanceSubStatus.Finish |
| | 0 | 258 | | _ => WorkflowInstanceStatus.Running |
| | 10 | 259 | | }; |
| | | 260 | | } |
| | | 261 | | |
| | | 262 | | private static void AddIfNotNull(ref TagList tags, string key, object? value) |
| | | 263 | | { |
| | 28 | 264 | | if (ShouldAddTag(value)) |
| | 4 | 265 | | tags.Add(key, value!); |
| | 28 | 266 | | } |
| | | 267 | | |
| | | 268 | | private static void AddIfNotNull(DiagnosticsActivity activity, string key, object? value) |
| | | 269 | | { |
| | 125 | 270 | | if (ShouldAddTag(value)) |
| | 18 | 271 | | activity.SetTag(key, value); |
| | 125 | 272 | | } |
| | | 273 | | |
| | | 274 | | private static bool ShouldAddTag(object? value) |
| | | 275 | | { |
| | 153 | 276 | | return value is not null && (value is not string stringValue || !string.IsNullOrWhiteSpace(stringValue)); |
| | | 277 | | } |
| | | 278 | | } |
| | | 279 | | |
| | | 280 | | internal readonly record struct WorkflowInstrumentationScope(DiagnosticsActivity? Activity); |
| | | 281 | | |
| | | 282 | | internal readonly record struct ActivityInstrumentationScope(DiagnosticsActivity? Activity, long StartTimestamp); |