From 5e96382025061231353b9292f5181644b5f37792 Mon Sep 17 00:00:00 2001 From: Max Holland Date: Fri, 13 Jan 2023 10:05:54 +0000 Subject: [PATCH] Transcode success rate metric fixes (#2684) * additonal monitoring to include these failure scenarios in success metric * add todo * Revert "add todo" This reverts commit abd13349d6c5791114a819e0f42889276ee227bf. * pending changelog --- CHANGELOG_PENDING.md | 1 + monitor/census.go | 3 +++ server/broadcast.go | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index d161e43c7..3762c4aa0 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -23,6 +23,7 @@ #### Broadcaster - \#2709 Add logging for high keyframe interval, reduce log level for discovery loop +- \#2684 Fix transcode success rate metric #### Orchestrator diff --git a/monitor/census.go b/monitor/census.go index afebcf18d..d6d0a7390 100644 --- a/monitor/census.go +++ b/monitor/census.go @@ -48,6 +48,9 @@ const ( SegmentTranscodeErrorSaveData SegmentTranscodeError = "SaveData" SegmentTranscodeErrorSessionEnded SegmentTranscodeError = "SessionEnded" SegmentTranscodeErrorDuplicateSegment SegmentTranscodeError = "DuplicateSegment" + SegmentTranscodeErrorMaxAttempts SegmentTranscodeError = "MaxAttempts" + SegmentTranscodeErrorNonRetryable SegmentTranscodeError = "NonRetryable" + SegmentTranscodeErrorCtxCancelled SegmentTranscodeError = "CtxCancelled" numberOfSegmentsToCalcAverage = 30 gweiConversionFactor = 1000000000 diff --git a/server/broadcast.go b/server/broadcast.go index f3f976a57..6207a03f1 100755 --- a/server/broadcast.go +++ b/server/broadcast.go @@ -923,11 +923,17 @@ func processSegment(ctx context.Context, cxn *rtmpConnection, seg *stream.HLSSeg } if isNonRetryableError(err) { clog.Warningf(ctx, "Not retrying current segment due to non-retryable error err=%q", err) + if monitor.Enabled { + monitor.SegmentTranscodeFailed(ctx, monitor.SegmentTranscodeErrorNonRetryable, nonce, seg.SeqNo, err, true) + } break } if ctxErr := ctx.Err(); ctxErr != nil { err = ctxErr clog.Warningf(ctx, "Not retrying current segment due to context cancellation err=%q", err) + if monitor.Enabled { + monitor.SegmentTranscodeFailed(ctx, monitor.SegmentTranscodeErrorCtxCancelled, nonce, seg.SeqNo, err, true) + } break } // recoverable error, retry @@ -950,6 +956,9 @@ func processSegment(ctx context.Context, cxn *rtmpConnection, seg *stream.HLSSeg } if len(attempts) == MaxAttempts && err != nil { err = fmt.Errorf("Hit max transcode attempts: %w", err) + if monitor.Enabled { + monitor.SegmentTranscodeFailed(ctx, monitor.SegmentTranscodeErrorMaxAttempts, nonce, seg.SeqNo, err, true) + } } return urls, err }