Bug 1940906 - Stream audio in smaller blocks; implement chunked downmixing. r=media-playback-reviewers,alwu

Send audio to the inference process in ~40ms blocks instead of 0.5s, so
end-to-end latency is bounded by the streaming model (~180ms) rather than IPC
batching. Implement the chunked downmixing path in DataCallback (slice-wise,
RT-safe) instead of crashing, and stop clearing the graph-thread scratch
buffer from the main thread in Stop() (a data race the lower latency exposed).

Count the amount of frame dropped, increase ringbuffer depth to smooth
over load spike events.

Differential Revision: https://phabricator.services.mozilla.com/D311103

Author: padenot

dom/media/webspeech/recognition/SpeechRecognitionBackend.cpp

@@ -9,6 +9,7 @@
 #include <speex/speex_resampler.h>
 
 #include <algorithm>
+#include <utility>
 
 #include "AudibilityMonitor.h"
 #include "AudioConfig.h"
@@ -150,14 +151,21 @@ SpeechRecognitionIPCActorUserGuard::~SpeechRecognitionIPCActorUserGuard() {
   }
 }
 
-static constexpr double IPC_BLOCK_SIZE_S = 0.5;
+// Audio is streamed to the inference process in small blocks to keep end-to-end
+// latency low. The block is well under the model's internal chunk (~80 ms), so
+// the buffering delay is hidden behind the model's own latency rather than
+// adding to it.
+static constexpr double IPC_BLOCK_SIZE_S = 0.04;
+// Depth costs no latency (the whole buffer is drained every poll), but having
+// this significantly larger than the block size allows not dropping audio
+// during long stalls under load -- some of the threads used to convey the audio
+// aren't real-time, and can be starved.
+static constexpr double RING_BUFFER_SIZE_S = 2.0;
 static constexpr uint32_t STREAMING_POLL_MS = 20;
 static constexpr int32_t SPEECH_RECOGNITION_TARGET_RATE = 16000;
 static constexpr auto SPEECH_RECOGNITION_ENGINE_ID = "parakeet-cpp"_ns;
-// The ring buffer holds graph-rate audio, not yet downsampled to
-// SPEECH_RECOGNITION_TARGET_RATE, with this much headroom for the resampling
-// thread being late to dequeue.
-static constexpr uint32_t RING_BUFFER_IPC_BLOCKS = 4;
+// DataCallback() downmixes into mMonoBuffer in slices of at most this many
+// frames, so this is the only allocation the graph thread normally needs.
 static constexpr uint32_t PER_CALLBACK_MONO_BUFFER_INITIAL_NUM_FRAMES = 512;
 
 /* static */
@@ -192,8 +200,8 @@ SpeechRecognitionBackend::SpeechRecognitionBackend(
     : mParent(aParent),
       mLanguage(NS_ConvertUTF16toUTF8(aLanguage)),
       mPhrases(aPhrases.Clone()),
-      mRingBuffer(MakeUnique<SPSCQueue<float>>(AssertedCast<int>(
-          aGraphRate * IPC_BLOCK_SIZE_S * RING_BUFFER_IPC_BLOCKS))),
+      mRingBuffer(MakeUnique<SPSCQueue<float>>(
+          AssertedCast<int>(aGraphRate * RING_BUFFER_SIZE_S))),
       mResamplingThread(aResamplingThread),
       mResamplingCapability(aResamplingThread),
       mMonoBuffer(PER_CALLBACK_MONO_BUFFER_INITIAL_NUM_FRAMES),
@@ -414,36 +422,40 @@ void SpeechRecognitionBackend::DataCallback(MediaTrackGraph* aGraph,
     return;
   }
 
-  size_t frameCount = static_cast<size_t>(aChunk.mDuration);
+  const size_t frameCount = static_cast<size_t>(aChunk.mDuration);
   // A null chunk is silence the graph did not bother to materialize, not an
   // absence of audio, so it is fed as zeros rather than dropped. Dropping it
   // would splice together the audio on either side of a silent gap, hiding the
   // silence that ends an utterance from the recognizer.
   const bool isSilence = aChunk.IsNull();
 
-  if (mMonoBuffer.Capacity() < frameCount) {
-    LOGE("Warning: chunk size {} exceeds pre-allocated buffer capacity {}",
-         frameCount, mMonoBuffer.Capacity());
-    mMonoBuffer.SetCapacity(frameCount);
-    MOZ_DIAGNOSTIC_CRASH("Implement chunked downmixing");
-  }
-
-  mMonoBuffer.SetLengthAndRetainStorage(frameCount);
-
+  // Downmix to mono into the fixed-size scratch buffer and enqueue. A single
+  // graph chunk can be larger than the scratch buffer, so process it in slices
+  // that fit, avoiding any allocation on the real-time graph thread.
   AudioDataValue* monoData = mMonoBuffer.Elements();
   Span<AudioDataValue* const> outputChannels(&monoData, 1);
+  const size_t capacity = mMonoBuffer.Capacity();
 
-  if (isSilence) {
-    PodZero(mMonoBuffer.Elements(), frameCount);
-  } else {
-    aChunk.DownMixTo(outputChannels);
-  }
+  for (size_t offset = 0; offset < frameCount; offset += capacity) {
+    const size_t sliceFrames = std::min(capacity, frameCount - offset);
+    mMonoBuffer.SetLengthAndRetainStorage(sliceFrames);
 
-  int written = mRingBuffer->Enqueue(mMonoBuffer.Elements(),
-                                     AssertedCast<int>(frameCount));
+    if (isSilence) {
+      PodZero(mMonoBuffer.Elements(), sliceFrames);
+    } else {
+      AudioChunk slice = aChunk;
+      slice.SliceTo(offset, offset + sliceFrames);
+      slice.DownMixTo(outputChannels);
+    }
 
-  if (written < static_cast<int>(frameCount)) {
-    LOG("Ring buffer overflow: wrote {} of {} frames", written, frameCount);
+    int written = mRingBuffer->Enqueue(mMonoBuffer.Elements(),
+                                       AssertedCast<int>(sliceFrames));
+    if (written < static_cast<int>(sliceFrames)) {
+      mFramesDropped += sliceFrames - written;
+      LOGE("Capture ring buffer overflow: wrote {} of {} frames, {}s"
+           " dropped total", written, sliceFrames,
+           double(mFramesDropped) / mGraphRate);
+    }
   }
 }
 

dom/media/webspeech/recognition/SpeechRecognitionBackend.h

@@ -245,6 +245,9 @@ class SpeechRecognitionBackend {
   // DetachFromTrack() has stopped the callbacks.
   nsTArray<AudioDataValue> mMonoBuffer;
   const uint32_t mGraphRate;
+  // Graph-thread only, number of frames that couldn't be pushed into
+  // mRingBuffer and has been dropped.
+  uint64_t mFramesDropped = 0;
   // Whether Shutdown() has already run, so it runs exactly once.
   bool mStopped MOZ_GUARDED_BY(sMainThreadCapability) = false;
   // Whether a soundstart was fired without its soundend. The main thread owns