Skip to content

Commit 2020fc4

Browse files
committed
Bug 1940906 - Introduce SpeechRecognitionBackend - Part 2 - Audio side. r=pehrsons
This patch adds: - Getting audio from a MediaTrackGraph, downmixing it, queuing it to a ring buffer - Running a processing loop on a dedicated thread (called the resampling thread), that dequeues from the ring buffer, downsamples the audio to 16kHz (which is the rate at which the inference is running), and sends it over to the IPC thread, that will send it to the HWInference process in chunks of (by default) 500ms - Receive events from the HWInference process, and relay them to the main thread (notably speech recognition events, but also errors and such) object, SpeechRecognition It also hardens session lifecycle handling: Init() now returns a Web Speech error token instead of a bool so failures (e.g. "concurrent-session", "network") can be surfaced properly, the single-session slot is released when init fails, Stop()/Abort() teardown is made idempotent and covers the case where Stop() races with an in-flight session init (avoiding a leaked session slot), and stale backend callbacks are dropped if a newer backend has since replaced the current one. The SpeechRecognitionChild destroyed callback takes the destroyed actor as an argument: capturing it by raw pointer for an identity comparison trips the clang-plugin refcounted-inside-lambda checker, and a strong capture would keep the actor alive. Differential Revision: https://phabricator.services.mozilla.com/D275657
1 parent bbdc696 commit 2020fc4

9 files changed

Lines changed: 882 additions & 110 deletions

‎build/clang-plugin/ThreadAllows.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ UpdateProcessor
2323
# Used by `BackgroundClipboardContentAnalysisParent` to handle clipboard requests
2424
# from content processes.
2525
BkgrndClipboard
26+
# Used by `SpeechRecognitionBackend` to resample captured audio to the rate the
27+
# recognition engine runs at, off the real-time graph thread. Needs to be
28+
# a dedicated thread because some scheduling guarantees are necessary.
29+
SpeechResampler
2630
# Blocking thread calling into the parakeet.cpp library.
2731
Parakeet
2832
# Used by `SpeechRecognitionBackend` to send audio chunks and other commands to

‎dom/media/webspeech/recognition/PSpeechRecognition.ipdl‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,22 @@ parent: // HWInference
3636
// Install speech recognition models for the given languages
3737
async InstallModels(nsCString[] languages) returns (bool success);
3838
// Initialize the speech recognition session with a language and a list of
39-
// biasing phrases
40-
async Init(nsCString engineId, nsCString language, nsString[] phrases) returns (bool success);
39+
// biasing phrases. Resolves with an empty string on success, otherwise with
40+
// the Web Speech error token describing the failure (e.g.
41+
// "concurrent-session" when another session is active, "network" when the
42+
// model could not be retrieved or the engine failed to start).
43+
async Init(nsCString engineId, nsCString language, nsString[] phrases) returns (nsCString error);
4144
// Send audio data for speech recognition (always 16kHz mono f32 samples)
4245
async ProcessAudioData(float[] audioData);
43-
// Stop the speech recognition session
44-
async Stop();
46+
// Stop the speech recognition session.
47+
// https://webaudio.github.io/web-speech-api/#dom-speechrecognition-stop
48+
// "The speech service must attempt to return a recognition result (or a
49+
// nomatch) based on the audio that it has already collected for this
50+
// recognition." This therefore resolves only once the engine has flushed its
51+
// end-of-stream tail and sent every result that flush produced.
52+
// anyFinalResult is false when the session finalized nothing at all, which
53+
// is what makes the content process fire "nomatch" before "end".
54+
async Stop() returns (bool anyFinalResult);
4555

4656
child: // Content process
4757
// Speech recognition results

‎dom/media/webspeech/recognition/SpeechRecognition.cpp‎

Lines changed: 67 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
#include "VideoUtils.h"
1919
#include "mozilla/AbstractThread.h"
2020
#include "mozilla/ClearOnShutdown.h"
21-
#include "mozilla/ErrorNames.h"
2221
#include "mozilla/MediaManager.h"
2322
#include "mozilla/dom/AudioStreamTrack.h"
2423
#include "mozilla/dom/BindingUtils.h"
@@ -138,7 +137,7 @@ SpeechRecognition::~SpeechRecognition() {
138137

139138
// Ensure backend is properly cleaned up
140139
if (mBackend) {
141-
mBackend->Abort();
140+
mBackend->Abort(TrailingEvents::Skip);
142141
mBackend = nullptr;
143142
}
144143
}
@@ -151,7 +150,7 @@ JSObject* SpeechRecognition::WrapObject(JSContext* aCx,
151150
void SpeechRecognition::DisconnectFromOwner() {
152151
AssertIsOnMainThread();
153152
if (mBackend) {
154-
mBackend->Abort();
153+
mBackend->Abort(TrailingEvents::Skip);
155154
mBackend = nullptr;
156155
}
157156
Reset();
@@ -178,6 +177,8 @@ void SpeechRecognition::Reset() {
178177
}
179178
}
180179
mStarted = false;
180+
mStopping = false;
181+
mAborting = false;
181182
mBackendListening = false;
182183
mStartDispatched = false;
183184
// A track obtained via our own getUserMedia() call (the microphone path)
@@ -188,7 +189,6 @@ void SpeechRecognition::Reset() {
188189
}
189190
mTrack = nullptr;
190191
mTrackIsOwned = false;
191-
mStopRecordingPromise = nullptr;
192192
// The microphone path (Start() with no explicit track) registers mListener
193193
// on mStream; it must be unregistered before being cleared (see
194194
// DOMMediaStream::TrackListener). Without this, mListener/mStream survive a
@@ -211,9 +211,18 @@ void SpeechRecognition::PostResetAndEnd() {
211211
RefPtr<SpeechRecognition> self = this;
212212
NS_DispatchToMainThread(NS_NewRunnableFunction(
213213
"SpeechRecognition::PostResetAndEnd", [self = std::move(self)]() {
214+
// Don't end a session that started since this was queued: an "end"
215+
// handler can call start().
214216
if (self->mBackend) {
215217
return;
216218
}
219+
// Reset() cleared [[started]], so this session has already ended. A
220+
// newer session that both started and finished in between gets its
221+
// "end" from here, and its own runnable then no-ops - exactly one
222+
// "end" either way.
223+
if (!self->mStarted) {
224+
return;
225+
}
217226
self->ResetAndEnd();
218227
}));
219228
}
@@ -249,30 +258,6 @@ SpeechRecognition::StartRecording(RefPtr<AudioStreamTrack>& aTrack) {
249258
return NS_OK;
250259
}
251260

252-
RefPtr<GenericNonExclusivePromise> SpeechRecognition::StopRecording() {
253-
AssertIsOnMainThread();
254-
if (!mTrack) {
255-
// Recording wasn't started, or has already been stopped.
256-
return GenericNonExclusivePromise::CreateAndResolve(true, __func__);
257-
}
258-
259-
if (mStopRecordingPromise) {
260-
return mStopRecordingPromise;
261-
}
262-
263-
if (mBackend) {
264-
mBackend->DetachFromTrack();
265-
}
266-
267-
if (mTrackIsOwned) {
268-
mTrack->Stop();
269-
}
270-
271-
DispatchTrustedEvent(u"audioend"_ns);
272-
273-
return nullptr;
274-
}
275-
276261
already_AddRefed<SpeechGrammarList> SpeechRecognition::Grammars() const {
277262
RefPtr<SpeechGrammarList> speechGrammarList = mSpeechGrammarList;
278263
return speechGrammarList.forget();
@@ -626,17 +611,15 @@ void SpeechRecognition::StartImpl(MediaStreamTrack* aAudioTrack,
626611
// Step 4: processLocally is always true here (on-device recognition). If the
627612
// backend cannot start (local recognition unavailable for this lang), fire a
628613
// service-not-allowed error and abort. DispatchErrorAndEnd queues the event.
629-
mBackend = new SpeechRecognitionBackend(this, graphRate, effectiveLang,
630-
phrasesForBackend);
631-
nsresult rv = mBackend->Start();
632-
if (NS_FAILED(rv)) {
633-
LOGE("Failed to start backend: {} ({:x})", GetStaticErrorName(rv),
634-
static_cast<uint32_t>(rv));
635-
mBackend = nullptr;
614+
mBackend = SpeechRecognitionBackend::Create(this, graphRate, effectiveLang,
615+
phrasesForBackend);
616+
if (!mBackend) {
617+
LOGE("Failed to create the backend");
636618
DispatchErrorAndEnd(SpeechRecognitionErrorCode::Service_not_allowed,
637619
"Local speech recognition is not available"_ns);
638620
return;
639621
}
622+
mBackend->Start();
640623

641624
// Step 5: set [[started]] to true.
642625
mStarted = true;
@@ -717,37 +700,67 @@ void SpeechRecognition::StartImpl(MediaStreamTrack* aAudioTrack,
717700

718701
void SpeechRecognition::Stop() {
719702
AssertIsOnMainThread();
720-
// If not started, ignore, per spec
721-
if (!mStarted) {
703+
// https://webaudio.github.io/web-speech-api/#dom-speechrecognition-stop
704+
// "If the stop method is called on an object which is already stopped or
705+
// being stopped [...] the user agent must ignore the call."
706+
if (!mStarted || mStopping || !mBackend) {
722707
return;
723708
}
709+
mStopping = true;
724710

725-
if (mBackend) {
726-
// Stop the backend/session. This will dispatch soundend (if needed) and
727-
// audioend via main thread runnables.
728-
mBackend->Stop();
729-
// We are conforming to spec semantics for stop(): finalize recognition and
730-
// fire 'end'. Clear the backend to avoid late results after end.
731-
mBackend = nullptr;
711+
// Same section: "The speech service must attempt to return a recognition
712+
// result (or a nomatch) based on the audio that it has already collected."
713+
// So mBackend stays live - late results are wanted here, unlike on the
714+
// abort path - and "end" waits for OnSessionFinished(). Dispatches soundend
715+
// (if needed) and audioend via main thread runnables in the meantime.
716+
mBackend->Stop();
717+
}
718+
719+
void SpeechRecognition::OnSessionFinished(bool aProducedResult) {
720+
AssertIsOnMainThread();
721+
LOG("OnSessionFinished: producedResult={}", aProducedResult);
722+
mBackend = nullptr;
732723

733-
// Ensure 'end' fires after the already-posted 'audioend' runnable from the
734-
// backend. PostResetAndEnd() calls ResetAndEnd() on the next turn. mBackend
735-
// was just cleared above and is only set again by a subsequent start();
736-
// if that happened by the time this runs, this stale continuation must
737-
// not reset the new session's state out from under it.
738-
PostResetAndEnd();
724+
if (!aProducedResult) {
725+
DispatchNoMatch();
739726
}
727+
PostResetAndEnd();
728+
}
729+
730+
void SpeechRecognition::DispatchNoMatch() {
731+
AssertIsOnMainThread();
732+
// https://webaudio.github.io/web-speech-api/#eventdef-speechrecognition-nomatch
733+
// The event's results "may contain speech recognition results that are below
734+
// the confidence threshold or may be null"; the engine hands us nothing at
735+
// all in this case, so the list is empty.
736+
RootedDictionary<SpeechRecognitionEventInit> init(RootingCx());
737+
init.mBubbles = true;
738+
init.mCancelable = false;
739+
init.mResultIndex = 0;
740+
init.mResults = new SpeechRecognitionResultList(this);
741+
init.mInterpretation = JS::NullValue();
742+
743+
RefPtr<SpeechRecognitionEvent> domEvent =
744+
SpeechRecognitionEvent::Constructor(this, u"nomatch"_ns, init);
745+
domEvent->SetTrusted(true);
746+
DispatchEvent(*domEvent);
740747
}
741748

742749
void SpeechRecognition::Abort() {
743750
AssertIsOnMainThread();
744-
// If not started, ignore per spec
745-
if (!mStarted) {
751+
// https://webaudio.github.io/web-speech-api/#dom-speechrecognition-abort
752+
// "If the abort method is called on an object which is already stopped or
753+
// aborting (that is, start was never called on it, the end or error event
754+
// has fired on it, or abort was previously called on it), the user agent
755+
// must ignore the call." Without this, a second abort() would queue a second
756+
// reset-and-end, firing "end" twice.
757+
if (!mStarted || mAborting) {
746758
return;
747759
}
760+
mAborting = true;
748761

749762
if (mBackend) {
750-
mBackend->Abort();
763+
mBackend->Abort(TrailingEvents::Fire);
751764
// Clear backend after abort since no more results are expected
752765
mBackend = nullptr;
753766
}
@@ -814,7 +827,7 @@ void SpeechRecognition::DispatchErrorAndEnd(
814827
return;
815828
}
816829
if (mBackend) {
817-
mBackend->Abort();
830+
mBackend->Abort(TrailingEvents::Skip);
818831
mBackend = nullptr;
819832
}
820833
PostResetAndEnd();

‎dom/media/webspeech/recognition/SpeechRecognition.h‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,16 +171,29 @@ class SpeechRecognition final : public DOMEventTargetHelper,
171171
void HandleRecognitionResultFromBackend(const nsCString& aTranscript,
172172
bool aIsFinal);
173173
void HandleRecognitionErrorFromBackend(const nsCString& aError);
174+
// Called once the backend's session is fully over: for stop(), only after
175+
// the engine's end-of-stream flush and the results it produced. Fires
176+
// nomatch when the engine finalized nothing, then "end".
177+
void OnSessionFinished(bool aProducedResult);
174178
// Called once the backend's session is initialized and ready to receive
175179
// audio; combined with a track being attached (mTrack), this determines
176180
// when "start" fires (see MaybeDispatchStart()).
177181
void NotifyBackendListening();
178182

183+
// A backend's callbacks are bound to that specific instance and can still
184+
// be in flight when it's superseded by a newer one (e.g. stop() followed
185+
// immediately by start()). DispatchToParentIfAlive uses this to drop
186+
// notifications from a backend that is no longer the current one, rather
187+
// than misattributing them to whatever session happens to be active by the
188+
// time the callback reaches the main thread.
189+
bool IsCurrentBackend(const SpeechRecognitionBackend* aBackend) const {
190+
return mBackend == aBackend;
191+
}
192+
179193
private:
180194
virtual ~SpeechRecognition();
181195

182196
NS_IMETHOD StartRecording(RefPtr<AudioStreamTrack>& aDOMStream);
183-
RefPtr<GenericNonExclusivePromise> StopRecording();
184197

185198
void Reset();
186199
void ResetAndEnd();
@@ -191,7 +204,14 @@ class SpeechRecognition final : public DOMEventTargetHelper,
191204
// expected to already be cleared by the caller; if a subsequent start()
192205
// set it again by the time this runs, this stale continuation must not
193206
// reset the new session's state out from under it.
207+
//
208+
// Two of these can be queued for one session: DispatchErrorAndEnd() fires
209+
// "error" synchronously, so a listener can call abort() - which posts one -
210+
// before DispatchErrorAndEnd() gets to post its own. The task therefore also
211+
// checks mStarted, which Reset() clears, so only the first one to run ends
212+
// the session and "end" fires exactly once.
194213
void PostResetAndEnd();
214+
void DispatchNoMatch();
195215
// Shared body of the two start() overloads. aAudioTrack is null for the
196216
// microphone start() and the passed track for start(MediaStreamTrack).
197217
void StartImpl(MediaStreamTrack* aAudioTrack, CallerType aCallerType,
@@ -203,11 +223,16 @@ class SpeechRecognition final : public DOMEventTargetHelper,
203223
RefPtr<DOMMediaStream> mStream;
204224
RefPtr<AudioStreamTrack> mTrack;
205225
bool mTrackIsOwned = false;
206-
RefPtr<GenericNonExclusivePromise> mStopRecordingPromise;
207226
RefPtr<SpeechTrackListener> mSpeechListener;
208227

209228
// Tracks if recognition has been started (spec's [[started]] internal slot)
210229
bool mStarted;
230+
// Set by stop() until the backend reports the session finished, so a second
231+
// stop() in that window is ignored per spec.
232+
bool mStopping = false;
233+
// Set by abort() until "end" has fired, so a second abort() in that window is
234+
// ignored per spec.
235+
bool mAborting = false;
211236
// Whether the backend has reported its session as initialized. See
212237
// MaybeDispatchStart().
213238
bool mBackendListening = false;

0 commit comments

Comments
 (0)