diff --git a/README.md b/README.md index 1ac30f7..e7395f0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ React Native wrapper for [FluidAudio](https://github.com/FluidInference/FluidAud ## Features - **ASR (Automatic Speech Recognition)** - High-quality speech-to-text using Parakeet TDT models -- **Streaming ASR** - Real-time transcription from microphone or system audio +- **Streaming ASR** - Real-time transcription from the microphone or from audio you feed in - **VAD (Voice Activity Detection)** - Detect speech segments in audio - **Speaker Diarization** - Identify and track different speakers - **TTS (Text-to-Speech)** - Natural voice synthesis using Kokoro TTS @@ -20,7 +20,9 @@ React Native wrapper for [FluidAudio](https://github.com/FluidInference/FluidAud ## Requirements - iOS 17.0+ -- React Native 0.71+ or Expo SDK 50+ +- React Native 0.75+ or Expo SDK 52+ + +FluidAudio (0.18.0 and later 0.18.x releases) is pulled from GitHub through Swift Package Manager with React Native's `spm_dependency` podspec helper, which needs React Native 0.75. CocoaPods trunk only has FluidAudio 0.7.8 and goes read-only on December 2, 2026. ## Installation @@ -42,6 +44,8 @@ npm install @fluidinference/react-native-fluidaudio cd ios && pod install ``` +Remove any `pod 'FluidAudio'` line from your Podfile: the package brings FluidAudio in itself. If the app fails to link with static pods, install with `USE_FRAMEWORKS=dynamic pod install` (see [react-native#44627](https://github.com/facebook/react-native/pull/44627#issuecomment-2123119711)). + ## Usage ### Basic Transcription @@ -75,20 +79,24 @@ import { StreamingASRManager, onStreamingUpdate } from '@fluidinference/react-na const streaming = new StreamingASRManager(); -// Start streaming with update callback +// Transcribe the microphone (the module records it) await streaming.start({ source: 'microphone' }, (update) => { - console.log('Confirmed:', update.confirmed); - console.log('Volatile:', update.volatile); + console.log('Transcript so far:', update.confirmed); }); -// Feed audio data (16-bit PCM, 16kHz, base64 encoded) -await streaming.feedAudio(base64AudioChunk); +// ...or transcribe audio you supply: little-endian 16-bit mono PCM at 16 kHz +await streaming.start({ source: 'manual' }, (update) => { + console.log('Transcript so far:', update.confirmed); +}); +await streaming.feedAudio(base64AudioChunk); // or feedBuffer(arrayBuffer) -// Stop and get final result +// Stop: queued audio is transcribed first, then the final text is returned const result = await streaming.stop(); console.log('Final transcription:', result.text); ``` +`feedAudio` / `feedBuffer` only work in a `'manual'` session; in a microphone session they reject. The default model is Parakeet EOU 320 ms (English); pick another with `model`, e.g. `{ source: 'manual', model: 'parakeet-eou-160ms' }`. The first start downloads and compiles the model. + ### Voice Activity Detection ```typescript diff --git a/__tests__/FluidAudio.test.ts b/__tests__/FluidAudio.test.ts index f6d03c3..1bab0cc 100644 --- a/__tests__/FluidAudio.test.ts +++ b/__tests__/FluidAudio.test.ts @@ -115,12 +115,20 @@ describe('FluidAudio', () => { }); it('should feed audio while streaming', async () => { - await streaming.start(); + await streaming.start({ source: 'manual' }); await streaming.feedAudio('base64chunk'); expect(mockModule.feedStreamingAudio).toHaveBeenCalledWith('base64chunk'); }); + it('should feed an ArrayBuffer as base64', async () => { + await streaming.start({ source: 'manual' }); + // Two 16-bit samples: 1 and -1, little-endian. + await streaming.feedBuffer(new Uint8Array([0x01, 0x00, 0xff, 0xff]).buffer); + + expect(mockModule.feedStreamingAudio).toHaveBeenCalledWith('AQD//w=='); + }); + it('should throw when feeding audio without starting', async () => { await expect(streaming.feedAudio('base64chunk')).rejects.toThrow( 'Streaming not started' diff --git a/__tests__/types.test.ts b/__tests__/types.test.ts index 011f532..9d269bc 100644 --- a/__tests__/types.test.ts +++ b/__tests__/types.test.ts @@ -94,18 +94,21 @@ describe('Type Definitions', () => { expect(config.source).toBe('microphone'); }); - it('should support system audio source', () => { + it('should support a manual source and a model choice', () => { const config: StreamingASRConfig = { - source: 'system', + source: 'manual', + model: 'parakeet-eou-160ms', }; - expect(config.source).toBe('system'); + expect(config.source).toBe('manual'); + expect(config.model).toBe('parakeet-eou-160ms'); }); it('should define StreamingUpdate correctly', () => { const update: StreamingUpdate = { - volatile: 'partial', + volatile: '', confirmed: 'full', + text: 'full', isFinal: false, }; diff --git a/example/App.tsx b/example/App.tsx index 20c1377..3db17f6 100644 --- a/example/App.tsx +++ b/example/App.tsx @@ -14,6 +14,7 @@ import { View, ActivityIndicator, TextInput, + Image, } from 'react-native'; import { @@ -30,6 +31,45 @@ import { type StreamingUpdate, } from 'react-native-fluidaudio'; +/** The PCM data of a 16 kHz mono 16-bit WAV file. */ +function pcm16FromWav(wav: ArrayBuffer): ArrayBuffer { + const view = new DataView(wav); + const tag = (offset: number) => + String.fromCharCode( + view.getUint8(offset), + view.getUint8(offset + 1), + view.getUint8(offset + 2), + view.getUint8(offset + 3) + ); + if (view.byteLength < 12 || tag(0) !== 'RIFF' || tag(8) !== 'WAVE') { + throw new Error('Not a WAV file'); + } + let formatOk = false; + let offset = 12; + while (offset + 8 <= view.byteLength) { + const id = tag(offset); + const size = view.getUint32(offset + 4, true); + const body = offset + 8; + if (id === 'fmt ') { + formatOk = + view.getUint16(body, true) === 1 && // PCM + view.getUint16(body + 2, true) === 1 && // mono + view.getUint32(body + 4, true) === 16000 && + view.getUint16(body + 14, true) === 16; + } else if (id === 'data') { + if (!formatOk) { + throw new Error('Expected 16 kHz mono 16-bit PCM'); + } + return wav.slice(body, body + size); + } + offset = body + size + (size % 2); + } + throw new Error('WAV file has no data chunk'); +} + +/** 320 ms of 16 kHz 16-bit audio per feedBuffer call. */ +const FEED_CHUNK_BYTES = 16000 * 2 * 0.32; + const App = () => { const [systemInfo, setSystemInfo] = useState(null); const [loading, setLoading] = useState(false); @@ -129,6 +169,36 @@ const App = () => { } }, [streaming]); + // Stream the bundled sample (FluidAudio test fixture, 16.9 s) through feedBuffer, + // to check streaming transcription without a microphone. + const streamSampleFile = useCallback(async () => { + setLoading(true); + setResult(''); + setStreamingText(''); + setConfirmedText(''); + try { + const asset = Image.resolveAssetSource(require('./assets/speech-16k-mono.wav')); + const response = await fetch(asset.uri); + const pcm = pcm16FromWav(await response.arrayBuffer()); + + await streaming.start({ source: 'manual' }, handleStreamingUpdate); + setIsStreaming(true); + for (let offset = 0; offset < pcm.byteLength; offset += FEED_CHUNK_BYTES) { + await streaming.feedBuffer(pcm.slice(offset, offset + FEED_CHUNK_BYTES)); + } + const stopResult = await streaming.stop(); + setResult(`File transcription:\n${stopResult.text || '(no speech detected)'}`); + } catch (error) { + if (streaming.isStreaming()) { + await streaming.stop().catch(() => undefined); + } + setResult(`Error: ${(error as Error).message}`); + } finally { + setIsStreaming(false); + setLoading(false); + } + }, [streaming, handleStreamingUpdate]); + // ============================================================================ // VAD Functions // ============================================================================ @@ -247,15 +317,26 @@ const App = () => { {/* Streaming ASR Section */} Streaming Speech-to-Text - Real-time transcription from microphone + + Real-time transcription from the microphone, or from a bundled file fed in chunks + {!isStreaming ? ( - - 🎤 Start Streaming - + <> + + 🎤 Start Streaming + + + 📄 Stream Sample File + + ) : ( NSAllowsLocalNetworking + NSMicrophoneUsageDescription + Streaming transcription listens to the microphone. NSLocationWhenInUseUsageDescription UILaunchStoryboardName diff --git a/example/ios/Podfile b/example/ios/Podfile index 88378e3..86d8f6d 100644 --- a/example/ios/Podfile +++ b/example/ios/Podfile @@ -17,9 +17,6 @@ end target 'FluidAudioExample' do config = use_native_modules! - # FluidAudio native library - pod 'FluidAudio', '~> 0.7' - use_react_native!( :path => config[:reactNativePath], :app_path => "#{Pod::Config.instance.installation_root}/.." diff --git a/example/package-lock.json b/example/package-lock.json index b098b9c..8ced04a 100644 --- a/example/package-lock.json +++ b/example/package-lock.json @@ -42,7 +42,7 @@ }, "peerDependencies": { "react": ">=18.0.0", - "react-native": ">=0.71.0" + "react-native": ">=0.75.0" } }, "node_modules/@babel/code-frame": { diff --git a/ios/FluidAudioModule.swift b/ios/FluidAudioModule.swift index 2e231ef..8c1bf80 100644 --- a/ios/FluidAudioModule.swift +++ b/ios/FluidAudioModule.swift @@ -59,6 +59,90 @@ enum AudioUtils { return outputSamples } + + /// Decode little-endian 16-bit PCM into Float samples in [-1, 1). + /// Base64-decoded `Data` has no alignment guarantee, so read each sample unaligned. + static func int16LEToFloat(_ data: Data) -> [Float] { + let count = data.count / 2 + return data.withUnsafeBytes { raw -> [Float] in + (0.. AVAudioPCMBuffer? { + let samples = int16LEToFloat(data) + guard !samples.isEmpty, + let format = AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false), + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count)), + let channel = buffer.floatChannelData?[0] + else { return nil } + buffer.frameLength = AVAudioFrameCount(samples.count) + samples.withUnsafeBufferPointer { source in + channel.update(from: source.baseAddress!, count: samples.count) + } + return buffer + } + + /// Copy a tap buffer: the engine reuses it once the tap block returns. + static func copy(_ buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { + guard let copy = AVAudioPCMBuffer(pcmFormat: buffer.format, frameCapacity: buffer.frameLength) else { + return nil + } + copy.frameLength = buffer.frameLength + let source = UnsafeMutableAudioBufferListPointer(UnsafeMutablePointer(mutating: buffer.audioBufferList)) + let destination = UnsafeMutableAudioBufferListPointer(copy.mutableAudioBufferList) + for (from, to) in zip(source, destination) { + guard let fromData = from.mData, let toData = to.mData else { continue } + memcpy(toData, fromData, Int(min(from.mDataByteSize, to.mDataByteSize))) + } + return copy + } +} + +/// Where a streaming session's audio comes from. +enum StreamingSource: String { + /// The module captures the microphone itself. + case microphone + /// The app feeds audio through `feedStreamingAudio`. + case manual +} + +/// One streaming session: audio from the microphone tap or `feedStreamingAudio` goes through +/// `audio` in order, and `processing` hands it to the manager one buffer at a time. +final class StreamingSession { + let manager: any StreamingAsrManager + let variant: StreamingModelVariant + let source: StreamingSource + let audio: AsyncStream.Continuation + let processing: Task + var engine: AVAudioEngine? + + init( + manager: any StreamingAsrManager, + variant: StreamingModelVariant, + source: StreamingSource, + audio: AsyncStream.Continuation, + processing: Task + ) { + self.manager = manager + self.variant = variant + self.source = source + self.audio = audio + self.processing = processing + } + + func stopMicrophone() { + guard let engine else { return } + engine.inputNode.removeTap(onBus: 0) + engine.stop() + self.engine = nil + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } } @objc(FluidAudioModule) @@ -66,11 +150,17 @@ class FluidAudioModule: RCTEventEmitter { private var hasListeners = false - // Streaming ASR - private var streamingManager: StreamingAsrManager? - private var audioEngine: AVAudioEngine? - private var streamingTask: Task? - private var isStreamingActive = false + // Streaming ASR. `streamingLock` guards both properties: React Native calls + // module methods on its own queue while sessions start and stop in Tasks. + private let streamingLock = NSLock() + private var streamingSession: StreamingSession? + /// Set from `startStreamingAsr` until its session is ready, so a second start is refused. + private var streamingStarting = false + /// The last streaming manager used, kept loaded so the next session with the + /// same model starts without reloading it. + private var idleStreamingManager: (variant: StreamingModelVariant, manager: any StreamingAsrManager)? + + static let defaultStreamingModel = StreamingModelVariant.parakeetEou320ms // ASR private var asrManager: AsrManager? @@ -150,7 +240,7 @@ class FluidAudioModule: RCTEventEmitter { ]) let startTime = Date() - let models = try await AsrModels.downloadAndLoad() + let models = try await AsrModels.downloadAndLoad(version: .v3) self.asrModels = models sendEvent(name: "onModelLoadProgress", body: [ @@ -159,9 +249,8 @@ class FluidAudioModule: RCTEventEmitter { "progress": 50 ]) - let asrConfig = ASRConfig(sampleRate: 16000, tdtConfig: TdtConfig()) - let manager = AsrManager(config: asrConfig) - try await manager.initialize(models: models) + let manager = AsrManager(config: .default) + try await manager.loadModels(models) self.asrManager = manager let duration = Date().timeIntervalSince(startTime) @@ -196,7 +285,8 @@ class FluidAudioModule: RCTEventEmitter { Task { do { let url = URL(fileURLWithPath: filePath) - let result = try await asrManager.transcribe(url) + var decoderState = TdtDecoderState.make(decoderLayers: await asrManager.decoderLayerCount) + let result = try await asrManager.transcribe(url, decoderState: &decoderState) resolve([ "text": result.text, @@ -230,12 +320,9 @@ class FluidAudioModule: RCTEventEmitter { Task { do { - let samples = audioData.withUnsafeBytes { ptr -> [Float] in - let int16Ptr = ptr.bindMemory(to: Int16.self) - return int16Ptr.map { Float($0) / 32768.0 } - } - - let result = try await asrManager.transcribe(samples) + let samples = AudioUtils.int16LEToFloat(audioData) + var decoderState = TdtDecoderState.make(decoderLayers: await asrManager.decoderLayerCount) + let result = try await asrManager.transcribe(samples, decoderState: &decoderState) resolve([ "text": result.text, @@ -260,128 +347,237 @@ class FluidAudioModule: RCTEventEmitter { // MARK: - Streaming ASR + private func withStreamingLock(_ body: () throws -> T) rethrows -> T { + streamingLock.lock() + defer { streamingLock.unlock() } + return try body() + } + + /// Reuse the idle manager for `variant`, or create and load a new one. + private func streamingManager(for variant: StreamingModelVariant) async throws -> any StreamingAsrManager { + let idle = withStreamingLock { () -> (variant: StreamingModelVariant, manager: any StreamingAsrManager)? in + defer { idleStreamingManager = nil } + return idleStreamingManager + } + if let idle { + if idle.variant == variant { + try await idle.manager.reset() + return idle.manager + } + await idle.manager.cleanup() + } + + sendEvent(name: "onModelLoadProgress", body: [ + "type": "streaming", + "status": "downloading", + "progress": 0 + ]) + let manager = variant.createManager() + try await manager.loadModels() + sendEvent(name: "onModelLoadProgress", body: [ + "type": "streaming", + "status": "ready", + "progress": 100 + ]) + return manager + } + + /// Start capturing the microphone into the session's audio queue. + private func startMicrophone(into audio: AsyncStream.Continuation) throws -> AVAudioEngine { + let session = AVAudioSession.sharedInstance() + try session.setCategory(.playAndRecord, mode: .measurement, options: [.defaultToSpeaker, .allowBluetoothHFP]) + try session.setActive(true) + + let engine = AVAudioEngine() + let inputNode = engine.inputNode + let inputFormat = inputNode.outputFormat(forBus: 0) + inputNode.installTap(onBus: 0, bufferSize: 4096, format: inputFormat) { buffer, _ in + // The streaming manager resamples to 16 kHz mono itself. + if let copy = AudioUtils.copy(buffer) { + audio.yield(copy) + } + } + do { + try engine.start() + } catch { + inputNode.removeTap(onBus: 0) + throw error + } + return engine + } + + /// Start a streaming session. + /// + /// Config: + /// - `source`: `"microphone"` (default) captures the microphone; `"manual"` takes audio + /// only from `feedStreamingAudio`. + /// - `model`: a `StreamingModelVariant` raw value; defaults to `"parakeet-eou-320ms"`. @objc(startStreamingAsr:resolver:rejecter:) func startStreamingAsr( config: NSDictionary?, resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock ) { - guard !isStreamingActive else { + let sourceName = config?["source"] as? String ?? StreamingSource.microphone.rawValue + guard let source = StreamingSource(rawValue: sourceName) else { + reject( + "STREAMING_SOURCE_UNSUPPORTED", + "Unsupported streaming source '\(sourceName)'. Use 'microphone' or 'manual'.", nil) + return + } + let modelName = config?["model"] as? String ?? Self.defaultStreamingModel.rawValue + guard let variant = StreamingModelVariant(rawValue: modelName) else { + let known = StreamingModelVariant.allCases.map(\.rawValue).joined(separator: ", ") + reject("STREAMING_MODEL_UNSUPPORTED", "Unknown streaming model '\(modelName)'. Use one of: \(known).", nil) + return + } + + let alreadyActive = withStreamingLock { () -> Bool in + if streamingSession != nil || streamingStarting { return true } + streamingStarting = true + return false + } + guard !alreadyActive else { reject("STREAMING_ACTIVE", "Streaming is already active", nil) return } - Task { + Task { [weak self] in + guard let self else { + reject("STREAMING_START_ERROR", "The FluidAudio module was released", nil) + return + } + var loadedManager: (any StreamingAsrManager)? + var startedSession: StreamingSession? do { - // Request microphone permission - let session = AVAudioSession.sharedInstance() - try session.setCategory(.playAndRecord, mode: .measurement, options: [.defaultToSpeaker, .allowBluetooth]) - try session.setActive(true) - - // Create streaming manager - let streamConfig = StreamingAsrConfig.streaming - let manager = StreamingAsrManager(config: streamConfig) - self.streamingManager = manager - - // Start the streaming engine - try await manager.start(source: .microphone) - - // Set up audio engine to capture microphone - let engine = AVAudioEngine() - self.audioEngine = engine - - let inputNode = engine.inputNode - let inputFormat = inputNode.outputFormat(forBus: 0) - - // Install tap to capture audio - inputNode.installTap(onBus: 0, bufferSize: 4096, format: inputFormat) { [weak self] buffer, _ in - guard let self = self, self.isStreamingActive else { return } - Task { - await self.streamingManager?.streamAudio(buffer) - } + let manager = try await streamingManager(for: variant) + loadedManager = manager + await manager.setPartialTranscriptCallback { [weak self] text in + // These models never revise emitted text, so the partial transcript is confirmed. + self?.sendEvent(name: "onStreamingUpdate", body: [ + "volatile": "", + "confirmed": text, + "text": text, + "isFinal": false + ]) } - // Start audio engine - try engine.start() - self.isStreamingActive = true - - // Listen for transcription updates - self.streamingTask = Task { [weak self] in - guard let self = self, let manager = self.streamingManager else { return } - - for await update in await manager.transcriptionUpdates { - let volatile = await manager.volatileTranscript - let confirmed = await manager.confirmedTranscript - - self.sendEvent(name: "onStreamingUpdate", body: [ - "volatile": volatile, - "confirmed": confirmed, - "text": update.text, - "isConfirmed": update.isConfirmed, - "confidence": update.confidence, - "isFinal": false - ]) + let (stream, audio) = AsyncStream.makeStream() + let processing = Task { [weak self] in + for await buffer in stream { + do { + try await manager.appendAudio(buffer) + try await manager.processBufferedAudio() + } catch { + self?.sendEvent(name: "onTranscriptionError", body: [ + "code": "STREAMING_PROCESS_ERROR", + "message": error.localizedDescription + ]) + } } } + let session = StreamingSession( + manager: manager, variant: variant, source: source, audio: audio, processing: processing) + startedSession = session - resolve(["success": true]) + if source == .microphone { + session.engine = try startMicrophone(into: audio) + } + withStreamingLock { + self.streamingSession = session + self.streamingStarting = false + } + resolve(["success": true, "source": source.rawValue, "model": variant.rawValue]) } catch { - self.isStreamingActive = false + if let startedSession { + startedSession.stopMicrophone() + startedSession.audio.finish() + } + if source == .microphone { + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } + withStreamingLock { + if let loadedManager { self.idleStreamingManager = (variant, loadedManager) } + self.streamingStarting = false + } reject("STREAMING_START_ERROR", "Failed to start streaming: \(error.localizedDescription)", error) } } } + /// Feed audio to a session started with `source: 'manual'`. + /// + /// `base64Audio` is little-endian 16-bit mono PCM at 16 kHz. The buffer is queued in order + /// behind any audio already fed; transcript updates arrive as `onStreamingUpdate` events. @objc(feedStreamingAudio:resolver:rejecter:) func feedStreamingAudio( base64Audio: String, resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock ) { - // This is handled automatically by the audio tap when using microphone source - // For manual feeding, we would need to convert base64 to AVAudioPCMBuffer - resolve(["success": true]) + guard let session = withStreamingLock({ streamingSession }) else { + reject("STREAMING_NOT_ACTIVE", "Streaming not started. Call startStreamingAsr first.", nil) + return + } + guard session.source == .manual else { + reject( + "STREAMING_SOURCE_MICROPHONE", + "This session captures the microphone itself. Start streaming with source: 'manual' to feed audio.", + nil) + return + } + guard let audioData = Data(base64Encoded: base64Audio) else { + reject("INVALID_AUDIO", "Invalid base64 audio data", nil) + return + } + guard audioData.count % 2 == 0 else { + reject("INVALID_AUDIO", "Audio must be 16-bit PCM, but has an odd number of bytes", nil) + return + } + guard let buffer = AudioUtils.pcm16MonoBuffer(from: audioData, sampleRate: 16000) else { + reject("INVALID_AUDIO", "Audio data is empty", nil) + return + } + + session.audio.yield(buffer) + resolve(["success": true, "samples": Int(buffer.frameLength)]) } + /// Stop the session: transcribe all audio queued so far, then return the final text. @objc(stopStreamingAsr:rejecter:) func stopStreamingAsr( resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock ) { - guard isStreamingActive else { + guard let session = withStreamingLock({ () -> StreamingSession? in + defer { streamingSession = nil } + return streamingSession + }) else { resolve(["text": "", "success": true]) return } Task { - do { - // Stop audio engine - audioEngine?.inputNode.removeTap(onBus: 0) - audioEngine?.stop() - audioEngine = nil - - // Get final transcription - let finalText = try await streamingManager?.finish() ?? "" - - // Cancel update task - streamingTask?.cancel() - streamingTask = nil - - // Clean up - streamingManager = nil - isStreamingActive = false - - // Deactivate audio session - try? AVAudioSession.sharedInstance().setActive(false) + session.stopMicrophone() + session.audio.finish() + await session.processing.value + do { + let finalText = try await session.manager.finish() + sendEvent(name: "onStreamingUpdate", body: [ + "volatile": "", + "confirmed": finalText, + "text": finalText, + "isFinal": true + ]) + withStreamingLock { idleStreamingManager = (session.variant, session.manager) } resolve([ "text": finalText, "success": true ]) - } catch { - isStreamingActive = false + await session.manager.cleanup() reject("STREAMING_STOP_ERROR", "Failed to stop streaming: \(error.localizedDescription)", error) } } @@ -463,10 +659,7 @@ class FluidAudioModule: RCTEventEmitter { Task { do { - let samples = audioData.withUnsafeBytes { ptr -> [Float] in - let int16Ptr = ptr.bindMemory(to: Int16.self) - return int16Ptr.map { Float($0) / 32768.0 } - } + let samples = AudioUtils.int16LEToFloat(audioData) let results = try await vadManager.process(samples) @@ -621,10 +814,7 @@ class FluidAudioModule: RCTEventEmitter { Task { do { - let samples = audioData.withUnsafeBytes { ptr -> [Float] in - let int16Ptr = ptr.bindMemory(to: Int16.self) - return int16Ptr.map { Float($0) / 32768.0 } - } + let samples = AudioUtils.int16LEToFloat(audioData) let result = try diarizerManager.performCompleteDiarization(samples, sampleRate: sampleRate) @@ -722,20 +912,29 @@ class FluidAudioModule: RCTEventEmitter { resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock ) { + let (session, idle) = withStreamingLock { + () -> (StreamingSession?, (variant: StreamingModelVariant, manager: any StreamingAsrManager)?) in + defer { + streamingSession = nil + idleStreamingManager = nil + } + return (streamingSession, idleStreamingManager) + } + Task { - // Stop streaming if active - if isStreamingActive { - audioEngine?.inputNode.removeTap(onBus: 0) - audioEngine?.stop() - audioEngine = nil - await streamingManager?.cancel() - streamingManager = nil - streamingTask?.cancel() - streamingTask = nil - isStreamingActive = false + // Stop streaming if active, discarding queued audio + if let session { + session.stopMicrophone() + session.audio.finish() + session.processing.cancel() + await session.manager.cleanup() + } + if let idle { + await idle.manager.cleanup() } // Clean up managers + await asrManager?.cleanup() asrManager = nil asrModels = nil vadManager = nil diff --git a/package-lock.json b/package-lock.json index 4e82c9c..69dc042 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fluidinference/react-native-fluidaudio", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@fluidinference/react-native-fluidaudio", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "devDependencies": { "@babel/core": "^7.24.0", @@ -24,7 +24,7 @@ }, "peerDependencies": { "react": ">=18.0.0", - "react-native": ">=0.71.0" + "react-native": ">=0.75.0" } }, "node_modules/@babel/code-frame": { diff --git a/package.json b/package.json index 9b24978..2584114 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fluidinference/react-native-fluidaudio", - "version": "0.1.0", + "version": "0.2.0", "description": "React Native wrapper for FluidAudio - ASR, VAD, Diarization, and TTS", "main": "lib/commonjs/index", "module": "lib/module/index", @@ -81,7 +81,7 @@ }, "peerDependencies": { "react": ">=18.0.0", - "react-native": ">=0.71.0" + "react-native": ">=0.75.0" }, "react-native-builder-bob": { "source": "src", diff --git a/react-native-fluidaudio.podspec b/react-native-fluidaudio.podspec index cfa11d5..4fa1ed6 100644 --- a/react-native-fluidaudio.podspec +++ b/react-native-fluidaudio.podspec @@ -18,8 +18,17 @@ Pod::Spec.new do |s| # React Native dependencies s.dependency "React-Core" - # FluidAudio dependency (fetched from GitHub) - s.dependency "FluidAudio", "~> 0.7" + # FluidAudio comes from GitHub through Swift Package Manager: CocoaPods trunk stops at + # FluidAudio 0.7.8 and goes read-only on December 2, 2026. `spm_dependency` is React + # Native's podspec helper (0.75+), defined once the Podfile loads react_native_pods.rb. + unless defined?(spm_dependency) + raise "react-native-fluidaudio needs React Native 0.75 or newer (for spm_dependency)." + end + spm_dependency(s, + url: "https://github.com/FluidInference/FluidAudio.git", + requirement: { kind: "upToNextMinorVersion", minimumVersion: "0.18.0" }, + products: ["FluidAudio"] + ) s.ios.frameworks = "CoreML", "AVFoundation", "Accelerate", "UIKit" diff --git a/src/FluidAudio.ts b/src/FluidAudio.ts index 364e086..7d4aa72 100644 --- a/src/FluidAudio.ts +++ b/src/FluidAudio.ts @@ -185,6 +185,9 @@ export class StreamingASRManager { /** * Start streaming transcription + * + * With `source: 'microphone'` (the default) the module records the microphone. + * With `source: 'manual'` it transcribes only the audio passed to `feedAudio` / `feedBuffer`. * @param config Streaming configuration * @param onUpdate Callback for transcription updates */ @@ -202,8 +205,8 @@ export class StreamingASRManager { } /** - * Feed audio data using ArrayBuffer - * @param audioBuffer ArrayBuffer containing 16-bit PCM audio at 16kHz + * Feed audio to a session started with `source: 'manual'` + * @param audioBuffer Little-endian 16-bit mono PCM at 16 kHz */ async feedBuffer(audioBuffer: ArrayBuffer): Promise { if (!this.streaming) { @@ -214,8 +217,8 @@ export class StreamingASRManager { } /** - * Feed audio data - * @param base64Audio Base64-encoded 16-bit PCM audio at 16kHz + * Feed audio to a session started with `source: 'manual'` + * @param base64Audio Base64-encoded little-endian 16-bit mono PCM at 16 kHz */ async feedAudio(base64Audio: string): Promise { if (!this.streaming) { @@ -226,14 +229,19 @@ export class StreamingASRManager { /** * Stop streaming and get final transcription + * + * Audio already fed is transcribed before this resolves, and the update + * handler receives a last update with `isFinal: true`. */ async stop(): Promise { - this.updateSubscription?.remove(); - this.updateSubscription = null; - - const result = await FluidAudioNative.stopStreamingAsr(); - this.streaming = false; - return result; + try { + const result = await FluidAudioNative.stopStreamingAsr(); + this.streaming = false; + return result; + } finally { + this.updateSubscription?.remove(); + this.updateSubscription = null; + } } /** diff --git a/src/NativeFluidAudio.ts b/src/NativeFluidAudio.ts index f5731c2..310b504 100644 --- a/src/NativeFluidAudio.ts +++ b/src/NativeFluidAudio.ts @@ -30,11 +30,18 @@ export interface Spec extends TurboModule { isAsrAvailable(): Promise; // Streaming ASR - startStreamingAsr(config: Object | null): Promise; - feedStreamingAudio(base64Audio: string): Promise; + startStreamingAsr(config: Object | null): Promise<{ + success: boolean; + source: string; + model: string; + }>; + feedStreamingAudio(base64Audio: string): Promise<{ + success: boolean; + samples: number; + }>; stopStreamingAsr(): Promise<{ - finalText: string; - totalDuration: number; + text: string; + success: boolean; }>; // VAD diff --git a/src/index.ts b/src/index.ts index e424b3a..5723d9f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,6 +33,7 @@ export type { ASRPerformanceMetrics, // Streaming ASR StreamingASRConfig, + StreamingModel, StreamingUpdate, StreamingStopResult, // VAD diff --git a/src/types.ts b/src/types.ts index f89b039..8b66530 100644 --- a/src/types.ts +++ b/src/types.ts @@ -65,19 +65,45 @@ export interface ASRInitResult { // Streaming ASR // ============================================================================ +/** + * Streaming ASR model: a FluidAudio `StreamingModelVariant` raw value. + * Parakeet EOU and Nemotron are English-only. Parakeet Unified's int8 encoder + * does not load on some iPhones (A16), so prefer Parakeet EOU on iOS. + */ +export type StreamingModel = + | 'parakeet-eou-160ms' + | 'parakeet-eou-320ms' + | 'parakeet-eou-1280ms' + | 'nemotron-560ms' + | 'nemotron-1120ms' + | 'nemotron-2240ms' + | 'parakeet-unified-320ms' + | 'parakeet-unified-640ms' + | 'parakeet-unified-1120ms' + | 'parakeet-unified-2080ms' + | 'parakeet-unified-offline-15s'; + export interface StreamingASRConfig { - /** Audio source: 'microphone' or 'system' (macOS only) */ - source?: 'microphone' | 'system'; - /** Chunk duration in seconds */ + /** + * Audio source (default: 'microphone'). + * - 'microphone': the module records the microphone itself. + * - 'manual': the app supplies the audio with `feedAudio` / `feedBuffer`. + */ + source?: 'microphone' | 'manual'; + /** Streaming model (default: 'parakeet-eou-320ms') */ + model?: StreamingModel; + /** @deprecated Ignored: the chunk size is set by `model`. */ chunkDuration?: number; } export interface StreamingUpdate { - /** Current volatile (unconfirmed) transcript */ + /** Unconfirmed text. Always empty: the streaming models never revise text they have emitted. */ volatile: string; - /** Confirmed transcript */ + /** Transcript so far */ confirmed: string; - /** Whether this is the final update */ + /** Same as `confirmed` */ + text: string; + /** True for the last update, sent when streaming stops */ isFinal: boolean; } @@ -206,7 +232,7 @@ export interface TTSResult { // ============================================================================ export interface ModelLoadProgressEvent { - type?: 'asr' | 'diarization' | 'vad' | 'tts'; + type?: 'asr' | 'streaming' | 'diarization' | 'vad' | 'tts'; status: 'downloading' | 'compiling' | 'ready'; progress: number; } @@ -236,8 +262,10 @@ export interface FluidAudioNativeModule { isAsrAvailable(): Promise; // Streaming ASR - startStreamingAsr(config?: StreamingASRConfig): Promise<{ success: boolean }>; - feedStreamingAudio(base64Audio: string): Promise<{ success: boolean }>; + startStreamingAsr( + config?: StreamingASRConfig + ): Promise<{ success: boolean; source: string; model: string }>; + feedStreamingAudio(base64Audio: string): Promise<{ success: boolean; samples: number }>; stopStreamingAsr(): Promise; // VAD