Hey folks,
i am trying to get Membrane and Whisper working together. For now, the goal is, to use Mebrane so stream a file into Whisper. I worked on the following Livebook but cannot figure out, why Whisper is not correctly transcribing.
When i pipe the output of the converter into portaudio it is very noisy. When i set the converter to s16le the PortAudio audio sounds fine! But when i use :f32le and write it to a PCM file and import it into Audacity, it sounds fine again, despite the PortAudio Sink is not.
Either way, Whisper is not translating correctly. Can you help me out, where the issue could be?
Mix.install(
[
{:membrane_core, "~> 0.12.7"},
{:membrane_file_plugin, "~> 0.15.0"},
{:membrane_mp4_plugin, "~> 0.27.0"},
{:membrane_fake_plugin, "~> 0.10.0"},
{:membrane_raw_audio_format, "~> 0.11.0"},
{:membrane_ffmpeg_swresample_plugin,
git: "https://github.com/membraneframework/membrane_ffmpeg_swresample_plugin",
branch: "master"},
{:membrane_aac_plugin, "~> 0.15.0"},
{:membrane_aac_fdk_plugin, "~> 0.15.1"},
{:membrane_portaudio_plugin, "~> 0.16.1"},
{:kino_bumblebee, "~> 0.3.0"},
{:exla, "~> 0.5.1"}
],
config: [nx: [default_backend: EXLA.Backend], membrane_core: [logger: [verbose: false]]]
)
#dependecies:
brew install fdk-aac
brew install portaudio
model = "tiny"
{:ok, model_info} = Bumblebee.load_model({:hf, "openai/whisper-#{model}"})
{:ok, featurizer} = Bumblebee.load_featurizer({:hf, "openai/whisper-#{model}"})
{:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "openai/whisper-#{model}"})
{:ok, generation_config} = Bumblebee.load_generation_config({:hf, "openai/whisper-#{model}"})
generation_config = Bumblebee.configure(generation_config, max_new_tokens: 100)
generation_config = %{
generation_config
| forced_token_ids: [
# tell whisper that spoken text is german
{1, Bumblebee.Tokenizer.token_to_id(tokenizer, "<|de|>")},
{2, Bumblebee.Tokenizer.token_to_id(tokenizer, "<|transcribe|>")}
]
}
serving =
Bumblebee.Audio.speech_to_text(model_info, featurizer, tokenizer, generation_config,
compile: [batch_size: 1],
defn_options: [compiler: EXLA]
)
Nx.Serving.start_link(serving: serving, name: WhisperServing)
defmodule WhisperSink do
use Membrane.Sink
require Logger
def_input_pad(:input, demand_unit: :buffers, accepted_format: _any)
@impl true
def handle_playing(_ctx, state) do
{[demand: :input], state}
end
@impl true
def handle_write_list(:input, buffers, _ctx, state) do
for buffer <- buffers do
data =
buffer.payload
|> Nx.from_binary(:f32)
Logger.info(inspect(Nx.Serving.batched_run(WhisperServing, data)))
end
{[demand: {:input, length(buffers)}], state}
end
end
defmodule Pipeline do
use Membrane.Pipeline
@in_file "/path/to/video.mp4"
@impl true
def handle_init(_context, _opts) do
structure = [
child(:file_source, %Membrane.File.Source{location: @in_file, seekable?: true})
|> child(:demuxer, %Membrane.MP4.Demuxer.ISOM{optimize_for_non_fast_start?: true})
|> via_out(Pad.ref(:output, 2))
|> child(:depayloader_audio, Membrane.MP4.Depayloader.AAC)
|> child(:audio_parser, %Membrane.AAC.Parser{
in_encapsulation: :none,
out_encapsulation: :ADTS
})
|> child(:aac_decoder, Membrane.AAC.FDK.Decoder)
|> child(:converter, %Membrane.FFmpeg.SWResample.Converter{
output_stream_format: %Membrane.RawAudio{
sample_format: :f32le,
sample_rate: 16_000,
channels: 1
}
})
# |> child(:speaker, Membrane.PortAudio.Sink),
|> child(:whisper, WhisperSink),
get_child(:demuxer)
|> via_out(Pad.ref(:output, 1))
|> child(:fake, Membrane.Fake.Sink.Buffers)
]
{[spec: structure, playback: :playing], %{}}
end
@impl true
def handle_element_end_of_stream(:file_sink, _pad, _ctx_, _state) do
IO.inspect("terminate")
{[terminate: :normal], nil}
end
@impl true
def handle_element_end_of_stream(_child, _pad, _ctx, _state) do
{[], nil}
end
def handle_child_notification(notification, _element, _context, state) do
# dbg(notification)
{[], state}
end
end
{:ok, _, _} = Pipeline.start_link()
