Bumblebee 101
Mix.install([
{:bumblebee, [github: "elixir-nx/bumblebee", override: true]},
{:nx, "~> 0.8.0", [override: true]},
{:exla, "~> 0.8.0", [override: true]},
{:kino, "~> 0.14.0", [override: true]},
{:kino_bumblebee, [github: "livebook-dev/kino_bumblebee", override: true]},
{:kino_db, "~> 0.2.12"}
])
Nx.global_default_backend({EXLA.Backend, client: :host})
Untitled
hf_token = System.fetch_env!("LB_HF_TOKEN")
{:ok, bert} = Bumblebee.load_model({:hf, "google-bert/bert-base-uncased"})
{:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "google-bert/bert-base-uncased"})
serving = Bumblebee.Text.fill_mask(bert, tokenizer)
text_input = Kino.Input.text("Sentence with mask", default: "The capital of [MASK] is Paris.")
text = Kino.Input.read(text_input)
Nx.Serving.run(serving, text)
phi3_5 = {:hf, "microsoft/Phi-3.5-mini-instruct"}
t5_flant = {:hf, "google/flan-t5-large"}
smollm = {:hf, "HuggingFaceTB/SmolLM-1.7B"}
llama_minitron = {:hf, "nvidia/Llama-3.1-Minitron-4B-Width-Base"}
repo = phi3_5
{:ok, model} = Bumblebee.load_model(repo)
{:ok, tokenizer} = Bumblebee.load_tokenizer(repo)
{:ok, generation_config} = Bumblebee.load_generation_config(repo)
generation_config = Bumblebee.configure(generation_config, max_new_tokens: 256)
serving = Bumblebee.Text.generation(model, tokenizer, generation_config,
compile: [batch_size: 1, sequence_length: 256],
stream: true,
defn_options: [compiler: EXLA]
)
Kino.start_child({Nx.Serving, name: Model, serving: serving})
prompt = "Complete the paragraph: our solar system is"
Nx.Serving.batched_run(Model, prompt) |> Enum.each(&IO.write/1)
