|
| 1 | +import base64 |
| 2 | +import dataclasses |
| 3 | +import io |
| 4 | +from typing import Any, Dict, List, Optional |
| 5 | + |
| 6 | +import librosa |
| 7 | +import numpy as np |
| 8 | +import soundfile as sf |
| 9 | +from numpy import typing as npt |
| 10 | + |
| 11 | +SAMPLE_RATE = 16000 |
| 12 | + |
| 13 | + |
| 14 | +def audio_from_file(path: str) -> np.ndarray: |
| 15 | + """Load audio from a file, converting to float32 PCM @ 16 kHz.""" |
| 16 | + audio, _ = librosa.load(path, sr=SAMPLE_RATE) |
| 17 | + assert audio.dtype == np.float32 |
| 18 | + return audio |
| 19 | + |
| 20 | + |
| 21 | +def audio_from_buf(buf: bytes) -> np.ndarray: |
| 22 | + """Load audio from a buffer, converting to float32 PCM @ 16 kHz.""" |
| 23 | + audio, _ = librosa.load(io.BytesIO(buf), sr=SAMPLE_RATE) |
| 24 | + assert audio.dtype == np.float32 |
| 25 | + return audio |
| 26 | + |
| 27 | + |
| 28 | +def audio_to_wav(audio: np.ndarray, sample_rate: int = SAMPLE_RATE) -> bytes: |
| 29 | + """Convert audio to WAV format, 16-bit PCM @ 16 kHz.""" |
| 30 | + assert audio.dtype == np.float32 |
| 31 | + with io.BytesIO() as buf: |
| 32 | + sf.write(buf, audio, sample_rate, format="WAV", subtype="PCM_16") |
| 33 | + return buf.getvalue() |
| 34 | + |
| 35 | + |
| 36 | +def audio_to_wav_base64(audio: np.ndarray, sample_rate: int = SAMPLE_RATE) -> str: |
| 37 | + """Convert audio to a base64-encoded WAV file.""" |
| 38 | + return base64.b64encode(audio_to_wav(audio, sample_rate)).decode("utf-8") |
| 39 | + |
| 40 | + |
| 41 | +def audio_to_data_uri(audio: np.ndarray, sample_rate: int = SAMPLE_RATE) -> str: |
| 42 | + """Convert audio to a data URI.""" |
| 43 | + return f"data:audio/wav;base64,{audio_to_wav_base64(audio, sample_rate)}" |
| 44 | + |
| 45 | + |
| 46 | +def messages_from_prompt(prompt: str) -> List[Dict[str, str]]: |
| 47 | + return [{"role": "user", "content": prompt}] |
| 48 | + |
| 49 | + |
| 50 | +@dataclasses.dataclass |
| 51 | +class VoiceSample: |
| 52 | + @staticmethod |
| 53 | + def from_json(data: Dict[str, Any]) -> "VoiceSample": |
| 54 | + """Convert from JSON format; audio is expected as base64ed WAV.""" |
| 55 | + bytes = base64.b64decode(data["audio"]) |
| 56 | + return VoiceSample(data["messages"], audio_from_buf(bytes)) |
| 57 | + |
| 58 | + @staticmethod |
| 59 | + def from_prompt(prompt: str) -> "VoiceSample": |
| 60 | + """Create a VoiceSample from a prompt only.""" |
| 61 | + return VoiceSample(messages_from_prompt(prompt), None) |
| 62 | + |
| 63 | + @staticmethod |
| 64 | + def from_prompt_and_file(prompt: str, path: str) -> "VoiceSample": |
| 65 | + """Create a VoiceSample from a prompt and an audio file.""" |
| 66 | + return VoiceSample(messages_from_prompt(prompt), audio_from_file(path)) |
| 67 | + |
| 68 | + @staticmethod |
| 69 | + def from_prompt_and_buf(prompt: str, buf: bytes) -> "VoiceSample": |
| 70 | + """Create a VoiceSample from a prompt and an encoded audio buffer.""" |
| 71 | + return VoiceSample(messages_from_prompt(prompt), audio_from_buf(buf)) |
| 72 | + |
| 73 | + @staticmethod |
| 74 | + def from_prompt_and_raw( |
| 75 | + prompt: str, buf: np.ndarray, sample_rate: int |
| 76 | + ) -> "VoiceSample": |
| 77 | + """Create a VoiceSample from a prompt and raw audio data with sample rate.""" |
| 78 | + # Keep in native sample rate; we'll resample later if needed. |
| 79 | + return VoiceSample(messages_from_prompt(prompt), buf, sample_rate) |
| 80 | + |
| 81 | + def to_json(self) -> Dict[str, Any]: |
| 82 | + """Convert to JSON format; audio is written as base64ed WAV.""" |
| 83 | + obj: Dict[str, Any] = {"messages": self.messages} |
| 84 | + if self.audio is not None: |
| 85 | + obj["audio"] = audio_to_wav_base64(self.audio, self.sample_rate) |
| 86 | + return obj |
| 87 | + |
| 88 | + def __post_init__(self): |
| 89 | + """Ensure audio is float32 PCM.""" |
| 90 | + if self.audio is not None: |
| 91 | + if self.audio.dtype == np.float64: |
| 92 | + self.audio = self.audio.astype(np.float32) |
| 93 | + elif self.audio.dtype == np.int16: |
| 94 | + self.audio = self.audio.astype(np.float32) / np.float32(32768.0) |
| 95 | + elif self.audio.dtype == np.int32: |
| 96 | + self.audio = self.audio.astype(np.float32) / np.float32(2147483648.0) |
| 97 | + assert ( |
| 98 | + self.audio.dtype == np.float32 |
| 99 | + ), f"Unexpected audio dtype: {self.audio.dtype}" |
| 100 | + assert self.audio.ndim == 1, f"Unexpected audio shape: {self.audio.shape}" |
| 101 | + |
| 102 | + def add_past_messages(self, past_messages: List[Dict[str, str]]): |
| 103 | + self.messages = past_messages + self.messages |
| 104 | + |
| 105 | + messages: List[Dict[str, str]] |
| 106 | + """List of messages, each with a "role" and "content" field.""" |
| 107 | + audio: Optional[npt.NDArray[np.float32]] = None |
| 108 | + """Audio data as float32 PCM @ `sample_rate`.""" |
| 109 | + sample_rate: int = SAMPLE_RATE |
| 110 | + """Audio sample rate in Hz.""" |
| 111 | + audio_transcript: Optional[str] = None |
| 112 | + """For evaluations, the known transcript of the audio.""" |
0 commit comments