Coverage for src/manimgx/audio/fal.py: 93%

121 statements  

« prev     ^ index     » next       coverage.py v7.16.1, created at 2026-10-26 05:52 +0000

1"""fal.ai's text-to-speech models, as voices: `Fal(model, **settings)`. 

2 

3[fal](https://fal.ai) runs text-to-speech models behind one API and one key. `Fal` supports 

4the models below: 

5 

6| Model | Its settings | Words timed | 

7| --- | --- | --- | 

8| `fal-ai/elevenlabs/tts/eleven-v3` (the default) | [`ElevenV3`][manimgx.audio.fal.ElevenV3] | by the model | 

9| `fal-ai/minimax/speech-2.8-hd` | [`MiniMax`][manimgx.audio.fal.MiniMax] | estimated | 

10| `google/gemini-3.8-flash-tts` | [`Gemini`][manimgx.audio.fal.Gemini] | estimated | 

11| `fal-ai/inworld-tts` | [`Inworld`][manimgx.audio.fal.Inworld] | estimated | 

12| `fal-ai/qwen-3-tts/text-to-speech/1.7b` | [`Qwen`][manimgx.audio.fal.Qwen] | estimated | 

13 

14A model takes settings of its own, typed per model: a type checker knows which settings a 

15model takes and which values (its voices, say). The key is `FAL_KEY`'s (fal's dashboard 

16gives one), or `key=`; it never names the voice in the cache. 

17""" 

18 

19import json 

20import math 

21import os 

22import urllib.error 

23import urllib.request 

24from dataclasses import dataclass, field 

25from typing import Literal, Unpack, overload 

26 

27from typing_extensions import TypedDict 

28 

29from manimgx.audio.sound import Speech, timed 

30 

31__all__ = ["Fal"] 

32 

33# ── the models' settings, as their schemas on fal have them ──────────────────────────────── 

34 

35 

36class ElevenV3(TypedDict, total=False, closed=True): 

37 """The settings of ElevenLabs' Eleven v3 (`fal-ai/elevenlabs/tts/eleven-v3`), which times 

38 its words. Its text may carry audio tags: `[whispers]`, `[laughs]`, `[excited]`…""" 

39 

40 voice: str 

41 """A voice's name or ID in ElevenLabs' library: Aria, Roger, Sarah, Laura, Charlie, 

42 George, Callum, River, Liam, Charlotte, Alice, Matilda, Will, Jessica, Eric, Chris, Brian, 

43 Daniel, Lily, Bill… (default "Rachel").""" 

44 stability: float 

45 """How steady the voice is, from 0 (expressive) to 1 (steady) (default 0.5).""" 

46 language_code: str | None 

47 """The language, as an ISO 639-1 code ("en", "de"…) (default: the text's).""" 

48 apply_text_normalization: Literal["auto", "on", "off"] 

49 """Whether numbers and the like are spelled out before they are said (default "auto": 

50 as the model sees fit).""" 

51 

52 

53class MiniMaxVoice(TypedDict, total=False): 

54 """A MiniMax voice and how it speaks.""" 

55 

56 voice_id: str 

57 """The voice: Wise_Woman, Friendly_Person, Inspirational_girl, Deep_Voice_Man, Calm_Woman, 

58 Casual_Guy, Lively_Girl, Patient_Man, Young_Knight, Determined_Man, Lovely_Girl, 

59 Decent_Boy, Imposing_Manner, Elegant_Man, Abbess, Sweet_Girl_2, Exuberant_Girl, or a 

60 cloned voice's ID (default "Wise_Woman").""" 

61 speed: float 

62 """How fast, from 0.5 to 2 (default 1).""" 

63 vol: float 

64 """How loud, from 0.01 to 10 (default 1).""" 

65 pitch: int 

66 """How high, in semitones from -12 to 12 (default 0).""" 

67 emotion: ( 

68 Literal["happy", "sad", "angry", "fearful", "disgusted", "surprised", "neutral"] 

69 | None 

70 ) 

71 """The emotion it speaks with (default: as the text reads).""" 

72 english_normalization: bool 

73 """Whether English numbers are read more carefully, a little slower (default False).""" 

74 

75 

76class MiniMaxAudio(TypedDict, total=False): 

77 """The audio MiniMax makes.""" 

78 

79 sample_rate: Literal[8000, 16000, 22050, 24000, 32000, 44100] 

80 """Samples a second (default 32000).""" 

81 bitrate: Literal[32000, 64000, 128000, 256000] 

82 """Bits a second, for MP3 (default 128000).""" 

83 format: Literal["mp3", "flac"] 

84 """The file's format (default "mp3").""" 

85 channel: Literal[1, 2] 

86 """Channels: 1 (mono) or 2 (stereo) (default 1).""" 

87 

88 

89class MiniMaxLoudness(TypedDict, total=False): 

90 """How MiniMax evens out the audio's loudness.""" 

91 

92 enabled: bool 

93 """Whether it does (default True).""" 

94 target_loudness: float 

95 """The loudness it aims at, in LUFS, from -70 to -10 (default -18).""" 

96 target_range: float 

97 """The loudness range, in LU, from 0 to 20 (default 8).""" 

98 target_peak: float 

99 """The highest peak, in dBTP, from -3 to 0 (default -0.5).""" 

100 

101 

102class MiniMaxTimbre(TypedDict, total=False): 

103 """How MiniMax changes the voice itself.""" 

104 

105 pitch: int 

106 """Higher or lower, from -100 to 100 (default 0).""" 

107 intensity: int 

108 """More or less energetic, from -100 to 100 (default 0).""" 

109 timbre: int 

110 """Its tone's color, from -100 to 100 (default 0).""" 

111 

112 

113class MiniMaxPronunciation(TypedDict, total=False): 

114 """How MiniMax pronounces particular words.""" 

115 

116 tone_list: list[str] 

117 """Each word and its pronunciation: `"text/(pronunciation)"` (Chinese tones 1 to 5: 

118 `"燕少飞/(yan4)(shao3)(fei1)"`).""" 

119 

120 

121class MiniMax(TypedDict, total=False, closed=True): 

122 """The settings of MiniMax Speech 2.8 HD (`fal-ai/minimax/speech-2.8-hd`). Its text may 

123 carry pauses, `<#0.5#>` (in seconds), and interjections: `(laughs)`, `(sighs)`, `(coughs)`, 

124 `(clears throat)`, `(gasps)`, `(sniffs)`, `(groans)`, `(yawns)`.""" 

125 

126 voice_setting: MiniMaxVoice 

127 """The voice and how it speaks ([`MiniMaxVoice`][manimgx.audio.fal.MiniMaxVoice]).""" 

128 audio_setting: MiniMaxAudio 

129 """The audio it makes ([`MiniMaxAudio`][manimgx.audio.fal.MiniMaxAudio]).""" 

130 language_boost: ( 

131 Literal[ 

132 "Chinese", 

133 "Chinese,Yue", 

134 "English", 

135 "Arabic", 

136 "Russian", 

137 "Spanish", 

138 "French", 

139 "Portuguese", 

140 "German", 

141 "Turkish", 

142 "Dutch", 

143 "Ukrainian", 

144 "Vietnamese", 

145 "Indonesian", 

146 "Japanese", 

147 "Italian", 

148 "Korean", 

149 "Thai", 

150 "Polish", 

151 "Romanian", 

152 "Greek", 

153 "Czech", 

154 "Finnish", 

155 "Hindi", 

156 "Bulgarian", 

157 "Danish", 

158 "Hebrew", 

159 "Malay", 

160 "Slovak", 

161 "Swedish", 

162 "Croatian", 

163 "Hungarian", 

164 "Norwegian", 

165 "Slovenian", 

166 "Catalan", 

167 "Nynorsk", 

168 "Afrikaans", 

169 "auto", 

170 ] 

171 | None 

172 ) 

173 """A language or dialect it listens for (default: none).""" 

174 normalization_setting: MiniMaxLoudness 

175 """How it evens out the loudness 

176 ([`MiniMaxLoudness`][manimgx.audio.fal.MiniMaxLoudness]).""" 

177 voice_modify: MiniMaxTimbre | None 

178 """How it changes the voice itself ([`MiniMaxTimbre`][manimgx.audio.fal.MiniMaxTimbre]).""" 

179 pronunciation_dict: MiniMaxPronunciation | None 

180 """How it pronounces particular words 

181 ([`MiniMaxPronunciation`][manimgx.audio.fal.MiniMaxPronunciation]).""" 

182 

183 

184class Gemini(TypedDict, total=False, closed=True): 

185 """The settings of Google's Gemini 3.8 Flash TTS (`google/gemini-3.8-flash-tts`). Its text 

186 may carry vocal events: `<laugh>`, `<sigh>`.""" 

187 

188 voice: Literal[ 

189 "Achernar", 

190 "Achird", 

191 "Algenib", 

192 "Algieba", 

193 "Alnilam", 

194 "Aoede", 

195 "Autonoe", 

196 "Callirrhoe", 

197 "Charon", 

198 "Despina", 

199 "Enceladus", 

200 "Erinome", 

201 "Fenrir", 

202 "Gacrux", 

203 "Iapetus", 

204 "Kore", 

205 "Laomedeia", 

206 "Leda", 

207 "Orus", 

208 "Pulcherrima", 

209 "Puck", 

210 "Rasalgethi", 

211 "Sadachbia", 

212 "Sadaltager", 

213 "Schedar", 

214 "Sulafat", 

215 "Umbriel", 

216 "Vindemiatrix", 

217 "Zephyr", 

218 "Zubenelgenubi", 

219 ] 

220 """The voice (default "Kore").""" 

221 style_instructions: str | None 

222 """How to say it, in words: "Warm and unhurried, like a patient teacher." (default: 

223 none).""" 

224 

225 

226class Inworld(TypedDict, total=False, closed=True): 

227 """The settings of Inworld TTS-1.5 Max (`fal-ai/inworld-tts`), which speaks up to 2,000 

228 characters a line.""" 

229 

230 voice: Literal[ 

231 "Loretta (en)", 

232 "Darlene (en)", 

233 "Marlene (en)", 

234 "Hank (en)", 

235 "Evelyn (en)", 

236 "Celeste (en)", 

237 "Pippa (en)", 

238 "Tessa (en)", 

239 "Liam (en)", 

240 "Callum (en)", 

241 "Hamish (en)", 

242 "Abby (en)", 

243 "Graham (en)", 

244 "Rupert (en)", 

245 "Mortimer (en)", 

246 "Snik (en)", 

247 "Anjali (en)", 

248 "Saanvi (en)", 

249 "Arjun (en)", 

250 "Claire (en)", 

251 "Oliver (en)", 

252 "Simon (en)", 

253 "Elliot (en)", 

254 "James (en)", 

255 "Serena (en)", 

256 "Gareth (en)", 

257 "Vinny (en)", 

258 "Lauren (en)", 

259 "Jessica (en)", 

260 "Ethan (en)", 

261 "Tyler (en)", 

262 "Jason (en)", 

263 "Chloe (en)", 

264 "Veronica (en)", 

265 "Victoria (en)", 

266 "Miranda (en)", 

267 "Sebastian (en)", 

268 "Victor (en)", 

269 "Malcolm (en)", 

270 "Kayla (en)", 

271 "Nate (en)", 

272 "Jake (en)", 

273 "Brian (en)", 

274 "Amina (en)", 

275 "Kelsey (en)", 

276 "Derek (en)", 

277 "Grant (en)", 

278 "Evan (en)", 

279 "Alex (en)", 

280 "Ashley (en)", 

281 "Craig (en)", 

282 "Deborah (en)", 

283 "Dennis (en)", 

284 "Edward (en)", 

285 "Elizabeth (en)", 

286 "Hades (en)", 

287 "Julia (en)", 

288 "Pixie (en)", 

289 "Mark (en)", 

290 "Olivia (en)", 

291 "Priya (en)", 

292 "Ronald (en)", 

293 "Sarah (en)", 

294 "Shaun (en)", 

295 "Theodore (en)", 

296 "Timothy (en)", 

297 "Wendy (en)", 

298 "Dominus (en)", 

299 "Hana (en)", 

300 "Clive (en)", 

301 "Carter (en)", 

302 "Blake (en)", 

303 "Luna (en)", 

304 "Yichen (zh)", 

305 "Xiaoyin (zh)", 

306 "Xinyi (zh)", 

307 "Jing (zh)", 

308 "Erik (nl)", 

309 "Katrien (nl)", 

310 "Lennart (nl)", 

311 "Lore (nl)", 

312 "Alain (fr)", 

313 "Hélène (fr)", 

314 "Mathieu (fr)", 

315 "Étienne (fr)", 

316 "Johanna (de)", 

317 "Josef (de)", 

318 "Gianni (it)", 

319 "Orietta (it)", 

320 "Asuka (ja)", 

321 "Satoshi (ja)", 

322 "Hyunwoo (ko)", 

323 "Minji (ko)", 

324 "Seojun (ko)", 

325 "Yoona (ko)", 

326 "Szymon (pl)", 

327 "Wojciech (pl)", 

328 "Heitor (pt)", 

329 "Maitê (pt)", 

330 "Diego (es)", 

331 "Lupita (es)", 

332 "Miguel (es)", 

333 "Rafael (es)", 

334 "Svetlana (ru)", 

335 "Elena (ru)", 

336 "Dmitry (ru)", 

337 "Nikolai (ru)", 

338 "Riya (hi)", 

339 "Manoj (hi)", 

340 "Yael (he)", 

341 "Oren (he)", 

342 "Nour (ar)", 

343 "Omar (ar)", 

344 ] 

345 """The voice, with its language (default "Craig (en)").""" 

346 sample_rate_hertz: Literal[8000, 16000, 24000, 32000, 40000, 48000] 

347 """Samples a second (default 48000).""" 

348 

349 

350class Qwen(TypedDict, total=False, closed=True): 

351 """The settings of Alibaba's Qwen3-TTS 1.7B (`fal-ai/qwen-3-tts/text-to-speech/1.7b`).""" 

352 

353 voice: ( 

354 Literal[ 

355 "Vivian", 

356 "Serena", 

357 "Uncle_Fu", 

358 "Dylan", 

359 "Eric", 

360 "Ryan", 

361 "Aiden", 

362 "Ono_Anna", 

363 "Sohee", 

364 ] 

365 | None 

366 ) 

367 """The voice (default: the model's).""" 

368 language: Literal[ 

369 "Auto", 

370 "English", 

371 "Chinese", 

372 "Spanish", 

373 "French", 

374 "German", 

375 "Italian", 

376 "Japanese", 

377 "Korean", 

378 "Portuguese", 

379 "Russian", 

380 ] 

381 """The language (default "Auto": the text's).""" 

382 prompt: str | None 

383 """How to say it, in words (not what to say) (default: none).""" 

384 speaker_voice_embedding_file_url: str | None 

385 """A cloned voice: the URL of the speaker embedding `fal-ai/qwen-3-tts/clone-voice` 

386 made; it overrides `voice` and `prompt` (default: none).""" 

387 reference_text: str | None 

388 """What the cloned voice's recording says, which helps it sound like it (default: 

389 none).""" 

390 temperature: float | None 

391 """How varied the delivery is, above 0 and up to 1 (default 0.9).""" 

392 top_k: int | None 

393 """Sampling: from the k likeliest sounds (default 50).""" 

394 top_p: float | None 

395 """Sampling: from the likeliest sounds that make up p of the chance, 0 to 1 (default 1).""" 

396 repetition_penalty: float | None 

397 """How strongly it avoids repeating itself (default 1.05).""" 

398 subtalker_dosample: bool | None 

399 """Whether its second stage samples (default True).""" 

400 subtalker_temperature: float | None 

401 """Its second stage's temperature, 0 to 1 (default 0.9).""" 

402 subtalker_top_k: int | None 

403 """Its second stage's top k (default 50).""" 

404 subtalker_top_p: float | None 

405 """Its second stage's top p, 0 to 1 (default 1).""" 

406 max_new_tokens: int | None 

407 """The most sound it makes, in its codec's tokens, 1 to 8192 (default 8192 here: fal's 

408 own, 200, can cut a long line short).""" 

409 

410 

411type Model = Literal[ 

412 "fal-ai/elevenlabs/tts/eleven-v3", 

413 "fal-ai/minimax/speech-2.8-hd", 

414 "google/gemini-3.8-flash-tts", 

415 "fal-ai/inworld-tts", 

416 "fal-ai/qwen-3-tts/text-to-speech/1.7b", 

417] 

418"""The models `Fal` speaks through.""" 

419 

420 

421@dataclass(frozen=True, slots=True) 

422class _Spec: 

423 """How a model is asked: the input its text goes in, its settings, what is sent unless a 

424 setting says otherwise, and what is always sent.""" 

425 

426 text: str 

427 settings: type 

428 defaults: tuple[tuple[str, object], ...] = () 

429 fixed: tuple[tuple[str, object], ...] = () 

430 

431 

432_MODELS: dict[Model, _Spec] = { 

433 "fal-ai/elevenlabs/tts/eleven-v3": _Spec( 

434 "text", ElevenV3, fixed=(("timestamps", True),) 

435 ), 

436 "fal-ai/minimax/speech-2.8-hd": _Spec("prompt", MiniMax), 

437 "google/gemini-3.8-flash-tts": _Spec("prompt", Gemini), 

438 "fal-ai/inworld-tts": _Spec("text", Inworld), 

439 "fal-ai/qwen-3-tts/text-to-speech/1.7b": _Spec( 

440 "text", Qwen, defaults=(("max_new_tokens", 8192),) 

441 ), 

442} 

443 

444 

445# ── the voice ─────────────────────────────────────────────────────────────────────────────── 

446 

447 

448@dataclass(frozen=True, slots=True, init=False, repr=False) 

449class Fal: 

450 """A text-to-speech model on fal.ai, as a voice: `Fal(model, **settings)`, where the 

451 settings are the model's own ([`ElevenV3`][manimgx.audio.fal.ElevenV3] for the default). 

452 

453 Args: 

454 model: The model's ID on fal. 

455 key: fal's API key; by default `FAL_KEY`. 

456 **settings: The model's settings. 

457 

458 Examples: 

459 A scene's voice, in its class: 

460 

461 ```py 

462 voice = m.Fal( 

463 "fal-ai/minimax/speech-2.8-hd", 

464 voice_setting={"voice_id": "Calm_Woman", "speed": 1.1}, 

465 ) 

466 ``` 

467 """ 

468 

469 model: Model 

470 """The model's ID on fal.""" 

471 settings: dict[str, object] 

472 """The model's settings, as given.""" 

473 key: str | None = field(default=None, compare=False) 

474 """fal's API key, if given; else `FAL_KEY` is read as the voice speaks.""" 

475 

476 @overload 

477 def __init__( 

478 self, 

479 model: Literal["fal-ai/elevenlabs/tts/eleven-v3"] = ..., 

480 /, 

481 *, 

482 key: str | None = None, 

483 **settings: Unpack[ElevenV3], 

484 ) -> None: ... 

485 @overload 

486 def __init__( 

487 self, 

488 model: Literal["fal-ai/minimax/speech-2.8-hd"], 

489 /, 

490 *, 

491 key: str | None = None, 

492 **settings: Unpack[MiniMax], 

493 ) -> None: ... 

494 @overload 

495 def __init__( 

496 self, 

497 model: Literal["google/gemini-3.8-flash-tts"], 

498 /, 

499 *, 

500 key: str | None = None, 

501 **settings: Unpack[Gemini], 

502 ) -> None: ... 

503 @overload 

504 def __init__( 

505 self, 

506 model: Literal["fal-ai/inworld-tts"], 

507 /, 

508 *, 

509 key: str | None = None, 

510 **settings: Unpack[Inworld], 

511 ) -> None: ... 

512 @overload 

513 def __init__( 

514 self, 

515 model: Literal["fal-ai/qwen-3-tts/text-to-speech/1.7b"], 

516 /, 

517 *, 

518 key: str | None = None, 

519 **settings: Unpack[Qwen], 

520 ) -> None: ... 

521 def __init__( 

522 self, 

523 model: Model = "fal-ai/elevenlabs/tts/eleven-v3", 

524 /, 

525 *, 

526 key: str | None = None, 

527 **settings: object, 

528 ) -> None: 

529 object.__setattr__(self, "model", model) 

530 object.__setattr__( 

531 self, "settings", {k: _ordered(v) for k, v in sorted(settings.items())} 

532 ) 

533 object.__setattr__(self, "key", key) 

534 

535 def __repr__(self) -> str: 

536 """The call that makes it, without its key: what names it in the cache.""" 

537 given = "".join(f", {k}={v!r}" for k, v in self.settings.items()) 

538 return f"Fal({self.model!r}{given})" 

539 

540 def __call__(self, text: str) -> Speech: 

541 key = self.key or os.environ.get("FAL_KEY") 

542 if not key: 

543 raise RuntimeError( 

544 "no key for fal.ai: set FAL_KEY (fal's dashboard gives one), or give" 

545 " `key=`" 

546 ) 

547 spec = _MODELS[self.model] 

548 body = { 

549 **dict(spec.defaults), 

550 **self.settings, 

551 **dict(spec.fixed), 

552 spec.text: text, 

553 } 

554 reply = json.loads( 

555 _fetch( 

556 f"https://fal.run/{self.model}", body, {"Authorization": f"Key {key}"} 

557 ) 

558 ) 

559 audio = _fetch(reply["audio"]["url"]) # fal keeps it days, not forever: now 

560 return Speech( 

561 audio, text=text, words=timed(text, _pieces(reply.get("timestamps"))) 

562 ) 

563 

564 

565def _ordered(value: object) -> object: 

566 """`value` with every dict in it ordered by its keys: one voice, one name, however its 

567 settings were written.""" 

568 if isinstance(value, dict): 

569 return {k: _ordered(v) for k, v in sorted(value.items())} 

570 if isinstance(value, list): 

571 return [_ordered(v) for v in value] 

572 return value 

573 

574 

575def _pieces(stamps: object) -> list[tuple[str, float, float]]: 

576 """A model's timing of its text, as fal passes it on, as timed pieces of the text: items of 

577 a text, a start and an end (words or characters), or ElevenLabs' alignments (characters, 

578 with their starts and ends). Empty if there is none, or if it runs backwards: the words 

579 are then estimated.""" 

580 pieces: list[tuple[str, float, float]] = [] 

581 for item in stamps if isinstance(stamps, list) else (): 

582 if not isinstance(item, dict): 

583 continue 

584 if "characters" in item: 

585 pieces += [ 

586 (str(c), float(a), float(b)) 

587 for c, a, b in zip( 

588 item["characters"], 

589 item["character_start_times_seconds"], 

590 item["character_end_times_seconds"], 

591 strict=True, 

592 ) 

593 ] 

594 elif {"text", "start", "end"} <= item.keys(): 

595 pieces.append((str(item["text"]), float(item["start"]), float(item["end"]))) 

596 starts = [a for _, a, _ in pieces] 

597 valid = all(0 <= a <= b and math.isfinite(b) for _, a, b in pieces) 

598 return pieces if valid and starts == sorted(starts) else [] 

599 

600 

601def _fetch( 

602 url: str, 

603 body: dict[str, object] | None = None, 

604 headers: dict[str, str] | None = None, 

605) -> bytes: 

606 """GET `url`, or POST `body` to it as JSON; the response's bytes (an HTTP error says what 

607 the service said).""" 

608 request = urllib.request.Request( 

609 url, 

610 data=None if body is None else json.dumps(body).encode(), 

611 headers={"Content-Type": "application/json", **(headers or {})}, 

612 method="GET" if body is None else "POST", 

613 ) 

614 try: 

615 with urllib.request.urlopen(request, timeout=300) as response: 

616 return response.read() 

617 except urllib.error.HTTPError as error: 

618 detail = error.read().decode(errors="replace")[:500] 

619 raise RuntimeError(f"{url}: HTTP {error.code}: {detail}") from None