Coverage for src/manimgx/drawing/typesetting.py: 97%

104 statements  

« prev     ^ index     » next       coverage.py v7.16.1, created at 2026-10-26 05:52 +0000

1"""Typst source → its layout, typeset in this process (`_engine.typeset`) and cached on disk. 

2A layout is one row per item (glyph or shape) in document order, each shape's curves, and each 

3labelled group's items; the layout owns each glyph's immutable outline and carets.""" 

4 

5import functools 

6import os 

7import pickle 

8import tempfile 

9from dataclasses import dataclass 

10from importlib.resources import files 

11from importlib.util import find_spec 

12from pathlib import Path 

13 

14import numpy as np 

15 

16from manimgx import _engine 

17from manimgx.caches import Memo, forgets 

18from manimgx.drawing.geometry import Shape 

19 

20# The fonts ManimGX ships, its font packages' (Noto): with Typst's own, every face a document is 

21# set in or falls back to, so text is the same on every machine; a system font only by name. The 

22# CJK one is not installed in the browser 

23FONTS = tuple( 

24 str(files(package)) 

25 for package in ("manimgx_fonts", "manimgx_fonts_cjk") 

26 if find_spec(package) is not None 

27) 

28TYPST_COMPILATION_FONT_SIZE = 10 

29TYPST_TEMPLATE = ( 

30 "#set page(width: auto, height: auto, margin: 0pt, fill: none)\n#set text(size:" 

31 " {text_size}pt)\n{preamble}\n" 

32) 

33TypstError = _engine.TypstError 

34"""The error raised when Typst can't typeset a document, or LaTeX can't be converted to 

35Typst: its message is Typst's (or mitex's), each error with the bytes of the source it 

36is at, counted from the start of the whole document typeset — a page setup and the 

37preamble come before the code.""" 

38 

39# a row's columns (see rust/engine/src/typeset.rs): an item's origin is its source node (start, 

40# end; its own, else the innermost element around it from the source) and the bytes of the 

41# node's text a glyph's cluster draws (start, end) 

42KIND, KEY, PLACEMENT, FILL, STROKE, WIDTH, ADVANCE, NODE, DRAWN = ( 

43 0, 

44 1, 

45 slice(2, 8), 

46 slice(8, 12), 

47 slice(12, 16), 

48 16, 

49 17, 

50 slice(18, 20), 

51 slice(20, 22), 

52) 

53ROW = 23 

54GLYPH, SHAPE = 0.0, 1.0 

55 

56 

57type Glyph = tuple[int, Shape | None, bytes] 

58 

59 

60@dataclass(frozen=True, slots=True) 

61class Layout: 

62 """A typeset drawing: `rows` (n × ROW, glyphs with outlines and shapes), `shapes` 

63 (cubic points, y up), `labels` (a label and its rows, including empty groups), 

64 `body` (where the body starts in the source, in bytes), and `glyphs` (the 

65 immutable outline and caret records its rows name).""" 

66 

67 rows: np.ndarray 

68 shapes: list[np.ndarray] 

69 labels: list[tuple[str, list[int]]] 

70 body: int 

71 glyphs: dict[int, Glyph] 

72 

73 

74# Reuse under disjoint native (<2^48) or content (>=2^52) keys. The bound counts 

75# aliases; either can be forgotten independently. Live layouts own their records. 

76_GLYPHS: Memo[int, Glyph] = Memo(1 << 12) 

77_CACHE = ( 

78 Path(directory) / "layouts" 

79 if (directory := _engine.cache_directory()) is not None 

80 else None 

81) 

82# what made a layout, part of its key: the entry's format, and the engine's own file (a rebuilt 

83# engine never reads a layout an older one made) 

84_ENGINE = Path(_engine.__file__).stat() 

85_VERSION = f"layout 4, engine {_ENGINE.st_size} {_ENGINE.st_mtime_ns}".encode() 

86_CONTENT = 1 << 52 # glyph keys by content: exact in float64, apart from the engine's 

87type Stored = tuple[ 

88 np.ndarray, 

89 list[np.ndarray], 

90 list[tuple[str, list[int]]], 

91 int, 

92 dict[int, tuple[bytes, bytes]], 

93] 

94 

95 

96def typst_string(text: str) -> str: 

97 """Write `text` as a Typst string literal, which Typst reads back as exactly `text`. 

98 

99 Args: 

100 text: Any text. 

101 

102 Returns: 

103 The literal, its quotes included. 

104 """ 

105 escaped = ( 

106 text.replace("\\", "\\\\") 

107 .replace('"', '\\"') 

108 .replace("\n", "\\n") 

109 .replace("\r", "\\r") 

110 ) 

111 return f'"{escaped}"' 

112 

113 

114def typeset( 

115 body: str, 

116 preamble: str = "", 

117 text_size: float = TYPST_COMPILATION_FONT_SIZE, 

118 font_paths: list[str | Path] | None = None, 

119 package_path: str | Path | None = None, 

120) -> Layout: 

121 """Typeset a Typst document, and read its layout off Typst's frames. 

122 

123 The document is set in the fonts ManimGX ships (Typst's own and the Noto faces), 

124 after those in `font_paths`; a system font is read only for a family the document 

125 names that none of them has. A layout is kept in the current user's cache directory 

126 (the browser's private filesystem in a page): the same document, with the same fonts 

127 and packages, is read back, not typeset again, in any process. 

128 

129 Args: 

130 body: The document's body: Typst markup. 

131 preamble: Typst code set before the body. 

132 text_size: The text's size, in points. 

133 font_paths: Directories of font files, searched, with their subdirectories, 

134 before the fonts ManimGX ships. 

135 package_path: The directory Typst packages are imported from, besides mitex 

136 (the engine's); None for none but mitex. 

137 """ 

138 prefix = TYPST_TEMPLATE.format(text_size=text_size, preamble=preamble) 

139 source = prefix + body + "\n" 

140 fonts = (*(str(p) for p in font_paths or ()), *FONTS) 

141 packages = None if package_path is None else str(package_path) 

142 depends = _files(fonts if packages is None else (*fonts, packages)) 

143 key = _engine.digest(_VERSION, b"\0", source.encode(), b"\0", depends) 

144 path = None if _CACHE is None else _CACHE / f"{key:016x}.layout" 

145 stored = None if path is None else _read(path) 

146 if stored is None: 

147 stored = _typeset(source, fonts, packages, int.from_bytes(depends, "little")) 

148 if path is not None: 

149 _write(path, stored) 

150 rows, shapes, labels, _, glyphs = stored 

151 records = { 

152 key: _glyph(key, points, carets) for key, (points, carets) in glyphs.items() 

153 } 

154 return Layout(rows, shapes, labels, len(prefix.encode()), records) 

155 

156 

157def _glyph(key: int, points: bytes, carets: bytes) -> Glyph: 

158 """Decode one immutable record, reusing it while remembered.""" 

159 record = _GLYPHS.get(key) 

160 if record is None: 

161 array = np.frombuffer(points).reshape(-1, 3) 

162 record = _GLYPHS.keep( 

163 key, (key, Shape(array.copy()) if len(array) else None, carets) 

164 ) 

165 return record 

166 

167 

168@forgets 

169@functools.cache 

170def _files(paths: tuple[str, ...]) -> bytes: 

171 """The files a layout depends on besides its source (fonts, packages), as paths, sizes and 

172 times, once per process.""" 

173 found = [] 

174 for root in map(Path, paths): 

175 found.append(f"{root}\0".encode()) 

176 within = sorted(root.rglob("*")) if root.is_dir() else [root] 

177 for file in within: 

178 if file.is_file(): 

179 stat = file.stat() 

180 found.append(f"{file}\0{stat.st_size}\0{stat.st_mtime_ns}\0".encode()) 

181 return _engine.digest(*found).to_bytes(8, "little") 

182 

183 

184def _read(path: Path) -> Stored | None: 

185 """A stored layout, if there is one and the fonts it was set with haven't changed.""" 

186 try: 

187 stored: Stored = pickle.loads(path.read_bytes()) 

188 except (OSError, pickle.UnpicklingError, EOFError, ValueError): 

189 return None 

190 system = stored[3] 

191 if system and system != _engine.system_fonts_fingerprint(): 

192 return None 

193 return stored 

194 

195 

196def _write(path: Path, stored: Stored) -> None: 

197 """Store a layout: written whole, then renamed, so a reader never sees half of one.""" 

198 try: 

199 path.parent.mkdir(parents=True, exist_ok=True) 

200 with tempfile.TemporaryDirectory(dir=path.parent) as own: 

201 partial = Path(own) / "layout" 

202 partial.write_bytes(pickle.dumps(stored, protocol=5)) 

203 os.replace(partial, path) 

204 except OSError: 

205 pass 

206 

207 

208def _typeset( 

209 source: str, fonts: tuple[str, ...], packages: str | None, revision: int 

210) -> Stored: 

211 """Typeset with Typst; every glyph's key by content, with its outline and carets along.""" 

212 raw, shapes, labels, system = _engine.typeset( 

213 source, fonts, packages, revision=revision 

214 ) 

215 rows = np.frombuffer(raw).reshape(-1, ROW).copy() 

216 glyphs = rows[:, KIND] == GLYPH 

217 keys = rows[glyphs, KEY].astype(np.int64).tolist() 

218 held = {k: record for k in set(keys) if (record := _GLYPHS.get(k)) is not None} 

219 new = sorted(set(keys) - held.keys()) 

220 for k, data in zip(new, _engine.glyph_outlines(new)): 

221 carets = np.array(_engine.ligature_carets(k), dtype="<f8").tobytes() 

222 content = _CONTENT | ( 

223 _engine.digest( 

224 len(data).to_bytes(8, "little"), 

225 data, 

226 carets, 

227 ) 

228 & (_CONTENT - 1) 

229 ) 

230 held[k] = _GLYPHS.keep(k, _glyph(content, data, carets)) 

231 records = {record[0]: record for record in held.values()} 

232 rows[glyphs, KEY] = [held[k][0] for k in keys] 

233 drawn = [ 

234 i 

235 for i, row in enumerate(rows) 

236 if row[KIND] != GLYPH or records[int(row[KEY])][1] is not None 

237 ] 

238 if len(drawn) != len(rows): 

239 row_of = {old: new for new, old in enumerate(drawn)} 

240 rows = rows[drawn] 

241 labels = [ 

242 (label, [row_of[i] for i in members if i in row_of]) 

243 for label, members in labels 

244 ] 

245 glyphs = { 

246 key: (b"" if shape is None else shape.array.tobytes(), carets) 

247 for key, shape, carets in records.values() 

248 } 

249 return ( 

250 rows, 

251 [np.frombuffer(s).reshape(-1, 3) for s in shapes], 

252 labels, 

253 _engine.system_fonts_fingerprint() if system else 0, 

254 glyphs, 

255 )