Created
July 19, 2024 04:17
-
-
Save djaffer/70d4c95aa706a440e4f188c9c68a13a2 to your computer and use it in GitHub Desktop.
Using Tiktoken WASM in React Environment Webpack
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ### To use tiktoken wasm in react for better performance. Hope it saves someone time due to poorly written guides. | |
| ### Follow this: | |
| **This is for webpack 5. In weppack config override add. You can directly use wasm from js cdn as well. Skip this step if you want to use cdn.** | |
| ``` | |
| module.exports = (config, env) => { | |
| config.experiments = { | |
| asyncWebAssembly: true, | |
| layers: true, | |
| }; | |
| } | |
| ``` | |
| **To use tiktoken with config override.** | |
| ``` | |
| import { get_encoding, init } from 'tiktoken/init'; | |
| import wasm from 'tiktoken/tiktoken_bg.wasm'; | |
| let tikInit = false; | |
| const countTokens= (str) =>{ | |
| if (!tikInit) { | |
| tikInit = true; | |
| const wasmFile = await fetch(wasm); | |
| const buffer = await wasmFile,arrayBuffer(); | |
| await init((imports) => WebAssembly.instantiate(buffer, imports)); | |
| } | |
| const e = get_encoding('cl100k_base'); | |
| const t = e.encode(str); | |
| e.free(); | |
| return t.length; | |
| } | |
| ``` | |
| **To use tiktoken without config override and js cdn** | |
| ``` | |
| import { get_encoding, init } from 'tiktoken/init'; | |
| import wasm from 'tiktoken/tiktoken_bg.wasm'; | |
| let tikInit = false; | |
| const countTokens= (str) =>{ | |
| if (!tikInit) { | |
| tikInit = true; | |
| } | |
| const wasmFile = await fetch( | |
| 'https://cdn.jsdelivr.net/npm/[email protected]/tiktoken_bg.wasm' | |
| ); | |
| const buffer = await wasmFile,arrayBuffer(); | |
| await init((imports) => WebAssembly.instantiate(buffer, imports)); | |
| } | |
| const e = get_encoding('cl100k_base'); | |
| const t = e.encode(str); | |
| e.free(); | |
| return t.length; | |
| } | |
| ``` |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment