lindera-wasm-jieba-bundler

import __wbg_init, { TokenizerBuilder, type Token } from 'lindera-wasm-jieba-bundler'; async function main() { // Initialize the WebAssembly module. This must be awaited. await __wbg_init(); // Create a new TokenizerBuilder instance. const builder = new TokenizerBuilder(); // Specify the Jieba dictionary to use. // 'embedded://jieba' refers to the dictionary bundled with this package. builder.setDictionary("embedded://jieba"); // Set the tokenization mode. 'normal' is a common default. builder.setMode("normal"); // Build the tokenizer. const tokenizer = builder.build(); // Text to tokenize. const textToAnalyze = "上海东方明珠广播电视塔"; // Perform tokenization. const tokens: Token[] = tokenizer.tokenize(textToAnalyze); console.log(`Tokens for: "${textToAnalyze}"`); tokens.forEach(token => { // Each token has a surface form and detailed information. console.log(`- ${token.surface}: ${token.details.join(" | ")}`); }); // Example with another Chinese sentence const anotherText = "我爱北京天安门"; const moreTokens: Token[] = tokenizer.tokenize(anotherText); console.log(`\nTokens for: "${anotherText}"`); moreTokens.forEach(token => { console.log(`- ${token.surface}: ${token.details.join(" | ")}`); }); } main().catch(console.error);