lindera-wasm-cjk-bundler

import __wbg_init, { TokenizerBuilder, Token } from 'lindera-wasm-cjk-bundler'; async function main() { // Initialize the WebAssembly module. This must be called before using other functions. await __wbg_init(); // Create a new TokenizerBuilder instance. const builder = new TokenizerBuilder(); // Set the dictionary to be used. "embedded://cjk" specifies the bundled CJK dictionary. builder.setDictionary("embedded://cjk"); // Set the tokenization mode. "normal" is a common choice. builder.setMode("normal"); // Build the tokenizer instance. const tokenizer = builder.build(); // Text for morphological analysis (Japanese, Korean, Chinese examples). const text = "すもももももももものうち。안녕하세요.我爱北京天安门。"; // Perform tokenization. const tokens: Token[] = tokenizer.tokenize(text); console.log(`Tokens for "${text}":`); tokens.forEach(token => { // Log the surface form and detailed analysis of each token. console.log(`- ${token.surface}: ${token.details.join(", ")}`); }); // Example with another CJK language: Korean const koreanText = "한국어를 공부합니다."; const koreanTokens: Token[] = tokenizer.tokenize(koreanText); console.log(`\nTokens for "${koreanText}":`); koreanTokens.forEach(token => { console.log(`- ${token.surface}: ${token.details.join(", ")}`); }); // Example with another CJK language: Chinese const chineseText = "上海东方明珠广播电视塔"; const chineseTokens: Token[] = tokenizer.tokenize(chineseText); console.log(`\nTokens for "${chineseText}":`); chineseTokens.forEach(token => { console.log(`- ${token.surface}: ${token.details.join(", ")}`); }); } main().catch(console.error);