lindera-wasm-cc-cedict-bundler

import __wbg_init, { TokenizerBuilder } from 'lindera-wasm-cc-cedict-bundler'; async function main() { // Initialize the WASM module. This is crucial and must be awaited. // Failure to await will result in runtime errors. await __wbg_init(); // Create a new tokenizer builder instance. const builder = new TokenizerBuilder(); // Specify the embedded CC-CEDICT dictionary for Chinese analysis. // The 'embedded://' prefix indicates a dictionary bundled with the package. builder.setDictionary("embedded://cc-cedict"); // Set the tokenization mode. Common modes include "normal", "decompose", "search". builder.setMode("normal"); // Build the tokenizer instance from the configured builder. const tokenizer = builder.build(); // Text to tokenize in Chinese. const text = "他们说汉语。"; // "They speak Chinese." // Tokenize the text into an array of tokens. const tokens = tokenizer.tokenize(text); // Process and log the tokens with their details. console.log(`Tokenizing "${text}":`); tokens.forEach(token => { // Each token object contains the surface form and an array of linguistic details. console.log(`- Surface: "${token.surface}", Details: [${token.details.join(", ")}]`); // Example details for Chinese might include Pinyin, part of speech, etc. }); } // Run the main function and catch any potential errors during execution. main().catch(console.error);