lindera-wasm-ipadic-bundler

import init, { TokenizerBuilder } from 'lindera-wasm-ipadic-bundler'; async function main() { // Initialize the WebAssembly module. This is crucial and must be awaited // before any other functions from the module can be used. await init(); // Create a new TokenizerBuilder instance to configure the tokenizer. const builder = new TokenizerBuilder(); // Specify the dictionary to use. 'embedded://ipadic' uses the dictionary // bundled with this package. builder.setDictionary("embedded://ipadic"); // Set the tokenization mode, 'normal' is suitable for general text. builder.setMode("normal"); // Build the tokenizer with the specified settings. const tokenizer = builder.build(); // Define the Japanese sentence to be tokenized. const sentence = "すもももももももものうち"; const tokens = tokenizer.tokenize(sentence); console.log(`Tokenizing sentence: "${sentence}"`); console.log("--- Tokens ---"); // Iterate over the resulting tokens and print their surface form and details. tokens.forEach(token => { console.log(`${token.surface} [${token.details.join(", ")}]`); }); // Demonstrate accessing specific details of a token. if (tokens.length > 0) { const firstToken = tokens[0]; console.log(`\nFirst token surface: ${firstToken.surface}`); // The details array contains information like part-of-speech, conjugation, etc. console.log(`First token part of speech: ${firstToken.details[0]}`); } } // Execute the main asynchronous function. main().catch(console.error);