/*******************************************************
 * Copyright (C) 2018 XMind Ltd. - All Rights Reserved
 *******************************************************/

#ifndef TOKENIZER_H
#define TOKENIZER_H

#include <napi.h>

/**
 * Tokenize a string into words with linguistic information
 *
 * @param info The callback info containing the input string to tokenize
 * @return An array of token objects with linguistic information
 */
Napi::Value Tokenize(const Napi::CallbackInfo& info);

#endif // TOKENIZER_H