//调用 transformers 计算 token 数: doc://example/Languages/Python/Python%203.x/tokenizer.html
import win.ui;
/*DSG{{*/
var winform = win.form(text="大模型 token 数快速估算工具";right=759;bottom=469)
winform.add(
edit={cls="richedit";left=12;top=9;right=747;bottom=418;db=1;dl=1;dr=1;dt=1;edge=1;hscroll=1;multiline=1;vscroll=1;z=1};
radioGpt={cls="radiobutton";text="GPT";left=465;top=428;right=532;bottom=456;z=3};
radioMmseg={cls="radiobutton";text="通用";left=554;top=428;right=624;bottom=454;checked=1;z=4};
richedit={cls="richedit";text="0 tokens";left=18;top=432;right=393;bottom=451;db=1;dl=1;dr=1;multiline=1;transparent=1;z=2}
)
/*}}*/

import mmseg; 
import string.gpt;
import win.debounce;
import math.format;
winform.edit.onChange = win.debounce( function(){ 
	var str = winform.edit.text;
	if(!#str) return;
	
	var tokens;
	if(winform.radioGpt.checked){
		tokens = string.gpt.tokens(winform.edit.text);
	}
	else{
		tokens = mmseg.llmTokens(winform.edit.text);
	}
	
	winform.richedit.text = math.format.size(tokens,"≈ %s%s tokens");
	
},500);

winform.radioMmseg.oncommand = winform.edit.onChange;
winform.radioGpt.oncommand = winform.edit.onChange;

winform.edit.limit = -1;

winform.edit.enablePopMenu();

winform.show();
win.loopMessage();

/*
// 调用 Gemini 接口计算 token 数
var http = web.rest.jsonClient();
http.setHeaders({"x-goog-api-key":'api_key'}); 
var gemini = http.api("https://generativelanguage.googleapis.com/v1beta/{resource}/{model}:{method}");

var resp,err = gemini.models["gemini-2.5-flash"].countTokens({
	contents = {
		{
			parts = {
				{text = prompt}
			}
		}
	},
});

var totalTokens = resp[["totalTokens"]]
*/

/*
// 调用 GLM 接口计算 token 数
import web.rest.jsonClient;
var http = web.rest.jsonClient();
http.setAuthToken('api_key');

var glm = http.api("https://open.bigmodel.cn/api/paas/v4");
var resp,err = glm.tokenizer({
  "model": "glm-4-plus",	
  "messages": [
    {
      "role": "user",
      "content": prompt
    }
  ]
})

var totalTokens = resp[["usage"]][["prompt_tokens"]]
*/