feat: change token calculation

lua/chatgpt_nvim/init.lua

@@ -179,8 +179,16 @@ end
 
 -- New token estimation function.
 local function estimate_token_count(text)
-  -- Use a simple heuristic: assume an average of 4 characters per token.
+  -- Instead of simply dividing the character count by 4,
-  return math.floor(#text / 4)
+  -- split the text into non-whitespace chunks and further split each chunk
+  -- into sequences of alphanumeric characters and punctuation.
+  local token_count = 0
+  for chunk in text:gmatch("%S+") do
+    for token in chunk:gmatch("(%w+|%p+)") do
+      token_count = token_count + 1
+    end
+  end
+  return token_count
 end
 
 local function handle_step_by_step_if_needed(prompt, conf)