feat: change max token handling

cmd/paraclub-ai-mailer/main.go

@@ -12,6 +12,7 @@ import (
 	"paraclub-ai-mailer/internal/fetcher"
 	"paraclub-ai-mailer/internal/imap"
 	"paraclub-ai-mailer/internal/logger"
+	"paraclub-ai-mailer/internal/tokens"
 
 	"github.com/sirupsen/logrus"
 )
@@ -56,6 +57,12 @@ func main() {
 
 	fetcher := fetcher.New()
 	aiProcessor := ai.New(cfg.AI)
+
+	tokenCounter, err := tokens.New()
+	if err != nil {
+		logger.WithError(err).Error("Failed to initialize token counter")
+		os.Exit(1)
+	}
 	logger.Debug("All components initialized successfully")
 
 	// Setup signal handling for graceful shutdown
@@ -76,7 +83,7 @@ func main() {
 				return
 			case <-ticker.C:
 				logger.Debug("Processing tick started")
-				processEmails(imapClient, fetcher, aiProcessor, cfg)
+				processEmails(imapClient, fetcher, aiProcessor, tokenCounter, cfg)
 				logger.Debug("Processing tick completed")
 			}
 		}
@@ -89,7 +96,7 @@ func main() {
 	logger.Info("Application shutdown complete")
 }
 
-func processEmails(imapClient *imap.IMAPClient, fetcher *fetcher.Fetcher, aiProcessor *ai.AI, cfg *config.Config) {
+func processEmails(imapClient *imap.IMAPClient, fetcher *fetcher.Fetcher, aiProcessor *ai.AI, tokenCounter *tokens.TokenCounter, cfg *config.Config) {
 	logger.Debug("Starting email processing cycle")
 
 	// Fetch unprocessed emails
@@ -118,44 +125,55 @@ func processEmails(imapClient *imap.IMAPClient, fetcher *fetcher.Fetcher, aiProc
 	// Process each email
 	var processedCount, errorCount, skippedCount int
 	for _, email := range emails {
-		emailBodySize := len(email.Body)
 		logger.WithFields(logrus.Fields{
-			"subject":       email.Subject,
-			"from":          email.From,
-			"messageId":     email.ID,
-			"bodySizeBytes": emailBodySize,
+			"subject":   email.Subject,
+			"from":      email.From,
+			"messageId": email.ID,
 		}).Info("Processing email")
 
-		// Check email size limit
-		if cfg.Processing.MaxEmailSizeBytes > 0 && emailBodySize > cfg.Processing.MaxEmailSizeBytes {
+		// Extract clean email content (removes attachments, MIME boundaries, headers, converts HTML to text)
+		cleanEmailContent := imap.ExtractMessageContent(email.Body)
+
+		logger.WithFields(logrus.Fields{
+			"subject":    email.Subject,
+			"cleanSize":  len(cleanEmailContent),
+		}).Debug("Extracted clean email content")
+
+		// Calculate token count for validation (use English as default to avoid API call)
+		// Language detection will happen during actual GenerateReply
+		systemPrompt := aiProcessor.BuildSystemPrompt("English")
+		userPrompt := aiProcessor.BuildUserPrompt(contextContent, cleanEmailContent)
+		estimatedTokens := tokenCounter.EstimateFullRequest(systemPrompt, userPrompt)
+
+		logger.WithFields(logrus.Fields{
+			"subject":         email.Subject,
+			"estimatedTokens": estimatedTokens,
+			"maxTokens":       cfg.Processing.MaxTokens,
+		}).Debug("Calculated token estimate for email")
+
+		// Check token limit
+		if cfg.Processing.MaxTokens > 0 && estimatedTokens > cfg.Processing.MaxTokens {
 			logger.WithFields(logrus.Fields{
-				"subject":       email.Subject,
-				"from":          email.From,
-				"bodySizeBytes": emailBodySize,
-				"maxSizeBytes":  cfg.Processing.MaxEmailSizeBytes,
-			}).Warn("Email body exceeds size limit, skipping")
+				"subject":         email.Subject,
+				"from":            email.From,
+				"estimatedTokens": estimatedTokens,
+				"maxTokens":       cfg.Processing.MaxTokens,
+			}).Warn("Email exceeds token limit, marking as AI-processed but keeping in inbox")
+
 			skippedCount++
 
-			// Mark as AI-processed to prevent reprocessing
+			// Mark as AI-processed to prevent reprocessing, but DON'T move the email
 			if markErr := imapClient.MarkAsAIProcessed(email); markErr != nil {
 				logger.WithFields(logrus.Fields{
 					"subject": email.Subject,
 					"error":   markErr,
 				}).Error("Failed to mark oversized email as AI-processed")
+			} else {
+				logger.WithField("subject", email.Subject).Info("Marked oversized email as AI-processed (email remains in inbox)")
 			}
 			continue
 		}
 
-		// Extract clean email content (remove MIME boundaries, headers, etc.)
-		cleanEmailContent := imap.ExtractMessageContent(email.Body)
-		cleanContentSize := len(cleanEmailContent)
-		logger.WithFields(logrus.Fields{
-			"subject":           email.Subject,
-			"rawSize":           emailBodySize,
-			"cleanSize":         cleanContentSize,
-			"sizeReduction":     emailBodySize - cleanContentSize,
-		}).Debug("Extracted clean email content")
-
 		// Generate AI response with clean content
 		response, err := aiProcessor.GenerateReply(cleanEmailContent, contextContent)
 		if err != nil {