feat: better handling of oversized mails, and add junk option

cmd/paraclub-ai-mailer/main.go

@@ -116,22 +116,64 @@ func processEmails(imapClient *imap.IMAPClient, fetcher *fetcher.Fetcher, aiProc
 	logger.WithField("contextCount", len(contextContent)).Debug("Successfully fetched context content")
 
 	// Process each email
-	var processedCount, errorCount int
+	var processedCount, errorCount, skippedCount int
 	for _, email := range emails {
+		emailBodySize := len(email.Body)
 		logger.WithFields(logrus.Fields{
-			"subject":   email.Subject,
-			"from":      email.From,
-			"messageId": email.ID,
+			"subject":       email.Subject,
+			"from":          email.From,
+			"messageId":     email.ID,
+			"bodySizeBytes": emailBodySize,
 		}).Info("Processing email")
 
-		// Generate AI response
-		response, err := aiProcessor.GenerateReply(email.Body, contextContent)
+		// Check email size limit
+		if cfg.Processing.MaxEmailSizeBytes > 0 && emailBodySize > cfg.Processing.MaxEmailSizeBytes {
+			logger.WithFields(logrus.Fields{
+				"subject":       email.Subject,
+				"from":          email.From,
+				"bodySizeBytes": emailBodySize,
+				"maxSizeBytes":  cfg.Processing.MaxEmailSizeBytes,
+			}).Warn("Email body exceeds size limit, skipping")
+			skippedCount++
+
+			// Mark as AI-processed to prevent reprocessing
+			if markErr := imapClient.MarkAsAIProcessed(email); markErr != nil {
+				logger.WithFields(logrus.Fields{
+					"subject": email.Subject,
+					"error":   markErr,
+				}).Error("Failed to mark oversized email as AI-processed")
+			}
+			continue
+		}
+
+		// Extract clean email content (remove MIME boundaries, headers, etc.)
+		cleanEmailContent := imap.ExtractMessageContent(email.Body)
+		cleanContentSize := len(cleanEmailContent)
+		logger.WithFields(logrus.Fields{
+			"subject":           email.Subject,
+			"rawSize":           emailBodySize,
+			"cleanSize":         cleanContentSize,
+			"sizeReduction":     emailBodySize - cleanContentSize,
+		}).Debug("Extracted clean email content")
+
+		// Generate AI response with clean content
+		response, err := aiProcessor.GenerateReply(cleanEmailContent, contextContent)
 		if err != nil {
 			logger.WithFields(logrus.Fields{
 				"subject": email.Subject,
 				"error":   err,
 			}).Error("Failed to generate reply")
 			errorCount++
+
+			// Mark as AI-processed even on failure to prevent infinite retry loop
+			if markErr := imapClient.MarkAsAIProcessed(email); markErr != nil {
+				logger.WithFields(logrus.Fields{
+					"subject": email.Subject,
+					"error":   markErr,
+				}).Error("Failed to mark failed email as AI-processed")
+			} else {
+				logger.WithField("subject", email.Subject).Info("Marked failed email as AI-processed to prevent reprocessing")
+			}
 			continue
 		}
 		logger.WithField("responseLength", len(response)).Debug("Generated AI response")
@@ -180,5 +222,6 @@ func processEmails(imapClient *imap.IMAPClient, fetcher *fetcher.Fetcher, aiProc
 		"totalEmails": len(emails),
 		"processed":   processedCount,
 		"errors":      errorCount,
+		"skipped":     skippedCount,
 	}).Info("Completed email processing cycle")
 }