Add IMAP settings to configuration and enhance email processing logic

README.md

@@ -42,6 +42,16 @@ go build -o paraclub-ai-mailer ./cmd/paraclub-ai-mailer
 
 The application uses a YAML configuration file (`config.yaml`) with the following structure:
 
+### IMAP Settings
+- `server`: IMAP server hostname (e.g., "imap.gmail.com")
+- `port`: IMAP port (typically 993 for TLS)
+- `username`: Your email address
+- `password`: Email password (use ${IMAP_PASSWORD} to load from environment)
+- `mailbox_in`: Mailbox to check for new emails (e.g., "INBOX")
+- `draft_box`: Folder to save AI-generated drafts (e.g., "Drafts")
+- `processed_box`: Folder to move processed emails to (they will also be marked as read)
+- `use_tls`: Whether to use TLS for connection (recommended: true)
+
 ```yaml
 imap:
   server: "imap.example.com"
@@ -111,7 +121,7 @@ Or specify a custom config file path:
    - Retrieves HTML content from configured URLs
    - Sends combined content to OpenRouter AI for response generation
    - Saves the AI-generated response as a draft email
-   - Marks the original email as processed
+   - Marks the original email as read and moves it to the configured processed folder
 
 ## Logging
 

config.yaml.example

@@ -5,6 +5,7 @@ imap:
   password: "${IMAP_PASSWORD}"  # Will be read from environment variable
   mailbox_in: "INBOX"
   draft_box: "Drafts"
+  processed_box: "Processed"  # Folder where processed emails will be moved
   use_tls: true
 
 ai:

config/config.go

@@ -19,13 +19,14 @@ type Config struct {
 }
 
 type IMAPConfig struct {
-	Server    string `yaml:"server"`
-	Port      int    `yaml:"port"`
-	Username  string `yaml:"username"`
-	Password  string `yaml:"password"`
-	MailboxIn string `yaml:"mailbox_in"`
-	DraftBox  string `yaml:"draft_box"`
-	UseTLS    bool   `yaml:"use_tls"`
+	Server       string `yaml:"server"`
+	Port         int    `yaml:"port"`
+	Username     string `yaml:"username"`
+	Password     string `yaml:"password"`
+	MailboxIn    string `yaml:"mailbox_in"`
+	DraftBox     string `yaml:"draft_box"`
+	ProcessedBox string `yaml:"processed_box"`
+	UseTLS       bool   `yaml:"use_tls"`
 }
 
 type AIConfig struct {

internal/ai/ai.go

@@ -43,12 +43,40 @@ func New(cfg config.AIConfig) *AI {
 	}
 }
 
+func (a *AI) detectLanguage(emailContent string) (string, error) {
+	logger.WithField("emailContentLength", len(emailContent)).Debug("Starting language detection")
+
+	systemMsg := "You are a language detection assistant. Analyze the provided text and respond ONLY with the language name (e.g., 'English', 'German', 'French', etc.). No other text or explanation."
+	userMsg := fmt.Sprintf("Detect the language of this text:\n\n%s", emailContent)
+
+	messages := []Message{
+		{Role: "system", Content: systemMsg},
+		{Role: "user", Content: userMsg},
+	}
+
+	response, err := a.makeAPIRequest(messages)
+	if err != nil {
+		return "", fmt.Errorf("language detection failed: %v", err)
+	}
+
+	// The response should be just the language code
+	langCode := response
+	logger.WithField("detectedLanguage", langCode).Debug("Language detection completed")
+	return langCode, nil
+}
+
 func (a *AI) GenerateReply(emailContent string, contextContent map[string]string) (string, error) {
 	logger.WithFields(logrus.Fields{
 		"emailContentLength": len(emailContent),
-		"contextUrls":       len(contextContent),
+		"contextUrls":        len(contextContent),
 	}).Debug("Starting AI reply generation")
 
+	// First, detect the language
+	lang, err := a.detectLanguage(emailContent)
+	if err != nil {
+		return "", err
+	}
+
 	// Prepare context from all URLs
 	var context string
 	for url, content := range contextContent {
@@ -59,19 +87,22 @@ func (a *AI) GenerateReply(emailContent string, contextContent map[string]string
 		}).Debug("Added context from URL")
 	}
 
-	// Prepare the system message and user message
-	// systemMsg := "You are a helpful assistant responding to emails. Analyze the language of the incoming email and translate the response to the same language. Format your response in HTML. Do not include any explanations or extra text - just write the email response directly in HTML format. Use appropriate HTML tags for formatting."
-  // systemMsg := "You are a helpful assistant who responds to emails. For each incoming email, first detect its language and then generate your response in the exact same language. Your reply must be written directly in HTML format using appropriate HTML tags for structure and styling. Do not include any explanations, commentary, or extra text—simply output the email response in HTML."
-  // systemMsg := "You are a helpful assistant who responds to emails. For each incoming email, first detect its language and then generate your response in the exact same language. Your reply must be written directly in HTML format using appropriate HTML tags for structure and styling. Do not include a subject line, explanations, commentary, or any extra text—simply output the email response content in HTML."
-  // systemMsg := "You are a helpful assistant who responds to emails. For every incoming email, carefully detect and confirm its language. Then, generate your email response entirely in that same language without deviation. Format your reply solely in HTML using appropriate HTML tags for structure and styling, and do not include a subject line, explanations, or any extra text. Ensure that every part of your response exactly matches the language of the incoming email."
-  systemMsg := "You are a helpful assistant who responds to emails. For every incoming email, strictly use only the email's content to detect its language and ignore any external or additional context. Then, generate your email response entirely in that same language without deviation. Format your reply solely in HTML using appropriate HTML tags for structure and styling, and do not include a subject line, explanations, commentary, or any extra text. Ensure that every part of your response exactly matches the language of the incoming email."
-	userMsg := fmt.Sprintf("Using the following context:\n%s\n\nPlease analyze the language of and generate a reply in the same language for this email:\n%s", context, emailContent)
+	// Prepare the system message with language-specific instruction
+	systemMsg := fmt.Sprintf("You are a helpful assistant who responds to emails.Regardless of the language used in the email content or context, your response must be entirely in %s. Format your reply solely in HTML using appropriate HTML tags for structure and styling. Do not include a subject line, explanations, commentary, or any extra text.", lang)
+	logger.WithFields(logrus.Fields{
+		"systemprompt": systemMsg,
+	}).Debug("Generating system prompt")
+	userMsg := fmt.Sprintf("### Additional Context:\n%s\n\n### Email Body:\n%s", context, emailContent)
 
 	messages := []Message{
 		{Role: "system", Content: systemMsg},
 		{Role: "user", Content: userMsg},
 	}
 
+	return a.makeAPIRequest(messages)
+}
+
+func (a *AI) makeAPIRequest(messages []Message) (string, error) {
 	const maxRetries = 3
 	var lastErr error
 

internal/imap/imap.go

@@ -18,9 +18,8 @@ import (
 )
 
 type IMAPClient struct {
-	client    *client.Client
-	config    config.IMAPConfig
-	processed map[string]bool
+	client *client.Client
+	config config.IMAPConfig
 }
 
 // MessageLiteral implements imap.Literal for draft messages
@@ -65,9 +64,8 @@ func New(cfg config.IMAPConfig) (*IMAPClient, error) {
 	}
 
 	return &IMAPClient{
-		client:    c,
-		config:    cfg,
-		processed: make(map[string]bool),
+		client: c,
+		config: cfg,
 	}, nil
 }
 
@@ -137,8 +135,9 @@ func (ic *IMAPClient) FetchUnprocessedEmails() ([]Email, error) {
 		return nil, nil
 	}
 
+	// Get all messages in the inbox that haven't been seen yet
 	criteria := imap.NewSearchCriteria()
-	criteria.WithoutFlags = []string{"\\Seen", "Processed"}
+	criteria.WithoutFlags = []string{"\\Seen"}
 
 	uids, err := ic.client.Search(criteria)
 	if err != nil {
@@ -170,11 +169,6 @@ func (ic *IMAPClient) FetchUnprocessedEmails() ([]Email, error) {
 			continue
 		}
 
-		// Skip if already processed
-		if ic.processed[msg.Envelope.MessageId] {
-			continue
-		}
-
 		// Skip if no message ID
 		if msg.Envelope.MessageId == "" {
 			continue
@@ -202,8 +196,6 @@ func (ic *IMAPClient) FetchUnprocessedEmails() ([]Email, error) {
 			From:    from,
 			Body:    bodyBuilder.String(),
 		})
-
-		ic.processed[msg.Envelope.MessageId] = true
 	}
 
 	if err := <-done; err != nil {
@@ -218,7 +210,13 @@ func (ic *IMAPClient) SaveDraft(email Email, response string) error {
 		return fmt.Errorf("failed to ensure IMAP connection: %v", err)
 	}
 
-	_, err := ic.client.Select(ic.config.DraftBox, false)
+	// Get proper folder path and ensure it exists
+	draftFolder, err := ic.ensureFolder(ic.config.DraftBox)
+	if err != nil {
+		return fmt.Errorf("failed to ensure draft folder exists: %v", err)
+	}
+
+	_, err = ic.client.Select(draftFolder, false)
 	if err != nil {
 		return fmt.Errorf("failed to select draft box: %v", err)
 	}
@@ -255,9 +253,9 @@ func (ic *IMAPClient) SaveDraft(email Email, response string) error {
 		pos:     0,
 	}
 
-	// Save the draft
+	// Save the draft to the proper folder path
 	flags := []string{"\\Draft"}
-	if err := ic.client.Append(ic.config.DraftBox, flags, time.Now(), literal); err != nil {
+	if err := ic.client.Append(draftFolder, flags, time.Now(), literal); err != nil {
 		return fmt.Errorf("failed to append draft: %v", err)
 	}
 
@@ -268,7 +266,6 @@ func (ic *IMAPClient) SaveDraft(email Email, response string) error {
 // by removing email headers and MIME boundaries
 func extractMessageContent(body string) string {
 	logger.WithField("bodyLength", len(body)).Debug("Starting message content extraction")
-
 	msg, err := mail.ReadMessage(strings.NewReader(body))
 	if err != nil {
 		logger.WithFields(logrus.Fields{
@@ -295,89 +292,106 @@ func extractMessageContent(body string) string {
 		"params":    params,
 	}).Debug("Parsed message Content-Type")
 
+	var content string
 	if strings.HasPrefix(mediaType, "multipart/") {
-		boundary := params["boundary"]
-		if boundary == "" {
-			logger.WithField("mediaType", mediaType).Debug("No boundary found in multipart message, falling back to simple extraction")
-			return fallbackExtractContent(body)
-		}
-
-		logger.WithFields(logrus.Fields{
-			"mediaType": mediaType,
-			"boundary":  boundary,
-		}).Debug("Processing multipart message")
-
-		reader := multipart.NewReader(msg.Body, boundary)
-		var textContent string
-		partIndex := 0
-
-		for {
-			part, err := reader.NextPart()
-			if err == io.EOF {
-				logger.Debug("Finished processing all multipart parts")
-				break
-			}
-			if err != nil {
-				logger.WithFields(logrus.Fields{
-					"error":     err,
-					"partIndex": partIndex,
-				}).Debug("Error reading multipart part, falling back to simple extraction")
-				return fallbackExtractContent(body)
-			}
-
-			contentType := part.Header.Get("Content-Type")
-			logger.WithFields(logrus.Fields{
-				"partIndex":   partIndex,
-				"contentType": contentType,
-			}).Debug("Processing message part")
-
-			if strings.HasPrefix(contentType, "text/plain") {
-				buf := new(bytes.Buffer)
-				_, err := buf.ReadFrom(part)
-				if err != nil {
-					logger.WithFields(logrus.Fields{
-						"error":     err,
-						"partIndex": partIndex,
-					}).Debug("Failed to read text/plain part")
-					continue
-				}
-				textContent = buf.String()
-				logger.WithFields(logrus.Fields{
-					"partIndex":     partIndex,
-					"contentLength": len(textContent),
-				}).Debug("Successfully extracted text/plain content")
-				textContent = strings.ReplaceAll(textContent, "\n", "<br>\n")
-				textContent = strings.ReplaceAll(textContent, "\r\n", "<br>\n")
-				break
-			}
-			partIndex++
-		}
-
-		if textContent != "" {
-			logger.WithField("contentLength", len(textContent)).Debug("Successfully extracted content from multipart message")
-			return textContent
-		}
-		logger.Debug("No text/plain content found in multipart message, trying to read body directly")
+		content = handleMultipartMessage(msg.Body, params["boundary"])
+	} else {
+		content = handleSinglePartMessage(msg.Body)
 	}
 
-	buf := new(bytes.Buffer)
-	_, err = buf.ReadFrom(msg.Body)
-	if err != nil {
-		logger.WithFields(logrus.Fields{
-			"error":     err,
-			"mediaType": mediaType,
-		}).Debug("Failed to read message body, falling back to simple extraction")
+	if content == "" {
+		logger.Debug("No content extracted, falling back to simple extraction")
 		return fallbackExtractContent(body)
 	}
 
-	content := buf.String()
+	// Clean up the content
+	content = cleanMessageContent(content)
+
+	logger.WithField("contentLength", len(content)).Debug("Successfully extracted and cleaned message content")
+	return content
+}
+
+func handleMultipartMessage(reader io.Reader, boundary string) string {
+	if boundary == "" {
+		logger.Debug("No boundary found in multipart message")
+		return ""
+	}
+
+	mReader := multipart.NewReader(reader, boundary)
+	var textContent string
+	partIndex := 0
+
+	for {
+		part, err := mReader.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			logger.WithError(err).Debug("Error reading multipart part")
+			return ""
+		}
+
+		contentType := part.Header.Get("Content-Type")
+		if strings.HasPrefix(contentType, "text/plain") {
+			buf := new(bytes.Buffer)
+			if _, err := buf.ReadFrom(part); err != nil {
+				continue
+			}
+			textContent = buf.String()
+			break
+		}
+		partIndex++
+	}
+
+	return textContent
+}
+
+func handleSinglePartMessage(reader io.Reader) string {
+	buf := new(bytes.Buffer)
+	if _, err := buf.ReadFrom(reader); err != nil {
+		logger.WithError(err).Debug("Failed to read message body")
+		return ""
+	}
+	return buf.String()
+}
+
+func cleanMessageContent(content string) string {
+	// Remove any remaining email headers that might be in the body
+	lines := strings.Split(content, "\n")
+	var cleanLines []string
+	headerSection := true
+
+	for _, line := range lines {
+		trimmed := strings.TrimSpace(line)
+
+		// Empty line marks the end of headers
+		if headerSection && trimmed == "" {
+			headerSection = false
+			continue
+		}
+
+		// Skip header lines
+		if headerSection && (strings.Contains(trimmed, ":") || trimmed == "") {
+			continue
+		}
+
+		// Add non-header lines
+		if !headerSection {
+			cleanLines = append(cleanLines, line)
+		}
+	}
+
+	content = strings.Join(cleanLines, "\n")
+
+	// Convert newlines to HTML breaks for display
 	content = strings.ReplaceAll(content, "\r\n", "<br>\n")
 	content = strings.ReplaceAll(content, "\n", "<br>\n")
-	logger.WithFields(logrus.Fields{
-		"contentLength": len(content),
-		"mediaType":     mediaType,
-	}).Debug("Successfully extracted content from message body")
-	return content
+
+	// Remove any remaining email signature markers
+	content = strings.Split(content, "\n-- ")[0]
+	content = strings.Split(content, "<br>-- ")[0]
+
+	return strings.TrimSpace(content)
 }
 
 // fallbackExtractContent is the previous implementation used as fallback
@@ -404,32 +418,106 @@ func fallbackExtractContent(body string) string {
 	return content
 }
 
+// ensureFolder makes sure a folder exists and returns its full path using proper delimiters
+func (ic *IMAPClient) ensureFolder(folderName string) (string, error) {
+	// List all mailboxes to get the delimiter
+	mailboxes := make(chan *imap.MailboxInfo, 10)
+	done := make(chan error, 1)
+	go func() {
+		done <- ic.client.List("", "*", mailboxes)
+	}()
+
+	var delimiter string
+	for m := range mailboxes {
+		delimiter = m.Delimiter
+		break // We just need the first one to get the delimiter
+	}
+
+	if err := <-done; err != nil {
+		return "", fmt.Errorf("failed to list mailboxes: %v", err)
+	}
+
+	if delimiter == "" {
+		delimiter = "/" // fallback to common delimiter
+	}
+
+	// Replace any forward slashes with the server's delimiter
+	folderPath := strings.ReplaceAll(folderName, "/", delimiter)
+
+	// Try to create the folder if it doesn't exist
+	if err := ic.client.Create(folderPath); err != nil {
+		// Ignore errors if the folder already exists
+		logger.WithFields(logrus.Fields{
+			"folder": folderPath,
+			"error":  err,
+		}).Debug("Folder creation failed (might already exist)")
+	}
+
+	return folderPath, nil
+}
+
 func (ic *IMAPClient) MarkAsProcessed(email Email) error {
 	if err := ic.ensureConnection(); err != nil {
 		return fmt.Errorf("failed to ensure IMAP connection: %v", err)
 	}
 
-	_, err := ic.client.Select(ic.config.MailboxIn, false)
-	if err != nil {
-		return err
+	if ic.config.ProcessedBox == "" {
+		return fmt.Errorf("processed_box not configured")
 	}
 
+	// Get proper folder path and ensure it exists
+	processedFolder, err := ic.ensureFolder(ic.config.ProcessedBox)
+	if err != nil {
+		return fmt.Errorf("failed to ensure processed folder exists: %v", err)
+	}
+
+	// Select source mailbox
+	_, err = ic.client.Select(ic.config.MailboxIn, false)
+	if err != nil {
+		return fmt.Errorf("failed to select source mailbox: %v", err)
+	}
+
+	// Find the email by Message-Id
 	criteria := imap.NewSearchCriteria()
 	criteria.Header.Set("Message-Id", email.ID)
 
 	uids, err := ic.client.Search(criteria)
 	if err != nil {
-		return err
+		return fmt.Errorf("search failed: %v", err)
 	}
 
 	if len(uids) == 0 {
 		return fmt.Errorf("email not found")
 	}
 
+	// Move the message to processed folder
 	seqSet := new(imap.SeqSet)
 	seqSet.AddNum(uids...)
 
-	return ic.client.Store(seqSet, imap.FormatFlagsOp(imap.AddFlags, true), []interface{}{"Processed"}, nil)
+	// Mark as read before moving
+	item := imap.FormatFlagsOp(imap.AddFlags, true)
+	flags := []interface{}{imap.SeenFlag}
+	if err := ic.client.Store(seqSet, item, flags, nil); err != nil {
+		return fmt.Errorf("failed to mark message as read: %v", err)
+	}
+
+	// Copy to processed folder using the proper path
+	if err := ic.client.Copy(seqSet, processedFolder); err != nil {
+		return fmt.Errorf("failed to copy to processed folder: %v", err)
+	}
+
+	// Delete from source folder
+	item = imap.FormatFlagsOp(imap.AddFlags, true)
+	flags = []interface{}{imap.DeletedFlag}
+	if err := ic.client.Store(seqSet, item, flags, nil); err != nil {
+		return fmt.Errorf("failed to mark message as deleted: %v", err)
+	}
+
+	if err := ic.client.Expunge(nil); err != nil {
+		return fmt.Errorf("failed to expunge message: %v", err)
+	}
+
+	return nil
 }
 
 func (ic *IMAPClient) Close() error {