improvements

go.mod

@@ -3,10 +3,14 @@ module paraclub-ai-mailer
 go 1.23.5
 
 require (
-	github.com/emersion/go-imap v1.2.1 // indirect
+	github.com/emersion/go-imap v1.2.1
-	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
+	github.com/sirupsen/logrus v1.9.3
-	github.com/sirupsen/logrus v1.9.3 // indirect
+	golang.org/x/net v0.35.0
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	gopkg.in/yaml.v3 v3.0.1
-	golang.org/x/text v0.3.7 // indirect
+)
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+
+require (
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
 )

go.sum

@@ -1,4 +1,5 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA=
 github.com/emersion/go-imap v1.2.1/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY=
@@ -7,17 +8,24 @@ github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTe
 github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
 github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
 github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/ai/ai.go

@@ -48,7 +48,11 @@ func (a *AI) GenerateReply(emailContent string, contextContent map[string]string
 	}
 
 	// Prepare the system message and user message
-	systemMsg := "You are a helpful assistant responding to emails. Analyze the language of the incoming email and translate the response to the same language. Format your response in HTML. Do not include any explanations or extra text - just write the email response directly in HTML format. Use appropriate HTML tags for formatting."
+	// systemMsg := "You are a helpful assistant responding to emails. Analyze the language of the incoming email and translate the response to the same language. Format your response in HTML. Do not include any explanations or extra text - just write the email response directly in HTML format. Use appropriate HTML tags for formatting."
+  // systemMsg := "You are a helpful assistant who responds to emails. For each incoming email, first detect its language and then generate your response in the exact same language. Your reply must be written directly in HTML format using appropriate HTML tags for structure and styling. Do not include any explanations, commentary, or extra text—simply output the email response in HTML."
+  // systemMsg := "You are a helpful assistant who responds to emails. For each incoming email, first detect its language and then generate your response in the exact same language. Your reply must be written directly in HTML format using appropriate HTML tags for structure and styling. Do not include a subject line, explanations, commentary, or any extra text—simply output the email response content in HTML."
+  // systemMsg := "You are a helpful assistant who responds to emails. For every incoming email, carefully detect and confirm its language. Then, generate your email response entirely in that same language without deviation. Format your reply solely in HTML using appropriate HTML tags for structure and styling, and do not include a subject line, explanations, or any extra text. Ensure that every part of your response exactly matches the language of the incoming email."
+  systemMsg := "You are a helpful assistant who responds to emails. For every incoming email, strictly use only the email's content to detect its language and ignore any external or additional context. Then, generate your email response entirely in that same language without deviation. Format your reply solely in HTML using appropriate HTML tags for structure and styling, and do not include a subject line, explanations, commentary, or any extra text. Ensure that every part of your response exactly matches the language of the incoming email."
 	userMsg := fmt.Sprintf("Using the following context:\n%s\n\nPlease analyze the language of and generate a reply in the same language for this email:\n%s", context, emailContent)
 
 	messages := []Message{

internal/fetcher/fetcher.go

@@ -3,7 +3,10 @@ package fetcher
 import (
 	"io"
 	"net/http"
+	"strings"
 	"time"
+
+	"golang.org/x/net/html"
 )
 
 type Fetcher struct {
@@ -18,6 +21,31 @@ func New() *Fetcher {
 	}
 }
 
+func (f *Fetcher) extractText(htmlContent string) string {
+	doc, err := html.Parse(strings.NewReader(htmlContent))
+	if err != nil {
+		return htmlContent // fallback to raw content if parsing fails
+	}
+
+	var result strings.Builder
+	var extractTextNode func(*html.Node)
+	extractTextNode = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			text := strings.TrimSpace(n.Data)
+			if text != "" {
+				result.WriteString(text)
+				result.WriteString(" ")
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			extractTextNode(c)
+		}
+	}
+
+	extractTextNode(doc)
+	return strings.TrimSpace(result.String())
+}
+
 func (f *Fetcher) FetchContent(url string) (string, error) {
 	resp, err := f.client.Get(url)
 	if err != nil {
@@ -30,12 +58,11 @@ func (f *Fetcher) FetchContent(url string) (string, error) {
 		return "", err
 	}
 
-	return string(body), nil
+	return f.extractText(string(body)), nil
 }
 
 func (f *Fetcher) FetchAllURLs(urls []string) (map[string]string, error) {
 	results := make(map[string]string)
-
 	for _, url := range urls {
 		content, err := f.FetchContent(url)
 		if err != nil {
@@ -43,6 +70,5 @@ func (f *Fetcher) FetchAllURLs(urls []string) (map[string]string, error) {
 		}
 		results[url] = content
 	}
-
 	return results, nil
 }

internal/imap/imap.go

@@ -303,12 +303,15 @@ func extractMessageContent(body string) string {
 					continue
 				}
 				textContent = buf.String()
+				// Convert line endings to HTML breaks
+				textContent = strings.ReplaceAll(textContent, "\n", "<br>\n")
+				textContent = strings.ReplaceAll(textContent, "\r\n", "<br>\n")
 				break
 			}
 		}
 
 		if textContent != "" {
-			return strings.TrimSpace(textContent)
+			return textContent // Remove TrimSpace to preserve formatting
 		}
 	}
 
@@ -319,16 +322,28 @@ func extractMessageContent(body string) string {
 		return fallbackExtractContent(body)
 	}
 
-	return strings.TrimSpace(buf.String())
+	content := buf.String()
+	// Convert line endings to HTML breaks
+	content = strings.ReplaceAll(content, "\r\n", "<br>\n")
+	content = strings.ReplaceAll(content, "\n", "<br>\n")
+	return content // Remove TrimSpace to preserve formatting
 }
 
 // fallbackExtractContent is the previous implementation used as fallback
 func fallbackExtractContent(body string) string {
 	parts := strings.Split(body, "\r\n\r\n")
 	if len(parts) > 1 {
-		return strings.TrimSpace(strings.Join(parts[1:], "\r\n\r\n"))
+		content := strings.Join(parts[1:], "\r\n\r\n")
+		// Convert line endings to HTML breaks
+		content = strings.ReplaceAll(content, "\r\n", "<br>\n")
+		content = strings.ReplaceAll(content, "\n", "<br>\n")
+		return content // Remove TrimSpace to preserve formatting
 	}
-	return strings.TrimSpace(body)
+	content := body
+	// Convert line endings to HTML breaks
+	content = strings.ReplaceAll(content, "\r\n", "<br>\n")
+	content = strings.ReplaceAll(content, "\n", "<br>\n")
+	return content // Remove TrimSpace to preserve formatting
 }
 
 func (ic *IMAPClient) MarkAsProcessed(email Email) error {