diff --git a/debug b/debug new file mode 100755 index 0000000..2f491f0 Binary files /dev/null and b/debug differ diff --git a/pkg/api/api.go b/pkg/api/api.go index ec064d0..a66e578 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -34,9 +34,11 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" "git.cloonar.com/openclawd/iso-bot/pkg/engine" + "git.cloonar.com/openclawd/iso-bot/pkg/engine/capture/backends" "git.cloonar.com/openclawd/iso-bot/pkg/engine/vision" ) @@ -73,6 +75,33 @@ type RegionInfo struct { Height int `json:"height"` } +// HSVColor represents an HSV color value. +type HSVColor struct { + H int `json:"h"` + S int `json:"s"` + V int `json:"v"` +} + +// RegionAnalysis represents analysis results for a specific region. +type RegionAnalysis struct { + Name string `json:"name"` + DominantHSV HSVColor `json:"dominantHSV"` + MatchPercent float64 `json:"matchPercent"` +} + +// AnalysisResponse represents the response from /api/analyze endpoint. +type AnalysisResponse struct { + GameState string `json:"gameState"` + Vitals VitalsInfo `json:"vitals"` + Regions []RegionAnalysis `json:"regions"` +} + +// VitalsInfo represents vital statistics. +type VitalsInfo struct { + HealthPct float64 `json:"healthPct"` + ManaPct float64 `json:"manaPct"` +} + // Server provides the HTTP API and WebSocket endpoint. type Server struct { mu sync.RWMutex @@ -114,6 +143,7 @@ func (s *Server) registerRoutes() { s.mux.HandleFunc("GET /api/pixel", s.handlePixel) s.mux.HandleFunc("GET /api/config", s.handleConfig) s.mux.HandleFunc("GET /api/plugins", s.handlePlugins) + s.mux.HandleFunc("GET /api/analyze", s.handleAnalyze) s.mux.HandleFunc("POST /api/start", s.handleStart) s.mux.HandleFunc("POST /api/stop", s.handleStop) s.mux.HandleFunc("POST /api/pause", s.handlePause) @@ -184,24 +214,45 @@ func (s *Server) handleCaptureFrameAnnotated(w http.ResponseWriter, r *http.Requ http.Error(w, "No frame available", http.StatusNoContent) return } - + // Create a copy of the frame to draw on bounds := frame.Bounds() annotated := image.NewRGBA(bounds) draw.Draw(annotated, bounds, frame, bounds.Min, draw.Src) - - // Draw region overlays + + // Get regions to draw (filtered by query parameter if present) regions := s.getRegions() + + // Filter regions by query parameter if provided + if r.URL.Query().Has("regions") { + regionsParam := r.URL.Query().Get("regions") + if regionsParam == "" { + regions = nil // Empty param = no overlays + } else { + regionNames := strings.Split(regionsParam, ",") + filteredRegions := make([]RegionInfo, 0) + for _, region := range regions { + for _, name := range regionNames { + if strings.TrimSpace(name) == region.Name { + filteredRegions = append(filteredRegions, region) + break + } + } + } + regions = filteredRegions + } + } + s.drawRegionOverlays(annotated, regions) - + w.Header().Set("Content-Type", "image/jpeg") - + var buf bytes.Buffer if err := jpeg.Encode(&buf, annotated, &jpeg.Options{Quality: 85}); err != nil { http.Error(w, "Failed to encode annotated frame", http.StatusInternalServerError) return } - + w.Write(buf.Bytes()) } @@ -251,13 +302,43 @@ func (s *Server) handleCaptureUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, "Failed to save file", http.StatusInternalServerError) return } - - // TODO: Auto-switch capture source to uploaded file - // For now, just inform user they need to restart with the new file + + // Auto-switch capture source to uploaded file + fileConfig := map[string]interface{}{ + "path": filename, + "type": "image", + } + + newSource, err := backends.NewFileSource(fileConfig) + if err != nil { + log.Printf("Failed to create file source: %v", err) + response := map[string]string{ + "filename": filename, + "message": fmt.Sprintf("File uploaded to %s but failed to switch capture source: %v", filename, err), + "autoSwitch": "failed", + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + err = s.engine.SetCaptureSource(newSource) + if err != nil { + log.Printf("Failed to set capture source: %v", err) + response := map[string]string{ + "filename": filename, + "message": fmt.Sprintf("File uploaded to %s but failed to switch capture source: %v", filename, err), + "autoSwitch": "failed", + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + response := map[string]string{ - "filename": filename, - "message": fmt.Sprintf("File uploaded to %s. Restart with --capture-file to use it.", filename), - "autoSwitch": "false", // Feature not implemented yet + "filename": filename, + "message": fmt.Sprintf("File uploaded to %s and capture source switched successfully", filename), + "autoSwitch": "true", } w.Header().Set("Content-Type", "application/json") @@ -449,23 +530,57 @@ func (s *Server) handlePause(w http.ResponseWriter, r *http.Request) { var req struct { Paused bool `json:"paused"` } - + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, "Invalid JSON", http.StatusBadRequest) return } - + s.engine.Pause(req.Paused) - + action := "resumed" if req.Paused { action = "paused" } - + response := map[string]string{ "message": fmt.Sprintf("Engine %s", action), } - + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleAnalyze(w http.ResponseWriter, r *http.Request) { + frame := s.engine.CurrentFrame() + if frame == nil { + http.Error(w, "No frame available", http.StatusNoContent) + return + } + + status := s.engine.Status() + + // Get vitals + vitals := VitalsInfo{ + HealthPct: status.Vitals.HealthPct, + ManaPct: status.Vitals.ManaPct, + } + + // Get regions and analyze each one + regions := s.getRegions() + regionAnalyses := make([]RegionAnalysis, 0, len(regions)) + + for _, region := range regions { + analysis := s.analyzeRegion(frame, region) + regionAnalyses = append(regionAnalyses, analysis) + } + + response := AnalysisResponse{ + GameState: string(status.GameState), + Vitals: vitals, + Regions: regionAnalyses, + } + w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(response) } @@ -498,6 +613,93 @@ func (s *Server) getRegions() []RegionInfo { return regions } +func (s *Server) analyzeRegion(frame image.Image, region RegionInfo) RegionAnalysis { + rect := image.Rect(region.X, region.Y, region.X+region.Width, region.Y+region.Height) + bounds := rect.Intersect(frame.Bounds()) + + if bounds.Empty() { + return RegionAnalysis{ + Name: region.Name, + DominantHSV: HSVColor{H: 0, S: 0, V: 0}, + MatchPercent: 0.0, + } + } + + // Calculate dominant HSV by averaging all pixels in region + var totalH, totalS, totalV int64 + var pixelCount int64 + + // Also count matches for the region's expected color range + var matchingPixels int64 + expectedColorRange := s.getExpectedColorRange(region.Name) + + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + c := frame.At(x, y) + hsv := vision.RGBToHSV(c) + + totalH += int64(hsv.H) + totalS += int64(hsv.S) + totalV += int64(hsv.V) + pixelCount++ + + // Check if this pixel matches the expected color range + if expectedColorRange != nil && s.colorInRange(hsv, *expectedColorRange) { + matchingPixels++ + } + } + } + + dominantHSV := HSVColor{H: 0, S: 0, V: 0} + matchPercent := 0.0 + + if pixelCount > 0 { + dominantHSV = HSVColor{ + H: int(totalH / pixelCount), + S: int(totalS / pixelCount), + V: int(totalV / pixelCount), + } + + if expectedColorRange != nil { + matchPercent = float64(matchingPixels) / float64(pixelCount) * 100.0 + } + } + + return RegionAnalysis{ + Name: region.Name, + DominantHSV: dominantHSV, + MatchPercent: matchPercent, + } +} + +func (s *Server) getExpectedColorRange(regionName string) *vision.ColorRange { + // Get D2R config to determine expected color ranges for each region type + // For now, we'll handle the main regions: health_orb and mana_orb + switch regionName { + case "health_orb": + return &vision.ColorRange{ + LowerH: 0, UpperH: 30, + LowerS: 50, UpperS: 255, + LowerV: 30, UpperV: 255, + } + case "mana_orb": + return &vision.ColorRange{ + LowerH: 200, UpperH: 240, + LowerS: 40, UpperS: 255, + LowerV: 20, UpperV: 255, + } + default: + // For other regions, we don't have specific expected color ranges + return nil + } +} + +func (s *Server) colorInRange(hsv vision.HSV, colorRange vision.ColorRange) bool { + return hsv.H >= colorRange.LowerH && hsv.H <= colorRange.UpperH && + hsv.S >= colorRange.LowerS && hsv.S <= colorRange.UpperS && + hsv.V >= colorRange.LowerV && hsv.V <= colorRange.UpperV +} + func (s *Server) drawRegionOverlays(img *image.RGBA, regions []RegionInfo) { // Simple colored rectangles for region overlays colors := []image.Uniform{ diff --git a/plugins/d2r/config.go b/plugins/d2r/config.go index add4fe6..fcc1dd4 100644 --- a/plugins/d2r/config.go +++ b/plugins/d2r/config.go @@ -56,11 +56,11 @@ type Config struct { func DefaultConfig() Config { return Config{ Colors: Colors{ - // Updated ranges based on real D2R screenshot analysis - // Health orb - includes the actual colors found (olive/brown when low, reds when high) - HealthFilled: HSVRange{0, 30, 10, 100, 255, 255}, // Wide range: reds through yellows/browns - // Mana orb - includes the actual colors found (dark/brown when low, blues when high) - ManaFilled: HSVRange{40, 20, 10, 250, 255, 255}, // Browns through blues + // Tightened HSV ranges based on real D2R screenshot calibration + // Health orb is RED: H 0-30 (covering red wrapping), S 50-255, V 30-255 + HealthFilled: HSVRange{0, 50, 30, 30, 255, 255}, + // Mana orb is BLUE: H 200-240, S 40-255, V 20-255 + ManaFilled: HSVRange{200, 40, 20, 240, 255, 255}, ItemUnique: HSVRange{15, 100, 180, 30, 255, 255}, ItemSet: HSVRange{35, 100, 150, 55, 255, 255}, ItemRare: HSVRange{15, 50, 200, 25, 150, 255}, diff --git a/plugins/d2r/detector.go b/plugins/d2r/detector.go index 808300b..e0d50a6 100644 --- a/plugins/d2r/detector.go +++ b/plugins/d2r/detector.go @@ -27,12 +27,19 @@ func NewDetector(config Config, services plugin.EngineServices) *Detector { // DetectState analyzes a screenshot and returns the current game state. func (d *Detector) DetectState(frame image.Image) plugin.GameState { // Priority-based detection: - // 1. Check for loading screen - // 2. Check for main menu - // 3. Check for character select - // 4. Check for in-game (health orb visible) + // 1. Check for in-game first (health orb visible) + // 2. Check for loading screen (but only if not in-game) + // 3. Check for main menu + // 4. Check for character select // 5. Check for death screen + if d.isInGame(frame) { + vitals := d.ReadVitals(frame) + if vitals.HealthPct <= 0.01 { // Consider very low health as dead + return plugin.StateDead + } + return plugin.StateInGame + } if d.isLoading(frame) { return plugin.StateLoading } @@ -42,13 +49,6 @@ func (d *Detector) DetectState(frame image.Image) plugin.GameState { if d.isCharacterSelect(frame) { return plugin.StateCharacterSelect } - if d.isInGame(frame) { - vitals := d.ReadVitals(frame) - if vitals.HealthPct <= 0.01 { // Consider very low health as dead - return plugin.StateDead - } - return plugin.StateInGame - } return plugin.StateUnknown } @@ -98,12 +98,12 @@ func (d *Detector) IsInGame(frame image.Image) bool { } func (d *Detector) isLoading(frame image.Image) bool { - // Check for loading screen by looking for mostly black screen - // This is a simple heuristic - could be improved + // Check for loading screen by looking for mostly black screen (>90%) + // AND ensuring no health orb colors are present in the health orb region bounds := frame.Bounds() totalPixels := 0 darkPixels := 0 - + // Sample every 10 pixels for performance step := 10 for y := bounds.Min.Y; y < bounds.Max.Y; y += step { @@ -112,7 +112,7 @@ func (d *Detector) isLoading(frame image.Image) bool { r, g, b, _ := c.RGBA() // Convert to 8-bit r, g, b = r>>8, g>>8, b>>8 - + totalPixels++ // Consider pixel dark if all channels are below 30 if r < 30 && g < 30 && b < 30 { @@ -120,13 +120,35 @@ func (d *Detector) isLoading(frame image.Image) bool { } } } - + if totalPixels == 0 { return false } - - // If more than 70% of screen is dark, likely loading - return float64(darkPixels)/float64(totalPixels) > 0.7 + + // Require more than 90% dark screen + darkPercentage := float64(darkPixels) / float64(totalPixels) + if darkPercentage <= 0.9 { + return false + } + + // Additional check: ensure no health colors in health orb region + healthRegion := d.services.Region("health_orb") + if !healthRegion.Empty() { + healthColor := vision.ColorRange{ + LowerH: d.config.Colors.HealthFilled.LowerH, + UpperH: d.config.Colors.HealthFilled.UpperH, + LowerS: d.config.Colors.HealthFilled.LowerS, + UpperS: d.config.Colors.HealthFilled.UpperS, + LowerV: d.config.Colors.HealthFilled.LowerV, + UpperV: d.config.Colors.HealthFilled.UpperV, + } + // If health colors are detected in health orb region, it's not loading + if d.vision.HasColorInRegion(frame, healthRegion, healthColor) { + return false + } + } + + return true } func (d *Detector) isMainMenu(frame image.Image) bool { diff --git a/web/dev/index.html b/web/dev/index.html index cdfe5f3..c488c1b 100644 --- a/web/dev/index.html +++ b/web/dev/index.html @@ -539,17 +539,23 @@ // Update capture image with overlays async function updateCaptureImage() { try { - const response = await fetch('/api/capture/frame/annotated'); + let url = '/api/capture/frame/annotated'; + + // Always pass regions param to control overlays (empty = no overlays) + const regionNames = Array.from(visibleRegions).join(','); + url += `?regions=${encodeURIComponent(regionNames)}`; + + const response = await fetch(url); const blob = await response.blob(); - const url = URL.createObjectURL(blob); - + const imageUrl = URL.createObjectURL(blob); + // Clean up previous URL if (captureImage.src && captureImage.src.startsWith('blob:')) { URL.revokeObjectURL(captureImage.src); } - - captureImage.src = url; - + + captureImage.src = imageUrl; + } catch (error) { addLogLine('ERROR', `Failed to update capture image: ${error.message}`); }