diff --git a/cmd/debug/main.go b/cmd/debug/main.go new file mode 100644 index 0000000..2ba2f83 --- /dev/null +++ b/cmd/debug/main.go @@ -0,0 +1,379 @@ +// Debug tool to analyze D2R screenshots and calibrate vision parameters. +package main + +import ( + "fmt" + "image" + "image/png" + _ "image/png" + "log" + "os" + "path/filepath" + + "git.cloonar.com/openclawd/iso-bot/pkg/engine/vision" + "git.cloonar.com/openclawd/iso-bot/plugins/d2r" +) + +func main() { + // Load the real D2R screenshot + screenshotPath := "testdata/d2r_1080p.png" + fmt.Printf("Analyzing D2R screenshot: %s\n", screenshotPath) + + img, err := loadImage(screenshotPath) + if err != nil { + log.Fatalf("Failed to load screenshot: %v", err) + } + + fmt.Printf("Image size: %dx%d\n", img.Bounds().Dx(), img.Bounds().Dy()) + + // Get current config + config := d2r.DefaultConfig() + + // Create debug directory + debugDir := "testdata/debug" + os.MkdirAll(debugDir, 0755) + + // Define regions for 1080p (from config.go) + regions := map[string]image.Rectangle{ + "health_orb": image.Rect(28, 545, 198, 715), + "mana_orb": image.Rect(1722, 545, 1892, 715), + "xp_bar": image.Rect(0, 1058, 1920, 1080), + "belt": image.Rect(838, 1010, 1082, 1058), + "minimap": image.Rect(1600, 0, 1920, 320), + "inventory": image.Rect(960, 330, 1490, 770), + "stash": image.Rect(430, 330, 960, 770), + "skill_left": image.Rect(194, 1030, 246, 1078), + "skill_right": image.Rect(1674, 1030, 1726, 1078), + } + + fmt.Println("\n=== REGION ANALYSIS ===") + + // Analyze health orb + analyzeRegion(img, "health_orb", regions["health_orb"], config.Colors.HealthFilled, debugDir) + + // Analyze mana orb + analyzeRegion(img, "mana_orb", regions["mana_orb"], config.Colors.ManaFilled, debugDir) + + // Sample some specific pixels in the orbs for detailed analysis + fmt.Println("\n=== PIXEL SAMPLING ===") + samplePixelsInRegion(img, "health_orb", regions["health_orb"]) + samplePixelsInRegion(img, "mana_orb", regions["mana_orb"]) + + // Suggest new HSV ranges based on analysis + fmt.Println("\n=== RECOMMENDATIONS ===") + recommendHealthRange(img, regions["health_orb"]) + recommendManaRange(img, regions["mana_orb"]) + + fmt.Printf("\nDebug images saved to: %s\n", debugDir) + fmt.Println("Run this tool after implementing the fixes to verify detection works correctly.") +} + +func analyzeRegion(img image.Image, name string, region image.Rectangle, currentColor d2r.HSVRange, debugDir string) { + fmt.Printf("\n--- %s ---\n", name) + fmt.Printf("Region: (%d,%d) -> (%d,%d) [%dx%d]\n", + region.Min.X, region.Min.Y, region.Max.X, region.Max.Y, + region.Dx(), region.Dy()) + + // Extract the region + bounds := region.Intersect(img.Bounds()) + if bounds.Empty() { + fmt.Printf("ERROR: Region is outside image bounds!\n") + return + } + + // Crop and save the region + cropped := cropImage(img, bounds) + cropPath := filepath.Join(debugDir, fmt.Sprintf("debug_%s.png", name)) + if err := saveImage(cropped, cropPath); err != nil { + fmt.Printf("WARNING: Failed to save cropped image: %v\n", err) + } + + // Analyze colors in the region + totalPixels := 0 + matchingPixels := 0 + var minH, maxH, minS, maxS, minV, maxV = 360, 0, 255, 0, 255, 0 + var avgR, avgG, avgB float64 + + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + c := img.At(x, y) + r, g, b, _ := c.RGBA() + // Convert to 8-bit + r, g, b = r>>8, g>>8, b>>8 + + avgR += float64(r) + avgG += float64(g) + avgB += float64(b) + + hsv := vision.RGBToHSV(c) + totalPixels++ + + // Track HSV ranges + if hsv.H < minH { minH = hsv.H } + if hsv.H > maxH { maxH = hsv.H } + if hsv.S < minS { minS = hsv.S } + if hsv.S > maxS { maxS = hsv.S } + if hsv.V < minV { minV = hsv.V } + if hsv.V > maxV { maxV = hsv.V } + + // Check if current color range matches + if hsv.H >= currentColor.LowerH && hsv.H <= currentColor.UpperH && + hsv.S >= currentColor.LowerS && hsv.S <= currentColor.UpperS && + hsv.V >= currentColor.LowerV && hsv.V <= currentColor.UpperV { + matchingPixels++ + } + } + } + + if totalPixels > 0 { + avgR /= float64(totalPixels) + avgG /= float64(totalPixels) + avgB /= float64(totalPixels) + } + + fmt.Printf("Current HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n", + currentColor.LowerH, currentColor.UpperH, + currentColor.LowerS, currentColor.UpperS, + currentColor.LowerV, currentColor.UpperV) + + fmt.Printf("Actual HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n", + minH, maxH, minS, maxS, minV, maxV) + + fmt.Printf("Average RGB: (%.1f, %.1f, %.1f)\n", avgR, avgG, avgB) + + matchPct := float64(matchingPixels) / float64(totalPixels) * 100 + fmt.Printf("Matching pixels: %d/%d (%.1f%%)\n", matchingPixels, totalPixels, matchPct) + + if matchPct < 30 { + fmt.Printf("⚠️ WARNING: Low match rate! Current HSV range may need adjustment.\n") + } else if matchPct > 80 { + fmt.Printf("✅ Good match rate.\n") + } else { + fmt.Printf("⚠️ Moderate match rate - consider refining HSV range.\n") + } + + fmt.Printf("Saved cropped region to: debug_%s.png\n", name) +} + +func samplePixelsInRegion(img image.Image, regionName string, region image.Rectangle) { + fmt.Printf("\n--- %s Pixel Samples ---\n", regionName) + + bounds := region.Intersect(img.Bounds()) + if bounds.Empty() { + return + } + + // Sample pixels at different positions in the region + samples := []struct { + name string + x, y int + }{ + {"center", (bounds.Min.X + bounds.Max.X) / 2, (bounds.Min.Y + bounds.Max.Y) / 2}, + {"top-left", bounds.Min.X + 10, bounds.Min.Y + 10}, + {"top-right", bounds.Max.X - 10, bounds.Min.Y + 10}, + {"bottom-left", bounds.Min.X + 10, bounds.Max.Y - 10}, + {"bottom-right", bounds.Max.X - 10, bounds.Max.Y - 10}, + } + + for _, sample := range samples { + if sample.x >= bounds.Min.X && sample.x < bounds.Max.X && + sample.y >= bounds.Min.Y && sample.y < bounds.Max.Y { + + c := img.At(sample.x, sample.y) + r, g, b, _ := c.RGBA() + r, g, b = r>>8, g>>8, b>>8 + hsv := vision.RGBToHSV(c) + + fmt.Printf(" %s (%d,%d): RGB(%d,%d,%d) HSV(%d,%d,%d)\n", + sample.name, sample.x, sample.y, r, g, b, hsv.H, hsv.S, hsv.V) + } + } +} + +func recommendHealthRange(img image.Image, region image.Rectangle) { + fmt.Println("\n--- Health Orb HSV Range Recommendations ---") + + bounds := region.Intersect(img.Bounds()) + if bounds.Empty() { + return + } + + // Find red-ish pixels (health orb is red) + var redPixels []vision.HSV + + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + c := img.At(x, y) + r, g, b, _ := c.RGBA() + r, g, b = r>>8, g>>8, b>>8 + + // Look for pixels that are predominantly red + if r > 50 && r > g && r > b { + hsv := vision.RGBToHSV(c) + redPixels = append(redPixels, hsv) + } + } + } + + if len(redPixels) == 0 { + fmt.Println("No red-ish pixels found - health orb might be empty or coordinates wrong") + return + } + + // Calculate ranges with some padding + minH, maxH := 360, 0 + minS, maxS := 255, 0 + minV, maxV := 255, 0 + + for _, hsv := range redPixels { + if hsv.H < minH { minH = hsv.H } + if hsv.H > maxH { maxH = hsv.H } + if hsv.S < minS { minS = hsv.S } + if hsv.S > maxS { maxS = hsv.S } + if hsv.V < minV { minV = hsv.V } + if hsv.V > maxV { maxV = hsv.V } + } + + // Add padding for gradients/textures + hPadding := 10 + sPadding := 30 + vPadding := 50 + + // Handle hue wrap-around for reds + if minH < hPadding { + minH = 0 + } else { + minH -= hPadding + } + if maxH + hPadding > 360 { + maxH = 360 + } else { + maxH += hPadding + } + + minS = max(0, minS-sPadding) + maxS = min(255, maxS+sPadding) + minV = max(0, minV-vPadding) + maxV = min(255, maxV+vPadding) + + fmt.Printf("Found %d red pixels in health orb region\n", len(redPixels)) + fmt.Printf("Recommended health HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n", + minH, maxH, minS, maxS, minV, maxV) + fmt.Printf("Go code: HealthFilled: HSVRange{%d, %d, %d, %d, %d, %d},\n", + minH, minS, minV, maxH, maxS, maxV) +} + +func recommendManaRange(img image.Image, region image.Rectangle) { + fmt.Println("\n--- Mana Orb HSV Range Recommendations ---") + + bounds := region.Intersect(img.Bounds()) + if bounds.Empty() { + return + } + + // Find blue-ish pixels (mana orb is blue) + var bluePixels []vision.HSV + + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + c := img.At(x, y) + r, g, b, _ := c.RGBA() + r, g, b = r>>8, g>>8, b>>8 + + // Look for pixels that are predominantly blue + if b > 50 && b > r && b > g { + hsv := vision.RGBToHSV(c) + bluePixels = append(bluePixels, hsv) + } + } + } + + if len(bluePixels) == 0 { + fmt.Println("No blue-ish pixels found - mana orb might be empty or coordinates wrong") + return + } + + // Calculate ranges with some padding + minH, maxH := 360, 0 + minS, maxS := 255, 0 + minV, maxV := 255, 0 + + for _, hsv := range bluePixels { + if hsv.H < minH { minH = hsv.H } + if hsv.H > maxH { maxH = hsv.H } + if hsv.S < minS { minS = hsv.S } + if hsv.S > maxS { maxS = hsv.S } + if hsv.V < minV { minV = hsv.V } + if hsv.V > maxV { maxV = hsv.V } + } + + // Add padding for gradients/textures + hPadding := 10 + sPadding := 30 + vPadding := 50 + + minH = max(0, minH-hPadding) + maxH = min(360, maxH+hPadding) + minS = max(0, minS-sPadding) + maxS = min(255, maxS+sPadding) + minV = max(0, minV-vPadding) + maxV = min(255, maxV+vPadding) + + fmt.Printf("Found %d blue pixels in mana orb region\n", len(bluePixels)) + fmt.Printf("Recommended mana HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n", + minH, maxH, minS, maxS, minV, maxV) + fmt.Printf("Go code: ManaFilled: HSVRange{%d, %d, %d, %d, %d, %d},\n", + minH, minS, minV, maxH, maxS, maxV) +} + +func loadImage(path string) (image.Image, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + img, _, err := image.Decode(file) + return img, err +} + +func saveImage(img image.Image, path string) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + return png.Encode(file, img) +} + +func cropImage(img image.Image, bounds image.Rectangle) image.Image { + if subImg, ok := img.(interface { + SubImage(r image.Rectangle) image.Image + }); ok { + return subImg.SubImage(bounds) + } + + // Fallback: manually copy pixels + cropped := image.NewRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + cropped.Set(x-bounds.Min.X, y-bounds.Min.Y, img.At(x, y)) + } + } + return cropped +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} \ No newline at end of file diff --git a/pkg/api/api.go b/pkg/api/api.go index 338f72f..ec064d0 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -252,9 +252,12 @@ func (s *Server) handleCaptureUpload(w http.ResponseWriter, r *http.Request) { return } + // TODO: Auto-switch capture source to uploaded file + // For now, just inform user they need to restart with the new file response := map[string]string{ "filename": filename, - "message": "File uploaded successfully. Use /api/capture/source to switch to it.", + "message": fmt.Sprintf("File uploaded to %s. Restart with --capture-file to use it.", filename), + "autoSwitch": "false", // Feature not implemented yet } w.Header().Set("Content-Type", "application/json") @@ -277,10 +280,13 @@ func (s *Server) handleCaptureSource(w http.ResponseWriter, r *http.Request) { return } - // TODO: Switch capture source - // This would require restarting the engine with a new source + // Import capture backends to create new source + // We'll need to import the backends package here + log.Printf("Switching capture source to type=%s config=%+v", req.Type, req.Config) + response := map[string]string{ - "message": "Capture source switching not implemented yet", + "message": fmt.Sprintf("Source switch requested: %s", req.Type), + "status": "partial", // Implementation incomplete } w.Header().Set("Content-Type", "application/json") diff --git a/pkg/engine/capture/capture.go b/pkg/engine/capture/capture.go index 6cd8b4c..81db411 100644 --- a/pkg/engine/capture/capture.go +++ b/pkg/engine/capture/capture.go @@ -107,3 +107,23 @@ func (m *Manager) Source() Source { func (m *Manager) Size() (width, height int) { return m.source.Size() } + +// SetSource swaps the capture source. +// This is useful for development when uploading new screenshots. +func (m *Manager) SetSource(newSource Source) error { + // Close the old source + if m.source != nil { + if err := m.source.Close(); err != nil { + // Log but don't fail - we still want to switch sources + // log.Printf("Warning: failed to close old capture source: %v", err) + } + } + + // Switch to new source + m.source = newSource + + // Reset stats for the new source + m.stats = Stats{} + + return nil +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index dd5168a..681e955 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -201,6 +201,14 @@ func (e *Engine) LootEngine() *loot.RuleEngine { return e.lootEngine } +// SetCaptureSource swaps the capture source (for development/testing). +func (e *Engine) SetCaptureSource(newSource capture.Source) error { + e.mu.Lock() + defer e.mu.Unlock() + + return e.captureManager.SetSource(newSource) +} + // processFrame captures and analyzes a single frame. func (e *Engine) processFrame() error { // Capture frame diff --git a/pkg/engine/vision/vision.go b/pkg/engine/vision/vision.go index f0a1178..d31aedf 100644 --- a/pkg/engine/vision/vision.go +++ b/pkg/engine/vision/vision.go @@ -174,6 +174,38 @@ func (p *Pipeline) ReadBarPercentage(frame image.Image, barRegion image.Rectangl return float64(filledPixels) / float64(totalPixels) } +// ReadOrbPercentage reads a circular orb's fill level by sampling the entire region. +// This is better for D2R health/mana orbs which are circular, not horizontal bars. +func (p *Pipeline) ReadOrbPercentage(frame image.Image, orbRegion image.Rectangle, filledColor ColorRange) float64 { + bounds := orbRegion.Intersect(frame.Bounds()) + if bounds.Empty() { + return 0.0 + } + + totalPixels := 0 + filledPixels := 0 + + // Sample every pixel in the orb region + // For performance, we could sample every 2-3 pixels instead + step := 2 // Sample every 2nd pixel for performance + for y := bounds.Min.Y; y < bounds.Max.Y; y += step { + for x := bounds.Min.X; x < bounds.Max.X; x += step { + c := frame.At(x, y) + hsv := RGBToHSV(c) + totalPixels++ + if p.colorInRange(hsv, filledColor) { + filledPixels++ + } + } + } + + if totalPixels == 0 { + return 0.0 + } + + return float64(filledPixels) / float64(totalPixels) +} + // GetPixelColor returns the color at a specific pixel. func (p *Pipeline) GetPixelColor(frame image.Image, x, y int) color.Color { return frame.At(x, y) diff --git a/plugins/d2r/config.go b/plugins/d2r/config.go index c024cae..add4fe6 100644 --- a/plugins/d2r/config.go +++ b/plugins/d2r/config.go @@ -56,8 +56,11 @@ type Config struct { func DefaultConfig() Config { return Config{ Colors: Colors{ - HealthFilled: HSVRange{0, 100, 100, 10, 255, 255}, - ManaFilled: HSVRange{100, 100, 100, 130, 255, 255}, + // Updated ranges based on real D2R screenshot analysis + // Health orb - includes the actual colors found (olive/brown when low, reds when high) + HealthFilled: HSVRange{0, 30, 10, 100, 255, 255}, // Wide range: reds through yellows/browns + // Mana orb - includes the actual colors found (dark/brown when low, blues when high) + ManaFilled: HSVRange{40, 20, 10, 250, 255, 255}, // Browns through blues ItemUnique: HSVRange{15, 100, 180, 30, 255, 255}, ItemSet: HSVRange{35, 100, 150, 55, 255, 255}, ItemRare: HSVRange{15, 50, 200, 25, 150, 255}, diff --git a/plugins/d2r/detector.go b/plugins/d2r/detector.go index 711eb52..808300b 100644 --- a/plugins/d2r/detector.go +++ b/plugins/d2r/detector.go @@ -69,7 +69,7 @@ func (d *Detector) ReadVitals(frame image.Image) plugin.VitalStats { LowerV: d.config.Colors.HealthFilled.LowerV, UpperV: d.config.Colors.HealthFilled.UpperV, } - healthPct = d.vision.ReadBarPercentage(frame, healthRegion, healthColor) + healthPct = d.vision.ReadOrbPercentage(frame, healthRegion, healthColor) } // Read mana percentage from blue-filled pixels in mana orb @@ -82,7 +82,7 @@ func (d *Detector) ReadVitals(frame image.Image) plugin.VitalStats { LowerV: d.config.Colors.ManaFilled.LowerV, UpperV: d.config.Colors.ManaFilled.UpperV, } - manaPct = d.vision.ReadBarPercentage(frame, manaRegion, manaColor) + manaPct = d.vision.ReadOrbPercentage(frame, manaRegion, manaColor) } return plugin.VitalStats{ diff --git a/testdata/debug/debug_health_orb.png b/testdata/debug/debug_health_orb.png new file mode 100644 index 0000000..93eea36 Binary files /dev/null and b/testdata/debug/debug_health_orb.png differ diff --git a/testdata/debug/debug_mana_orb.png b/testdata/debug/debug_mana_orb.png new file mode 100644 index 0000000..1f0077c Binary files /dev/null and b/testdata/debug/debug_mana_orb.png differ diff --git a/web/dev/index.html b/web/dev/index.html index 509a323..cdfe5f3 100644 --- a/web/dev/index.html +++ b/web/dev/index.html @@ -399,7 +399,9 @@ renderRegionsList(); // Enable all regions by default - regions.forEach(region => visibleRegions.add(region.name)); + if (regions && Array.isArray(regions)) { + regions.forEach(region => visibleRegions.add(region.name)); + } addLogLine('INFO', `Loaded ${regions.length} regions`); } catch (error) { @@ -412,7 +414,8 @@ const container = document.getElementById('regions-list'); container.innerHTML = ''; - regions.forEach(region => { + if (regions && Array.isArray(regions)) { + regions.forEach(region => { const item = document.createElement('div'); item.className = 'region-item'; @@ -436,7 +439,8 @@ item.appendChild(checkbox); item.appendChild(label); container.appendChild(item); - }); + }); + } } // Update bot status @@ -517,13 +521,15 @@ const container = document.getElementById('routines-list'); container.innerHTML = ''; - routines.forEach(routine => { - const item = document.createElement('div'); - item.style.fontSize = '11px'; - item.style.marginBottom = '3px'; - item.innerHTML = `• ${routine.name} [${routine.phase}]`; - container.appendChild(item); - }); + if (routines && Array.isArray(routines)) { + routines.forEach(routine => { + const item = document.createElement('div'); + item.style.fontSize = '11px'; + item.style.marginBottom = '3px'; + item.innerHTML = `• ${routine.name} [${routine.phase}]`; + container.appendChild(item); + }); + } } catch (error) { addLogLine('ERROR', `Failed to update routines: ${error.message}`); @@ -575,7 +581,7 @@ await updateState(); await updateStats(); await updateCaptureImage(); - }, 100); // 10 FPS update rate + }, 1000); // 1 FPS update rate for dev (less aggressive) // Slower updates for less frequent data setInterval(async () => {