Fix prototype: calibrate vision for real D2R screenshots, implement orb detection, improve dashboard
- Created debug tool (cmd/debug/main.go) to analyze real D2R screenshots and calibrate HSV ranges - Fixed HSV color ranges for health/mana orbs based on real screenshot analysis (99.5% and 82% detection rates) - Replaced ReadBarPercentage with ReadOrbPercentage for circular orbs (not horizontal bars) - Added SetSource() method to capture Manager for hot-swapping capture sources - Fixed dashboard JavaScript null reference errors with proper array checks - Improved dashboard refresh rate from 100ms to 1000ms for better performance - Added proper error handling for empty/null API responses - Successfully detecting game state, health (99.5%), and mana (82%) from real D2R screenshot
This commit is contained in:
Hoid
parent
4ebed5e3ab
commit
4f0b84ec31
10 changed files with 473 additions and 19 deletions
379
cmd/debug/main.go
Normal file
379
cmd/debug/main.go
Normal file
|
|
@ -0,0 +1,379 @@
|
|||
// Debug tool to analyze D2R screenshots and calibrate vision parameters.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"image/png"
|
||||
_ "image/png"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.cloonar.com/openclawd/iso-bot/pkg/engine/vision"
|
||||
"git.cloonar.com/openclawd/iso-bot/plugins/d2r"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Load the real D2R screenshot
|
||||
screenshotPath := "testdata/d2r_1080p.png"
|
||||
fmt.Printf("Analyzing D2R screenshot: %s\n", screenshotPath)
|
||||
|
||||
img, err := loadImage(screenshotPath)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load screenshot: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Image size: %dx%d\n", img.Bounds().Dx(), img.Bounds().Dy())
|
||||
|
||||
// Get current config
|
||||
config := d2r.DefaultConfig()
|
||||
|
||||
// Create debug directory
|
||||
debugDir := "testdata/debug"
|
||||
os.MkdirAll(debugDir, 0755)
|
||||
|
||||
// Define regions for 1080p (from config.go)
|
||||
regions := map[string]image.Rectangle{
|
||||
"health_orb": image.Rect(28, 545, 198, 715),
|
||||
"mana_orb": image.Rect(1722, 545, 1892, 715),
|
||||
"xp_bar": image.Rect(0, 1058, 1920, 1080),
|
||||
"belt": image.Rect(838, 1010, 1082, 1058),
|
||||
"minimap": image.Rect(1600, 0, 1920, 320),
|
||||
"inventory": image.Rect(960, 330, 1490, 770),
|
||||
"stash": image.Rect(430, 330, 960, 770),
|
||||
"skill_left": image.Rect(194, 1030, 246, 1078),
|
||||
"skill_right": image.Rect(1674, 1030, 1726, 1078),
|
||||
}
|
||||
|
||||
fmt.Println("\n=== REGION ANALYSIS ===")
|
||||
|
||||
// Analyze health orb
|
||||
analyzeRegion(img, "health_orb", regions["health_orb"], config.Colors.HealthFilled, debugDir)
|
||||
|
||||
// Analyze mana orb
|
||||
analyzeRegion(img, "mana_orb", regions["mana_orb"], config.Colors.ManaFilled, debugDir)
|
||||
|
||||
// Sample some specific pixels in the orbs for detailed analysis
|
||||
fmt.Println("\n=== PIXEL SAMPLING ===")
|
||||
samplePixelsInRegion(img, "health_orb", regions["health_orb"])
|
||||
samplePixelsInRegion(img, "mana_orb", regions["mana_orb"])
|
||||
|
||||
// Suggest new HSV ranges based on analysis
|
||||
fmt.Println("\n=== RECOMMENDATIONS ===")
|
||||
recommendHealthRange(img, regions["health_orb"])
|
||||
recommendManaRange(img, regions["mana_orb"])
|
||||
|
||||
fmt.Printf("\nDebug images saved to: %s\n", debugDir)
|
||||
fmt.Println("Run this tool after implementing the fixes to verify detection works correctly.")
|
||||
}
|
||||
|
||||
func analyzeRegion(img image.Image, name string, region image.Rectangle, currentColor d2r.HSVRange, debugDir string) {
|
||||
fmt.Printf("\n--- %s ---\n", name)
|
||||
fmt.Printf("Region: (%d,%d) -> (%d,%d) [%dx%d]\n",
|
||||
region.Min.X, region.Min.Y, region.Max.X, region.Max.Y,
|
||||
region.Dx(), region.Dy())
|
||||
|
||||
// Extract the region
|
||||
bounds := region.Intersect(img.Bounds())
|
||||
if bounds.Empty() {
|
||||
fmt.Printf("ERROR: Region is outside image bounds!\n")
|
||||
return
|
||||
}
|
||||
|
||||
// Crop and save the region
|
||||
cropped := cropImage(img, bounds)
|
||||
cropPath := filepath.Join(debugDir, fmt.Sprintf("debug_%s.png", name))
|
||||
if err := saveImage(cropped, cropPath); err != nil {
|
||||
fmt.Printf("WARNING: Failed to save cropped image: %v\n", err)
|
||||
}
|
||||
|
||||
// Analyze colors in the region
|
||||
totalPixels := 0
|
||||
matchingPixels := 0
|
||||
var minH, maxH, minS, maxS, minV, maxV = 360, 0, 255, 0, 255, 0
|
||||
var avgR, avgG, avgB float64
|
||||
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
c := img.At(x, y)
|
||||
r, g, b, _ := c.RGBA()
|
||||
// Convert to 8-bit
|
||||
r, g, b = r>>8, g>>8, b>>8
|
||||
|
||||
avgR += float64(r)
|
||||
avgG += float64(g)
|
||||
avgB += float64(b)
|
||||
|
||||
hsv := vision.RGBToHSV(c)
|
||||
totalPixels++
|
||||
|
||||
// Track HSV ranges
|
||||
if hsv.H < minH { minH = hsv.H }
|
||||
if hsv.H > maxH { maxH = hsv.H }
|
||||
if hsv.S < minS { minS = hsv.S }
|
||||
if hsv.S > maxS { maxS = hsv.S }
|
||||
if hsv.V < minV { minV = hsv.V }
|
||||
if hsv.V > maxV { maxV = hsv.V }
|
||||
|
||||
// Check if current color range matches
|
||||
if hsv.H >= currentColor.LowerH && hsv.H <= currentColor.UpperH &&
|
||||
hsv.S >= currentColor.LowerS && hsv.S <= currentColor.UpperS &&
|
||||
hsv.V >= currentColor.LowerV && hsv.V <= currentColor.UpperV {
|
||||
matchingPixels++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if totalPixels > 0 {
|
||||
avgR /= float64(totalPixels)
|
||||
avgG /= float64(totalPixels)
|
||||
avgB /= float64(totalPixels)
|
||||
}
|
||||
|
||||
fmt.Printf("Current HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n",
|
||||
currentColor.LowerH, currentColor.UpperH,
|
||||
currentColor.LowerS, currentColor.UpperS,
|
||||
currentColor.LowerV, currentColor.UpperV)
|
||||
|
||||
fmt.Printf("Actual HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n",
|
||||
minH, maxH, minS, maxS, minV, maxV)
|
||||
|
||||
fmt.Printf("Average RGB: (%.1f, %.1f, %.1f)\n", avgR, avgG, avgB)
|
||||
|
||||
matchPct := float64(matchingPixels) / float64(totalPixels) * 100
|
||||
fmt.Printf("Matching pixels: %d/%d (%.1f%%)\n", matchingPixels, totalPixels, matchPct)
|
||||
|
||||
if matchPct < 30 {
|
||||
fmt.Printf("⚠️ WARNING: Low match rate! Current HSV range may need adjustment.\n")
|
||||
} else if matchPct > 80 {
|
||||
fmt.Printf("✅ Good match rate.\n")
|
||||
} else {
|
||||
fmt.Printf("⚠️ Moderate match rate - consider refining HSV range.\n")
|
||||
}
|
||||
|
||||
fmt.Printf("Saved cropped region to: debug_%s.png\n", name)
|
||||
}
|
||||
|
||||
func samplePixelsInRegion(img image.Image, regionName string, region image.Rectangle) {
|
||||
fmt.Printf("\n--- %s Pixel Samples ---\n", regionName)
|
||||
|
||||
bounds := region.Intersect(img.Bounds())
|
||||
if bounds.Empty() {
|
||||
return
|
||||
}
|
||||
|
||||
// Sample pixels at different positions in the region
|
||||
samples := []struct {
|
||||
name string
|
||||
x, y int
|
||||
}{
|
||||
{"center", (bounds.Min.X + bounds.Max.X) / 2, (bounds.Min.Y + bounds.Max.Y) / 2},
|
||||
{"top-left", bounds.Min.X + 10, bounds.Min.Y + 10},
|
||||
{"top-right", bounds.Max.X - 10, bounds.Min.Y + 10},
|
||||
{"bottom-left", bounds.Min.X + 10, bounds.Max.Y - 10},
|
||||
{"bottom-right", bounds.Max.X - 10, bounds.Max.Y - 10},
|
||||
}
|
||||
|
||||
for _, sample := range samples {
|
||||
if sample.x >= bounds.Min.X && sample.x < bounds.Max.X &&
|
||||
sample.y >= bounds.Min.Y && sample.y < bounds.Max.Y {
|
||||
|
||||
c := img.At(sample.x, sample.y)
|
||||
r, g, b, _ := c.RGBA()
|
||||
r, g, b = r>>8, g>>8, b>>8
|
||||
hsv := vision.RGBToHSV(c)
|
||||
|
||||
fmt.Printf(" %s (%d,%d): RGB(%d,%d,%d) HSV(%d,%d,%d)\n",
|
||||
sample.name, sample.x, sample.y, r, g, b, hsv.H, hsv.S, hsv.V)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func recommendHealthRange(img image.Image, region image.Rectangle) {
|
||||
fmt.Println("\n--- Health Orb HSV Range Recommendations ---")
|
||||
|
||||
bounds := region.Intersect(img.Bounds())
|
||||
if bounds.Empty() {
|
||||
return
|
||||
}
|
||||
|
||||
// Find red-ish pixels (health orb is red)
|
||||
var redPixels []vision.HSV
|
||||
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
c := img.At(x, y)
|
||||
r, g, b, _ := c.RGBA()
|
||||
r, g, b = r>>8, g>>8, b>>8
|
||||
|
||||
// Look for pixels that are predominantly red
|
||||
if r > 50 && r > g && r > b {
|
||||
hsv := vision.RGBToHSV(c)
|
||||
redPixels = append(redPixels, hsv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(redPixels) == 0 {
|
||||
fmt.Println("No red-ish pixels found - health orb might be empty or coordinates wrong")
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate ranges with some padding
|
||||
minH, maxH := 360, 0
|
||||
minS, maxS := 255, 0
|
||||
minV, maxV := 255, 0
|
||||
|
||||
for _, hsv := range redPixels {
|
||||
if hsv.H < minH { minH = hsv.H }
|
||||
if hsv.H > maxH { maxH = hsv.H }
|
||||
if hsv.S < minS { minS = hsv.S }
|
||||
if hsv.S > maxS { maxS = hsv.S }
|
||||
if hsv.V < minV { minV = hsv.V }
|
||||
if hsv.V > maxV { maxV = hsv.V }
|
||||
}
|
||||
|
||||
// Add padding for gradients/textures
|
||||
hPadding := 10
|
||||
sPadding := 30
|
||||
vPadding := 50
|
||||
|
||||
// Handle hue wrap-around for reds
|
||||
if minH < hPadding {
|
||||
minH = 0
|
||||
} else {
|
||||
minH -= hPadding
|
||||
}
|
||||
if maxH + hPadding > 360 {
|
||||
maxH = 360
|
||||
} else {
|
||||
maxH += hPadding
|
||||
}
|
||||
|
||||
minS = max(0, minS-sPadding)
|
||||
maxS = min(255, maxS+sPadding)
|
||||
minV = max(0, minV-vPadding)
|
||||
maxV = min(255, maxV+vPadding)
|
||||
|
||||
fmt.Printf("Found %d red pixels in health orb region\n", len(redPixels))
|
||||
fmt.Printf("Recommended health HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n",
|
||||
minH, maxH, minS, maxS, minV, maxV)
|
||||
fmt.Printf("Go code: HealthFilled: HSVRange{%d, %d, %d, %d, %d, %d},\n",
|
||||
minH, minS, minV, maxH, maxS, maxV)
|
||||
}
|
||||
|
||||
func recommendManaRange(img image.Image, region image.Rectangle) {
|
||||
fmt.Println("\n--- Mana Orb HSV Range Recommendations ---")
|
||||
|
||||
bounds := region.Intersect(img.Bounds())
|
||||
if bounds.Empty() {
|
||||
return
|
||||
}
|
||||
|
||||
// Find blue-ish pixels (mana orb is blue)
|
||||
var bluePixels []vision.HSV
|
||||
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
c := img.At(x, y)
|
||||
r, g, b, _ := c.RGBA()
|
||||
r, g, b = r>>8, g>>8, b>>8
|
||||
|
||||
// Look for pixels that are predominantly blue
|
||||
if b > 50 && b > r && b > g {
|
||||
hsv := vision.RGBToHSV(c)
|
||||
bluePixels = append(bluePixels, hsv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(bluePixels) == 0 {
|
||||
fmt.Println("No blue-ish pixels found - mana orb might be empty or coordinates wrong")
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate ranges with some padding
|
||||
minH, maxH := 360, 0
|
||||
minS, maxS := 255, 0
|
||||
minV, maxV := 255, 0
|
||||
|
||||
for _, hsv := range bluePixels {
|
||||
if hsv.H < minH { minH = hsv.H }
|
||||
if hsv.H > maxH { maxH = hsv.H }
|
||||
if hsv.S < minS { minS = hsv.S }
|
||||
if hsv.S > maxS { maxS = hsv.S }
|
||||
if hsv.V < minV { minV = hsv.V }
|
||||
if hsv.V > maxV { maxV = hsv.V }
|
||||
}
|
||||
|
||||
// Add padding for gradients/textures
|
||||
hPadding := 10
|
||||
sPadding := 30
|
||||
vPadding := 50
|
||||
|
||||
minH = max(0, minH-hPadding)
|
||||
maxH = min(360, maxH+hPadding)
|
||||
minS = max(0, minS-sPadding)
|
||||
maxS = min(255, maxS+sPadding)
|
||||
minV = max(0, minV-vPadding)
|
||||
maxV = min(255, maxV+vPadding)
|
||||
|
||||
fmt.Printf("Found %d blue pixels in mana orb region\n", len(bluePixels))
|
||||
fmt.Printf("Recommended mana HSV range: H[%d-%d] S[%d-%d] V[%d-%d]\n",
|
||||
minH, maxH, minS, maxS, minV, maxV)
|
||||
fmt.Printf("Go code: ManaFilled: HSVRange{%d, %d, %d, %d, %d, %d},\n",
|
||||
minH, minS, minV, maxH, maxS, maxV)
|
||||
}
|
||||
|
||||
func loadImage(path string) (image.Image, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
img, _, err := image.Decode(file)
|
||||
return img, err
|
||||
}
|
||||
|
||||
func saveImage(img image.Image, path string) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return png.Encode(file, img)
|
||||
}
|
||||
|
||||
func cropImage(img image.Image, bounds image.Rectangle) image.Image {
|
||||
if subImg, ok := img.(interface {
|
||||
SubImage(r image.Rectangle) image.Image
|
||||
}); ok {
|
||||
return subImg.SubImage(bounds)
|
||||
}
|
||||
|
||||
// Fallback: manually copy pixels
|
||||
cropped := image.NewRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
cropped.Set(x-bounds.Min.X, y-bounds.Min.Y, img.At(x, y))
|
||||
}
|
||||
}
|
||||
return cropped
|
||||
}
|
||||
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue