35a1944179
- Add ReadOrbSlicePercentage: thin vertical strip method for accurate fill reading - Support HSV hue wrapping in colorInRange (for red hues crossing 360°) - Health HSV: H 350-370 (wrapping), S 60+, V 20+ (calibrated from screenshot) - Mana HSV: H 226-240, S 100+, V 20+ (calibrated from screenshot) - Add health_slice/mana_slice regions (10px wide vertical strips through orb centers) - Update health_globe/mana_globe to full botty-referenced regions - Verified: health 96.3%, mana 100% on testdata/d2r_1080p.png (both at 100% fill)
426 lines
11 KiB
Go
426 lines
11 KiB
Go
// Package vision provides computer vision utilities for game screen analysis.
|
|
//
|
|
// Pure Go implementation without external dependencies like OpenCV.
|
|
// Designed for high-throughput real-time analysis of game screens.
|
|
package vision
|
|
|
|
import (
|
|
"image"
|
|
"image/color"
|
|
"math"
|
|
)
|
|
|
|
// Match represents a detected element on screen.
|
|
type Match struct {
|
|
Position image.Point
|
|
BBox image.Rectangle
|
|
Confidence float64
|
|
Label string
|
|
}
|
|
|
|
// Template is a pre-loaded image template for matching.
|
|
type Template struct {
|
|
Name string
|
|
Image image.Image
|
|
Width int
|
|
Height int
|
|
}
|
|
|
|
// ColorRange defines an HSV color range for detection.
|
|
type ColorRange struct {
|
|
LowerH, LowerS, LowerV int
|
|
UpperH, UpperS, UpperV int
|
|
}
|
|
|
|
// HSV represents a color in HSV color space.
|
|
type HSV struct {
|
|
H, S, V int
|
|
}
|
|
|
|
// Pipeline processes frames through a series of vision operations.
|
|
type Pipeline struct {
|
|
templates map[string]*Template
|
|
threshold float64
|
|
}
|
|
|
|
// NewPipeline creates a vision pipeline with the given confidence threshold.
|
|
func NewPipeline(threshold float64) *Pipeline {
|
|
return &Pipeline{
|
|
templates: make(map[string]*Template),
|
|
threshold: threshold,
|
|
}
|
|
}
|
|
|
|
// LoadTemplate loads a template image for matching.
|
|
func (p *Pipeline) LoadTemplate(name string, img image.Image) {
|
|
bounds := img.Bounds()
|
|
p.templates[name] = &Template{
|
|
Name: name,
|
|
Image: img,
|
|
Width: bounds.Dx(),
|
|
Height: bounds.Dy(),
|
|
}
|
|
}
|
|
|
|
// FindTemplate searches for a template in the frame.
|
|
// Returns the best match above threshold, or nil.
|
|
// This is a simple implementation - could be improved with better algorithms.
|
|
func (p *Pipeline) FindTemplate(frame image.Image, templateName string) *Match {
|
|
template, exists := p.templates[templateName]
|
|
if !exists {
|
|
return nil
|
|
}
|
|
|
|
frameBounds := frame.Bounds()
|
|
templateBounds := template.Image.Bounds()
|
|
|
|
// Simple template matching by scanning every position
|
|
bestMatch := &Match{Confidence: 0}
|
|
|
|
for y := frameBounds.Min.Y; y <= frameBounds.Max.Y-templateBounds.Dy(); y++ {
|
|
for x := frameBounds.Min.X; x <= frameBounds.Max.X-templateBounds.Dx(); x++ {
|
|
confidence := p.compareAtPosition(frame, template.Image, x, y)
|
|
if confidence > bestMatch.Confidence {
|
|
bestMatch.Position = image.Point{X: x, Y: y}
|
|
bestMatch.BBox = image.Rect(x, y, x+templateBounds.Dx(), y+templateBounds.Dy())
|
|
bestMatch.Confidence = confidence
|
|
bestMatch.Label = templateName
|
|
}
|
|
}
|
|
}
|
|
|
|
if bestMatch.Confidence >= p.threshold {
|
|
return bestMatch
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// FindAllTemplates finds all matches of a template above threshold.
|
|
func (p *Pipeline) FindAllTemplates(frame image.Image, templateName string) []Match {
|
|
// For simplicity, just return the best match
|
|
match := p.FindTemplate(frame, templateName)
|
|
if match != nil {
|
|
return []Match{*match}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// FindByColor detects regions matching an HSV color range.
|
|
func (p *Pipeline) FindByColor(frame image.Image, colorRange ColorRange, minArea int) []Match {
|
|
bounds := frame.Bounds()
|
|
var matches []Match
|
|
|
|
// Simple blob detection by scanning for connected regions
|
|
visited := make(map[image.Point]bool)
|
|
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
|
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
|
pt := image.Point{X: x, Y: y}
|
|
if visited[pt] {
|
|
continue
|
|
}
|
|
|
|
c := frame.At(x, y)
|
|
hsv := RGBToHSV(c)
|
|
|
|
if p.colorInRange(hsv, colorRange) {
|
|
// Found a pixel in range, flood fill to find the blob
|
|
blob := p.floodFill(frame, pt, colorRange, visited)
|
|
if len(blob) >= minArea {
|
|
bbox := p.getBoundingBox(blob)
|
|
center := image.Point{
|
|
X: (bbox.Min.X + bbox.Max.X) / 2,
|
|
Y: (bbox.Min.Y + bbox.Max.Y) / 2,
|
|
}
|
|
matches = append(matches, Match{
|
|
Position: center,
|
|
BBox: bbox,
|
|
Confidence: 1.0, // Binary detection for color matching
|
|
Label: "color_match",
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return matches
|
|
}
|
|
|
|
// ReadBarPercentage reads a horizontal bar's fill level (health, mana, xp).
|
|
func (p *Pipeline) ReadBarPercentage(frame image.Image, barRegion image.Rectangle, filledColor ColorRange) float64 {
|
|
bounds := barRegion.Intersect(frame.Bounds())
|
|
if bounds.Empty() {
|
|
return 0.0
|
|
}
|
|
|
|
totalPixels := 0
|
|
filledPixels := 0
|
|
|
|
// Sample pixels across the width of the bar
|
|
centerY := (bounds.Min.Y + bounds.Max.Y) / 2
|
|
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
|
c := frame.At(x, centerY)
|
|
hsv := RGBToHSV(c)
|
|
totalPixels++
|
|
if p.colorInRange(hsv, filledColor) {
|
|
filledPixels++
|
|
}
|
|
}
|
|
|
|
if totalPixels == 0 {
|
|
return 0.0
|
|
}
|
|
|
|
return float64(filledPixels) / float64(totalPixels)
|
|
}
|
|
|
|
// ReadOrbPercentage reads a circular orb's fill level by sampling the entire region.
|
|
// This is better for D2R health/mana orbs which are circular, not horizontal bars.
|
|
func (p *Pipeline) ReadOrbPercentage(frame image.Image, orbRegion image.Rectangle, filledColor ColorRange) float64 {
|
|
bounds := orbRegion.Intersect(frame.Bounds())
|
|
if bounds.Empty() {
|
|
return 0.0
|
|
}
|
|
|
|
totalPixels := 0
|
|
filledPixels := 0
|
|
|
|
// Sample every pixel in the orb region
|
|
// For performance, we could sample every 2-3 pixels instead
|
|
step := 2 // Sample every 2nd pixel for performance
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y += step {
|
|
for x := bounds.Min.X; x < bounds.Max.X; x += step {
|
|
c := frame.At(x, y)
|
|
hsv := RGBToHSV(c)
|
|
totalPixels++
|
|
if p.colorInRange(hsv, filledColor) {
|
|
filledPixels++
|
|
}
|
|
}
|
|
}
|
|
|
|
if totalPixels == 0 {
|
|
return 0.0
|
|
}
|
|
|
|
return float64(filledPixels) / float64(totalPixels)
|
|
}
|
|
|
|
// ReadOrbSlicePercentage reads an orb's fill level using a thin vertical slice.
|
|
// The orb fills from bottom to top, so we count matching pixels in the slice
|
|
// and return the percentage. This is much more accurate than scanning the whole orb.
|
|
func (p *Pipeline) ReadOrbSlicePercentage(frame image.Image, sliceRegion image.Rectangle, filledColor ColorRange) float64 {
|
|
bounds := sliceRegion.Intersect(frame.Bounds())
|
|
if bounds.Empty() {
|
|
return 0.0
|
|
}
|
|
|
|
totalPixels := 0
|
|
filledPixels := 0
|
|
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
|
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
|
c := frame.At(x, y)
|
|
hsv := RGBToHSV(c)
|
|
totalPixels++
|
|
if p.colorInRange(hsv, filledColor) {
|
|
filledPixels++
|
|
}
|
|
}
|
|
}
|
|
|
|
if totalPixels == 0 {
|
|
return 0.0
|
|
}
|
|
|
|
return float64(filledPixels) / float64(totalPixels)
|
|
}
|
|
|
|
// GetPixelColor returns the color at a specific pixel.
|
|
func (p *Pipeline) GetPixelColor(frame image.Image, x, y int) color.Color {
|
|
return frame.At(x, y)
|
|
}
|
|
|
|
// GetPixelHSV returns the HSV values at a specific pixel.
|
|
func (p *Pipeline) GetPixelHSV(frame image.Image, x, y int) HSV {
|
|
c := frame.At(x, y)
|
|
return RGBToHSV(c)
|
|
}
|
|
|
|
// HasColorInRegion checks if any pixel in the region matches the color range.
|
|
func (p *Pipeline) HasColorInRegion(frame image.Image, region image.Rectangle, colorRange ColorRange) bool {
|
|
bounds := region.Intersect(frame.Bounds())
|
|
if bounds.Empty() {
|
|
return false
|
|
}
|
|
|
|
// Sample every few pixels for performance
|
|
step := 2
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y += step {
|
|
for x := bounds.Min.X; x < bounds.Max.X; x += step {
|
|
c := frame.At(x, y)
|
|
hsv := RGBToHSV(c)
|
|
if p.colorInRange(hsv, colorRange) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// compareAtPosition compares template with frame at given position.
|
|
func (p *Pipeline) compareAtPosition(frame, template image.Image, frameX, frameY int) float64 {
|
|
templateBounds := template.Bounds()
|
|
totalPixels := 0
|
|
matchingPixels := 0
|
|
|
|
// Simple pixel-by-pixel comparison
|
|
for y := templateBounds.Min.Y; y < templateBounds.Max.Y; y++ {
|
|
for x := templateBounds.Min.X; x < templateBounds.Max.X; x++ {
|
|
frameColor := frame.At(frameX+x, frameY+y)
|
|
templateColor := template.At(x, y)
|
|
|
|
totalPixels++
|
|
if p.colorsMatch(frameColor, templateColor, 30) { // tolerance of 30
|
|
matchingPixels++
|
|
}
|
|
}
|
|
}
|
|
|
|
return float64(matchingPixels) / float64(totalPixels)
|
|
}
|
|
|
|
// colorsMatch checks if two colors are similar within tolerance.
|
|
func (p *Pipeline) colorsMatch(c1, c2 color.Color, tolerance int) bool {
|
|
r1, g1, b1, _ := c1.RGBA()
|
|
r2, g2, b2, _ := c2.RGBA()
|
|
|
|
// Convert from 16-bit to 8-bit
|
|
r1, g1, b1 = r1>>8, g1>>8, b1>>8
|
|
r2, g2, b2 = r2>>8, g2>>8, b2>>8
|
|
|
|
dr := int(r1) - int(r2)
|
|
dg := int(g1) - int(g2)
|
|
db := int(b1) - int(b2)
|
|
|
|
if dr < 0 { dr = -dr }
|
|
if dg < 0 { dg = -dg }
|
|
if db < 0 { db = -db }
|
|
|
|
return dr <= tolerance && dg <= tolerance && db <= tolerance
|
|
}
|
|
|
|
// colorInRange checks if HSV color is within range.
|
|
// Supports hue wrapping: if UpperH > 360, it wraps around (e.g., 350-370 means 350-360 OR 0-10).
|
|
func (p *Pipeline) colorInRange(hsv HSV, colorRange ColorRange) bool {
|
|
if hsv.S < colorRange.LowerS || hsv.S > colorRange.UpperS ||
|
|
hsv.V < colorRange.LowerV || hsv.V > colorRange.UpperV {
|
|
return false
|
|
}
|
|
// Handle hue wrapping
|
|
if colorRange.UpperH > 360 {
|
|
// Wrapping range: e.g., 350-370 means H >= 350 OR H <= 10
|
|
return hsv.H >= colorRange.LowerH || hsv.H <= (colorRange.UpperH-360)
|
|
}
|
|
return hsv.H >= colorRange.LowerH && hsv.H <= colorRange.UpperH
|
|
}
|
|
|
|
// floodFill finds connected pixels of the same color.
|
|
func (p *Pipeline) floodFill(frame image.Image, start image.Point, colorRange ColorRange, visited map[image.Point]bool) []image.Point {
|
|
bounds := frame.Bounds()
|
|
var blob []image.Point
|
|
stack := []image.Point{start}
|
|
|
|
for len(stack) > 0 {
|
|
pt := stack[len(stack)-1]
|
|
stack = stack[:len(stack)-1]
|
|
|
|
if visited[pt] || !pt.In(bounds) {
|
|
continue
|
|
}
|
|
|
|
c := frame.At(pt.X, pt.Y)
|
|
hsv := RGBToHSV(c)
|
|
if !p.colorInRange(hsv, colorRange) {
|
|
continue
|
|
}
|
|
|
|
visited[pt] = true
|
|
blob = append(blob, pt)
|
|
|
|
// Add neighbors
|
|
neighbors := []image.Point{
|
|
{X: pt.X-1, Y: pt.Y},
|
|
{X: pt.X+1, Y: pt.Y},
|
|
{X: pt.X, Y: pt.Y-1},
|
|
{X: pt.X, Y: pt.Y+1},
|
|
}
|
|
stack = append(stack, neighbors...)
|
|
}
|
|
|
|
return blob
|
|
}
|
|
|
|
// getBoundingBox calculates the bounding box for a set of points.
|
|
func (p *Pipeline) getBoundingBox(points []image.Point) image.Rectangle {
|
|
if len(points) == 0 {
|
|
return image.Rectangle{}
|
|
}
|
|
|
|
minX, minY := points[0].X, points[0].Y
|
|
maxX, maxY := minX, minY
|
|
|
|
for _, pt := range points {
|
|
if pt.X < minX { minX = pt.X }
|
|
if pt.X > maxX { maxX = pt.X }
|
|
if pt.Y < minY { minY = pt.Y }
|
|
if pt.Y > maxY { maxY = pt.Y }
|
|
}
|
|
|
|
return image.Rect(minX, minY, maxX+1, maxY+1)
|
|
}
|
|
|
|
// RGBToHSV converts RGB color to HSV.
|
|
func RGBToHSV(c color.Color) HSV {
|
|
r, g, b, _ := c.RGBA()
|
|
// Convert from 16-bit to float [0,1]
|
|
rf := float64(r>>8) / 255.0
|
|
gf := float64(g>>8) / 255.0
|
|
bf := float64(b>>8) / 255.0
|
|
|
|
max := math.Max(rf, math.Max(gf, bf))
|
|
min := math.Min(rf, math.Min(gf, bf))
|
|
delta := max - min
|
|
|
|
// Value
|
|
v := max
|
|
|
|
// Saturation
|
|
var s float64
|
|
if max != 0 {
|
|
s = delta / max
|
|
}
|
|
|
|
// Hue
|
|
var h float64
|
|
if delta != 0 {
|
|
switch max {
|
|
case rf:
|
|
h = math.Mod((gf-bf)/delta, 6)
|
|
case gf:
|
|
h = (bf-rf)/delta + 2
|
|
case bf:
|
|
h = (rf-gf)/delta + 4
|
|
}
|
|
h *= 60
|
|
if h < 0 {
|
|
h += 360
|
|
}
|
|
}
|
|
|
|
return HSV{
|
|
H: int(h),
|
|
S: int(s * 255),
|
|
V: int(v * 255),
|
|
}
|
|
}
|