Working prototype with dev dashboard
- Created core Engine that coordinates all subsystems - Extended API with comprehensive endpoints for dev dashboard - Implemented pure Go vision processing (no GoCV dependency) - Built full-featured dev dashboard with live capture viewer, region overlays, pixel inspector - Added D2R detector with actual health/mana reading from orb regions - Fixed resolution profile registration and region validation - Generated synthetic test data for development - Added dev mode support with file backend for testing - Fixed build tag issues for cross-platform compilation Prototype features: ✅ Live capture viewer with region overlays ✅ Real-time state detection (game state, health %, mana %) ✅ Pixel inspector (hover for RGB/HSV values) ✅ Capture stats monitoring (FPS, frame count) ✅ Region management with toggle visibility ✅ File upload for testing screenshots ✅ Dark theme dev-focused UI ✅ CORS enabled for dev convenience Ready for: go run ./cmd/iso-bot --dev --api :8080
This commit is contained in:
Hoid
parent
80ba9b1b90
commit
6a9562c406
10 changed files with 1884 additions and 62 deletions
|
|
@ -1,12 +1,13 @@
|
|||
// Package vision provides computer vision utilities for game screen analysis.
|
||||
//
|
||||
// Uses GoCV (OpenCV bindings for Go) for template matching, color detection,
|
||||
// and contour analysis. Designed for high-throughput real-time analysis.
|
||||
// Pure Go implementation without external dependencies like OpenCV.
|
||||
// Designed for high-throughput real-time analysis of game screens.
|
||||
package vision
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/color"
|
||||
"math"
|
||||
)
|
||||
|
||||
// Match represents a detected element on screen.
|
||||
|
|
@ -31,6 +32,11 @@ type ColorRange struct {
|
|||
UpperH, UpperS, UpperV int
|
||||
}
|
||||
|
||||
// HSV represents a color in HSV color space.
|
||||
type HSV struct {
|
||||
H, S, V int
|
||||
}
|
||||
|
||||
// Pipeline processes frames through a series of vision operations.
|
||||
type Pipeline struct {
|
||||
templates map[string]*Template
|
||||
|
|
@ -58,31 +64,293 @@ func (p *Pipeline) LoadTemplate(name string, img image.Image) {
|
|||
|
||||
// FindTemplate searches for a template in the frame.
|
||||
// Returns the best match above threshold, or nil.
|
||||
// This is a simple implementation - could be improved with better algorithms.
|
||||
func (p *Pipeline) FindTemplate(frame image.Image, templateName string) *Match {
|
||||
// TODO: Implement with GoCV matchTemplate
|
||||
// This is a stub — actual implementation needs gocv.MatchTemplate
|
||||
template, exists := p.templates[templateName]
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
frameBounds := frame.Bounds()
|
||||
templateBounds := template.Image.Bounds()
|
||||
|
||||
// Simple template matching by scanning every position
|
||||
bestMatch := &Match{Confidence: 0}
|
||||
|
||||
for y := frameBounds.Min.Y; y <= frameBounds.Max.Y-templateBounds.Dy(); y++ {
|
||||
for x := frameBounds.Min.X; x <= frameBounds.Max.X-templateBounds.Dx(); x++ {
|
||||
confidence := p.compareAtPosition(frame, template.Image, x, y)
|
||||
if confidence > bestMatch.Confidence {
|
||||
bestMatch.Position = image.Point{X: x, Y: y}
|
||||
bestMatch.BBox = image.Rect(x, y, x+templateBounds.Dx(), y+templateBounds.Dy())
|
||||
bestMatch.Confidence = confidence
|
||||
bestMatch.Label = templateName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if bestMatch.Confidence >= p.threshold {
|
||||
return bestMatch
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindAllTemplates finds all matches of a template above threshold.
|
||||
func (p *Pipeline) FindAllTemplates(frame image.Image, templateName string) []Match {
|
||||
// TODO: Implement with GoCV + NMS
|
||||
// For simplicity, just return the best match
|
||||
match := p.FindTemplate(frame, templateName)
|
||||
if match != nil {
|
||||
return []Match{*match}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindByColor detects regions matching an HSV color range.
|
||||
func (p *Pipeline) FindByColor(frame image.Image, colorRange ColorRange, minArea int) []Match {
|
||||
// TODO: Implement with GoCV inRange + findContours
|
||||
return nil
|
||||
bounds := frame.Bounds()
|
||||
var matches []Match
|
||||
|
||||
// Simple blob detection by scanning for connected regions
|
||||
visited := make(map[image.Point]bool)
|
||||
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
pt := image.Point{X: x, Y: y}
|
||||
if visited[pt] {
|
||||
continue
|
||||
}
|
||||
|
||||
c := frame.At(x, y)
|
||||
hsv := RGBToHSV(c)
|
||||
|
||||
if p.colorInRange(hsv, colorRange) {
|
||||
// Found a pixel in range, flood fill to find the blob
|
||||
blob := p.floodFill(frame, pt, colorRange, visited)
|
||||
if len(blob) >= minArea {
|
||||
bbox := p.getBoundingBox(blob)
|
||||
center := image.Point{
|
||||
X: (bbox.Min.X + bbox.Max.X) / 2,
|
||||
Y: (bbox.Min.Y + bbox.Max.Y) / 2,
|
||||
}
|
||||
matches = append(matches, Match{
|
||||
Position: center,
|
||||
BBox: bbox,
|
||||
Confidence: 1.0, // Binary detection for color matching
|
||||
Label: "color_match",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matches
|
||||
}
|
||||
|
||||
// ReadBarPercentage reads a horizontal bar's fill level (health, mana, xp).
|
||||
func (p *Pipeline) ReadBarPercentage(frame image.Image, barRegion image.Rectangle, filledColor ColorRange) float64 {
|
||||
// TODO: Implement — scan columns for filled color ratio
|
||||
return 0.0
|
||||
bounds := barRegion.Intersect(frame.Bounds())
|
||||
if bounds.Empty() {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
totalPixels := 0
|
||||
filledPixels := 0
|
||||
|
||||
// Sample pixels across the width of the bar
|
||||
centerY := (bounds.Min.Y + bounds.Max.Y) / 2
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
c := frame.At(x, centerY)
|
||||
hsv := RGBToHSV(c)
|
||||
totalPixels++
|
||||
if p.colorInRange(hsv, filledColor) {
|
||||
filledPixels++
|
||||
}
|
||||
}
|
||||
|
||||
if totalPixels == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
return float64(filledPixels) / float64(totalPixels)
|
||||
}
|
||||
|
||||
// GetPixelColor returns the color at a specific pixel.
|
||||
func (p *Pipeline) GetPixelColor(frame image.Image, x, y int) color.Color {
|
||||
return frame.At(x, y)
|
||||
}
|
||||
|
||||
// GetPixelHSV returns the HSV values at a specific pixel.
|
||||
func (p *Pipeline) GetPixelHSV(frame image.Image, x, y int) HSV {
|
||||
c := frame.At(x, y)
|
||||
return RGBToHSV(c)
|
||||
}
|
||||
|
||||
// HasColorInRegion checks if any pixel in the region matches the color range.
|
||||
func (p *Pipeline) HasColorInRegion(frame image.Image, region image.Rectangle, colorRange ColorRange) bool {
|
||||
bounds := region.Intersect(frame.Bounds())
|
||||
if bounds.Empty() {
|
||||
return false
|
||||
}
|
||||
|
||||
// Sample every few pixels for performance
|
||||
step := 2
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y += step {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x += step {
|
||||
c := frame.At(x, y)
|
||||
hsv := RGBToHSV(c)
|
||||
if p.colorInRange(hsv, colorRange) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// compareAtPosition compares template with frame at given position.
|
||||
func (p *Pipeline) compareAtPosition(frame, template image.Image, frameX, frameY int) float64 {
|
||||
templateBounds := template.Bounds()
|
||||
totalPixels := 0
|
||||
matchingPixels := 0
|
||||
|
||||
// Simple pixel-by-pixel comparison
|
||||
for y := templateBounds.Min.Y; y < templateBounds.Max.Y; y++ {
|
||||
for x := templateBounds.Min.X; x < templateBounds.Max.X; x++ {
|
||||
frameColor := frame.At(frameX+x, frameY+y)
|
||||
templateColor := template.At(x, y)
|
||||
|
||||
totalPixels++
|
||||
if p.colorsMatch(frameColor, templateColor, 30) { // tolerance of 30
|
||||
matchingPixels++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return float64(matchingPixels) / float64(totalPixels)
|
||||
}
|
||||
|
||||
// colorsMatch checks if two colors are similar within tolerance.
|
||||
func (p *Pipeline) colorsMatch(c1, c2 color.Color, tolerance int) bool {
|
||||
r1, g1, b1, _ := c1.RGBA()
|
||||
r2, g2, b2, _ := c2.RGBA()
|
||||
|
||||
// Convert from 16-bit to 8-bit
|
||||
r1, g1, b1 = r1>>8, g1>>8, b1>>8
|
||||
r2, g2, b2 = r2>>8, g2>>8, b2>>8
|
||||
|
||||
dr := int(r1) - int(r2)
|
||||
dg := int(g1) - int(g2)
|
||||
db := int(b1) - int(b2)
|
||||
|
||||
if dr < 0 { dr = -dr }
|
||||
if dg < 0 { dg = -dg }
|
||||
if db < 0 { db = -db }
|
||||
|
||||
return dr <= tolerance && dg <= tolerance && db <= tolerance
|
||||
}
|
||||
|
||||
// colorInRange checks if HSV color is within range.
|
||||
func (p *Pipeline) colorInRange(hsv HSV, colorRange ColorRange) bool {
|
||||
return hsv.H >= colorRange.LowerH && hsv.H <= colorRange.UpperH &&
|
||||
hsv.S >= colorRange.LowerS && hsv.S <= colorRange.UpperS &&
|
||||
hsv.V >= colorRange.LowerV && hsv.V <= colorRange.UpperV
|
||||
}
|
||||
|
||||
// floodFill finds connected pixels of the same color.
|
||||
func (p *Pipeline) floodFill(frame image.Image, start image.Point, colorRange ColorRange, visited map[image.Point]bool) []image.Point {
|
||||
bounds := frame.Bounds()
|
||||
var blob []image.Point
|
||||
stack := []image.Point{start}
|
||||
|
||||
for len(stack) > 0 {
|
||||
pt := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
|
||||
if visited[pt] || !pt.In(bounds) {
|
||||
continue
|
||||
}
|
||||
|
||||
c := frame.At(pt.X, pt.Y)
|
||||
hsv := RGBToHSV(c)
|
||||
if !p.colorInRange(hsv, colorRange) {
|
||||
continue
|
||||
}
|
||||
|
||||
visited[pt] = true
|
||||
blob = append(blob, pt)
|
||||
|
||||
// Add neighbors
|
||||
neighbors := []image.Point{
|
||||
{X: pt.X-1, Y: pt.Y},
|
||||
{X: pt.X+1, Y: pt.Y},
|
||||
{X: pt.X, Y: pt.Y-1},
|
||||
{X: pt.X, Y: pt.Y+1},
|
||||
}
|
||||
stack = append(stack, neighbors...)
|
||||
}
|
||||
|
||||
return blob
|
||||
}
|
||||
|
||||
// getBoundingBox calculates the bounding box for a set of points.
|
||||
func (p *Pipeline) getBoundingBox(points []image.Point) image.Rectangle {
|
||||
if len(points) == 0 {
|
||||
return image.Rectangle{}
|
||||
}
|
||||
|
||||
minX, minY := points[0].X, points[0].Y
|
||||
maxX, maxY := minX, minY
|
||||
|
||||
for _, pt := range points {
|
||||
if pt.X < minX { minX = pt.X }
|
||||
if pt.X > maxX { maxX = pt.X }
|
||||
if pt.Y < minY { minY = pt.Y }
|
||||
if pt.Y > maxY { maxY = pt.Y }
|
||||
}
|
||||
|
||||
return image.Rect(minX, minY, maxX+1, maxY+1)
|
||||
}
|
||||
|
||||
// RGBToHSV converts RGB color to HSV.
|
||||
func RGBToHSV(c color.Color) HSV {
|
||||
r, g, b, _ := c.RGBA()
|
||||
// Convert from 16-bit to float [0,1]
|
||||
rf := float64(r>>8) / 255.0
|
||||
gf := float64(g>>8) / 255.0
|
||||
bf := float64(b>>8) / 255.0
|
||||
|
||||
max := math.Max(rf, math.Max(gf, bf))
|
||||
min := math.Min(rf, math.Min(gf, bf))
|
||||
delta := max - min
|
||||
|
||||
// Value
|
||||
v := max
|
||||
|
||||
// Saturation
|
||||
var s float64
|
||||
if max != 0 {
|
||||
s = delta / max
|
||||
}
|
||||
|
||||
// Hue
|
||||
var h float64
|
||||
if delta != 0 {
|
||||
switch max {
|
||||
case rf:
|
||||
h = math.Mod((gf-bf)/delta, 6)
|
||||
case gf:
|
||||
h = (bf-rf)/delta + 2
|
||||
case bf:
|
||||
h = (rf-gf)/delta + 4
|
||||
}
|
||||
h *= 60
|
||||
if h < 0 {
|
||||
h += 360
|
||||
}
|
||||
}
|
||||
|
||||
return HSV{
|
||||
H: int(h),
|
||||
S: int(s * 255),
|
||||
V: int(v * 255),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue