Working prototype with dev dashboard

- Created core Engine that coordinates all subsystems
- Extended API with comprehensive endpoints for dev dashboard
- Implemented pure Go vision processing (no GoCV dependency)
- Built full-featured dev dashboard with live capture viewer, region overlays, pixel inspector
- Added D2R detector with actual health/mana reading from orb regions
- Fixed resolution profile registration and region validation
- Generated synthetic test data for development
- Added dev mode support with file backend for testing
- Fixed build tag issues for cross-platform compilation

Prototype features:
 Live capture viewer with region overlays
 Real-time state detection (game state, health %, mana %)
 Pixel inspector (hover for RGB/HSV values)
 Capture stats monitoring (FPS, frame count)
 Region management with toggle visibility
 File upload for testing screenshots
 Dark theme dev-focused UI
 CORS enabled for dev convenience

Ready for: go run ./cmd/iso-bot --dev --api :8080
This commit is contained in:
Hoid 2026-02-14 10:23:31 +00:00
parent 80ba9b1b90
commit 6a9562c406
10 changed files with 1884 additions and 62 deletions

View file

@ -2,30 +2,49 @@
//
// Endpoints:
// GET /api/status — bot status, current state, routine, stats
// GET /api/config — current configuration
// PUT /api/config — update configuration
// POST /api/start — start bot with routine
// POST /api/stop — stop bot
// POST /api/pause — pause/resume
// GET /api/capture/frame — returns current frame as JPEG
// GET /api/capture/frame/annotated — frame with region overlays
// GET /api/capture/stats — capture performance stats
// POST /api/capture/upload — upload a screenshot for testing
// POST /api/capture/source — switch capture source
// GET /api/regions — list all defined regions
// GET /api/state — current detected game state + vitals
// GET /api/loot/rules — get current loot filter rules
// GET /api/routines — list available routines
// GET /api/loot/rules — get loot filter rules
// PUT /api/loot/rules — update loot filter rules
// GET /api/stats — run statistics, items found, etc.
// WS /api/ws — real-time status stream
// GET /api/pixel — get pixel color at coordinates
// GET /api/config — current config
// GET /api/plugins — list loaded plugins and info
// WS /api/ws — WebSocket for real-time updates
// GET / — serve dev dashboard
//
// The API is served by the bot process itself (single binary).
package api
import (
"bytes"
"encoding/json"
"fmt"
"image"
"image/color"
"image/draw"
"image/jpeg"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strconv"
"sync"
"git.cloonar.com/openclawd/iso-bot/pkg/engine"
"git.cloonar.com/openclawd/iso-bot/pkg/engine/vision"
)
// Status represents the current bot status.
type Status struct {
Running bool `json:"running"`
Paused bool `json:"paused"`
DevMode bool `json:"devMode"`
GameState string `json:"gameState"`
Routine string `json:"routine,omitempty"`
Phase string `json:"phase,omitempty"`
@ -37,19 +56,39 @@ type Status struct {
ManaPct float64 `json:"manaPct"`
}
// PixelInfo represents RGB and HSV values at a pixel.
type PixelInfo struct {
X int `json:"x"`
Y int `json:"y"`
RGB [3]int `json:"rgb"`
HSV [3]int `json:"hsv"`
}
// RegionInfo represents a screen region.
type RegionInfo struct {
Name string `json:"name"`
X int `json:"x"`
Y int `json:"y"`
Width int `json:"width"`
Height int `json:"height"`
}
// Server provides the HTTP API and WebSocket endpoint.
type Server struct {
mu sync.RWMutex
status Status
engine *engine.Engine
addr string
mux *http.ServeMux
webRoot string
}
// NewServer creates an API server on the given address.
func NewServer(addr string) *Server {
func NewServer(addr string, eng *engine.Engine, webRoot string) *Server {
s := &Server{
addr: addr,
mux: http.NewServeMux(),
engine: eng,
addr: addr,
mux: http.NewServeMux(),
webRoot: webRoot,
}
s.registerRoutes()
return s
@ -57,37 +96,427 @@ func NewServer(addr string) *Server {
// Start begins serving the API.
func (s *Server) Start() error {
return http.ListenAndServe(s.addr, s.mux)
}
// UpdateStatus updates the bot status (called by the engine).
func (s *Server) UpdateStatus(status Status) {
s.mu.Lock()
defer s.mu.Unlock()
s.status = status
// TODO: Broadcast to WebSocket clients
return http.ListenAndServe(s.addr, s.enableCORS(s.mux))
}
func (s *Server) registerRoutes() {
// API routes
s.mux.HandleFunc("GET /api/status", s.handleStatus)
s.mux.HandleFunc("GET /api/capture/frame", s.handleCaptureFrame)
s.mux.HandleFunc("GET /api/capture/frame/annotated", s.handleCaptureFrameAnnotated)
s.mux.HandleFunc("GET /api/capture/stats", s.handleCaptureStats)
s.mux.HandleFunc("POST /api/capture/upload", s.handleCaptureUpload)
s.mux.HandleFunc("POST /api/capture/source", s.handleCaptureSource)
s.mux.HandleFunc("GET /api/regions", s.handleRegions)
s.mux.HandleFunc("GET /api/state", s.handleState)
s.mux.HandleFunc("GET /api/loot/rules", s.handleLootRules)
s.mux.HandleFunc("GET /api/routines", s.handleRoutines)
s.mux.HandleFunc("GET /api/pixel", s.handlePixel)
s.mux.HandleFunc("GET /api/config", s.handleConfig)
s.mux.HandleFunc("GET /api/plugins", s.handlePlugins)
s.mux.HandleFunc("POST /api/start", s.handleStart)
s.mux.HandleFunc("POST /api/stop", s.handleStop)
// TODO: Remaining routes
s.mux.HandleFunc("POST /api/pause", s.handlePause)
// WebSocket endpoint
// s.mux.HandleFunc("/api/ws", s.handleWebSocket)
// Serve static files for dev dashboard
s.mux.Handle("/", http.FileServer(http.Dir(s.webRoot)))
}
func (s *Server) enableCORS(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization")
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
next.ServeHTTP(w, r)
})
}
func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
s.mu.RLock()
defer s.mu.RUnlock()
status := s.engine.Status()
response := Status{
Running: status.Running,
Paused: status.Paused,
DevMode: status.DevMode,
GameState: string(status.GameState),
Uptime: status.Uptime.String(),
CaptureFPS: status.CaptureStats.FPS,
HealthPct: status.Vitals.HealthPct,
ManaPct: status.Vitals.ManaPct,
ItemsFound: status.ItemsFound,
RunCount: status.RunCount,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(s.status)
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleCaptureFrame(w http.ResponseWriter, r *http.Request) {
frame := s.engine.CurrentFrame()
if frame == nil {
http.Error(w, "No frame available", http.StatusNoContent)
return
}
w.Header().Set("Content-Type", "image/jpeg")
var buf bytes.Buffer
if err := jpeg.Encode(&buf, frame, &jpeg.Options{Quality: 85}); err != nil {
http.Error(w, "Failed to encode frame", http.StatusInternalServerError)
return
}
w.Write(buf.Bytes())
}
func (s *Server) handleCaptureFrameAnnotated(w http.ResponseWriter, r *http.Request) {
frame := s.engine.CurrentFrame()
if frame == nil {
http.Error(w, "No frame available", http.StatusNoContent)
return
}
// Create a copy of the frame to draw on
bounds := frame.Bounds()
annotated := image.NewRGBA(bounds)
draw.Draw(annotated, bounds, frame, bounds.Min, draw.Src)
// Draw region overlays
regions := s.getRegions()
s.drawRegionOverlays(annotated, regions)
w.Header().Set("Content-Type", "image/jpeg")
var buf bytes.Buffer
if err := jpeg.Encode(&buf, annotated, &jpeg.Options{Quality: 85}); err != nil {
http.Error(w, "Failed to encode annotated frame", http.StatusInternalServerError)
return
}
w.Write(buf.Bytes())
}
func (s *Server) handleCaptureStats(w http.ResponseWriter, r *http.Request) {
status := s.engine.Status()
stats := status.CaptureStats
response := map[string]interface{}{
"frameCount": stats.FrameCount,
"avgCaptureMs": stats.AvgCaptureMs,
"fps": stats.FPS,
"lastCapture": stats.LastCapture,
"backend": "file", // TODO: Get actual backend name
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleCaptureUpload(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "Failed to read uploaded file", http.StatusBadRequest)
return
}
defer file.Close()
// Save to temporary directory
tempDir := "/tmp/iso-bot-uploads"
os.MkdirAll(tempDir, 0755)
filename := filepath.Join(tempDir, header.Filename)
out, err := os.Create(filename)
if err != nil {
http.Error(w, "Failed to save file", http.StatusInternalServerError)
return
}
defer out.Close()
_, err = io.Copy(out, file)
if err != nil {
http.Error(w, "Failed to save file", http.StatusInternalServerError)
return
}
response := map[string]string{
"filename": filename,
"message": "File uploaded successfully. Use /api/capture/source to switch to it.",
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleCaptureSource(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
Type string `json:"type"`
Config map[string]interface{} `json:"config"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "Invalid JSON", http.StatusBadRequest)
return
}
// TODO: Switch capture source
// This would require restarting the engine with a new source
response := map[string]string{
"message": "Capture source switching not implemented yet",
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleRegions(w http.ResponseWriter, r *http.Request) {
regions := s.getRegions()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(regions)
}
func (s *Server) handleState(w http.ResponseWriter, r *http.Request) {
status := s.engine.Status()
response := map[string]interface{}{
"gameState": string(status.GameState),
"healthPct": status.Vitals.HealthPct,
"manaPct": status.Vitals.ManaPct,
"xpPct": status.Vitals.XPPct,
"frameCount": status.FrameCount,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleLootRules(w http.ResponseWriter, r *http.Request) {
lootEngine := s.engine.LootEngine()
if lootEngine == nil {
response := map[string]interface{}{
"rules": []interface{}{},
"count": 0,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
return
}
response := map[string]interface{}{
"rules": lootEngine.Rules,
"count": len(lootEngine.Rules),
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleRoutines(w http.ResponseWriter, r *http.Request) {
routines := s.engine.GamePlugin().Routines()
var response []map[string]interface{}
for _, routine := range routines {
response = append(response, map[string]interface{}{
"name": routine.Name(),
"phase": routine.Phase(),
})
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handlePixel(w http.ResponseWriter, r *http.Request) {
xStr := r.URL.Query().Get("x")
yStr := r.URL.Query().Get("y")
x, err := strconv.Atoi(xStr)
if err != nil {
http.Error(w, "Invalid x coordinate", http.StatusBadRequest)
return
}
y, err := strconv.Atoi(yStr)
if err != nil {
http.Error(w, "Invalid y coordinate", http.StatusBadRequest)
return
}
frame := s.engine.CurrentFrame()
if frame == nil {
http.Error(w, "No frame available", http.StatusNoContent)
return
}
bounds := frame.Bounds()
point := image.Point{X: x, Y: y}
if !point.In(bounds) {
http.Error(w, "Coordinates out of bounds", http.StatusBadRequest)
return
}
c := frame.At(x, y)
red, green, blue, _ := c.RGBA()
// Convert to 8-bit
red, green, blue = red>>8, green>>8, blue>>8
hsv := vision.RGBToHSV(c)
response := PixelInfo{
X: x,
Y: y,
RGB: [3]int{int(red), int(green), int(blue)},
HSV: [3]int{hsv.H, hsv.S, hsv.V},
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) {
// TODO: Return actual config
response := map[string]interface{}{
"game": s.engine.GamePlugin().Info().ID,
"devMode": s.engine.Status().DevMode,
"resolution": fmt.Sprintf("%dx%d", 1920, 1080), // TODO: Get actual resolution
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handlePlugins(w http.ResponseWriter, r *http.Request) {
pluginInfo := s.engine.GamePlugin().Info()
response := []map[string]interface{}{
{
"id": pluginInfo.ID,
"name": pluginInfo.Name,
"version": pluginInfo.Version,
"description": pluginInfo.Description,
"resolution": fmt.Sprintf("%dx%d", pluginInfo.Resolution.X, pluginInfo.Resolution.Y),
},
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
// TODO: Signal engine to start
w.WriteHeader(http.StatusAccepted)
// TODO: Start the engine
response := map[string]string{
"message": "Start not implemented yet",
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) {
// TODO: Signal engine to stop
w.WriteHeader(http.StatusAccepted)
s.engine.Stop()
response := map[string]string{
"message": "Engine stopped",
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handlePause(w http.ResponseWriter, r *http.Request) {
var req struct {
Paused bool `json:"paused"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "Invalid JSON", http.StatusBadRequest)
return
}
s.engine.Pause(req.Paused)
action := "resumed"
if req.Paused {
action = "paused"
}
response := map[string]string{
"message": fmt.Sprintf("Engine %s", action),
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
// Helper functions
func (s *Server) getRegions() []RegionInfo {
gameID := s.engine.GamePlugin().Info().ID
// TODO: Get actual current resolution
width, height := 1920, 1080
registry := s.engine.ResolutionRegistry()
profile, err := registry.Get(gameID, width, height)
if err != nil {
log.Printf("Failed to get resolution profile: %v", err)
return nil
}
var regions []RegionInfo
for name, rect := range profile.Regions {
regions = append(regions, RegionInfo{
Name: name,
X: rect.Min.X,
Y: rect.Min.Y,
Width: rect.Dx(),
Height: rect.Dy(),
})
}
return regions
}
func (s *Server) drawRegionOverlays(img *image.RGBA, regions []RegionInfo) {
// Simple colored rectangles for region overlays
colors := []image.Uniform{
{C: color.RGBA{255, 0, 0, 128}}, // Red
{C: color.RGBA{0, 255, 0, 128}}, // Green
{C: color.RGBA{0, 0, 255, 128}}, // Blue
{C: color.RGBA{255, 255, 0, 128}}, // Yellow
{C: color.RGBA{255, 0, 255, 128}}, // Magenta
{C: color.RGBA{0, 255, 255, 128}}, // Cyan
}
for i, region := range regions {
color := &colors[i%len(colors)]
rect := image.Rect(region.X, region.Y, region.X+region.Width, region.Y+region.Height)
// Draw border (simple approach - just draw a few pixel wide border)
borderWidth := 2
// Top border
draw.Draw(img, image.Rect(rect.Min.X, rect.Min.Y, rect.Max.X, rect.Min.Y+borderWidth), color, image.Point{}, draw.Over)
// Bottom border
draw.Draw(img, image.Rect(rect.Min.X, rect.Max.Y-borderWidth, rect.Max.X, rect.Max.Y), color, image.Point{}, draw.Over)
// Left border
draw.Draw(img, image.Rect(rect.Min.X, rect.Min.Y, rect.Min.X+borderWidth, rect.Max.Y), color, image.Point{}, draw.Over)
// Right border
draw.Draw(img, image.Rect(rect.Max.X-borderWidth, rect.Min.Y, rect.Max.X, rect.Max.Y), color, image.Point{}, draw.Over)
}
}

View file

@ -129,14 +129,24 @@ func ParseBackendType(s string) (BackendType, error) {
func GetDefault() *Registry {
reg := NewRegistry()
// Register platform-specific backends
reg.Register(BackendWindowWin32, NewWin32Source)
reg.Register(BackendWindowX11, NewX11Source)
reg.Register(BackendWayland, NewWaylandSource)
reg.Register(BackendVNC, NewVNCSource)
reg.Register(BackendSpice, NewSpiceSource)
reg.Register(BackendMonitor, NewMonitorSource)
// Always register these core backends that work on all platforms
reg.Register(BackendFile, NewFileSource)
// Platform-specific backends are registered via init() functions
// with appropriate build tags to avoid compilation errors
return reg
}
// init function registers platform-specific backends.
// Each platform will have its own init function with build tags.
func init() {
defaultRegistry := GetDefault()
// Register monitor backend (should work on most platforms)
if MonitorSourceAvailable() {
defaultRegistry.Register(BackendMonitor, NewMonitorSource)
}
// Other backends registered via platform-specific init() functions
}

View file

@ -0,0 +1,15 @@
// Helper functions for backends
package backends
import "runtime"
// MonitorSourceAvailable returns true if monitor capture is available on this platform.
func MonitorSourceAvailable() bool {
// Monitor capture should work on most platforms, but let's be conservative
switch runtime.GOOS {
case "windows", "linux", "darwin":
return true
default:
return false
}
}

View file

@ -97,3 +97,13 @@ func (m *Manager) Stats() Stats {
func (m *Manager) Close() error {
return m.source.Close()
}
// Source returns the underlying capture source.
func (m *Manager) Source() Source {
return m.source
}
// Size returns the source dimensions.
func (m *Manager) Size() (width, height int) {
return m.source.Size()
}

345
pkg/engine/engine.go Normal file
View file

@ -0,0 +1,345 @@
// Package engine provides the core bot engine that wires all components together.
package engine
import (
"context"
"fmt"
"image"
"log"
"sync"
"time"
"git.cloonar.com/openclawd/iso-bot/pkg/engine/capture"
"git.cloonar.com/openclawd/iso-bot/pkg/engine/resolution"
"git.cloonar.com/openclawd/iso-bot/pkg/engine/state"
"git.cloonar.com/openclawd/iso-bot/pkg/engine/loot"
"git.cloonar.com/openclawd/iso-bot/pkg/plugin"
)
// Engine is the core bot engine that coordinates all subsystems.
type Engine struct {
mu sync.RWMutex
running bool
paused bool
devMode bool
// Core components
captureManager *capture.Manager
resolutionRegistry *resolution.Registry
stateManager *state.Manager
lootEngine *loot.RuleEngine
// Plugin system
gamePlugin plugin.Plugin
services *engineServices
// Current state
currentFrame image.Image
gameState plugin.GameState
vitals plugin.VitalStats
// Statistics
frameCount uint64
itemsFound int
runCount int
startTime time.Time
// Control channels
stopChan chan struct{}
pauseChan chan bool
}
// NewEngine creates a new bot engine.
func NewEngine(captureSource capture.Source, gamePlugin plugin.Plugin, devMode bool) (*Engine, error) {
engine := &Engine{
captureManager: capture.NewManager(captureSource),
resolutionRegistry: resolution.NewRegistry(),
devMode: devMode,
gamePlugin: gamePlugin,
stopChan: make(chan struct{}),
pauseChan: make(chan bool, 1),
startTime: time.Now(),
}
// Create engine services for the plugin
engine.services = &engineServices{engine: engine}
// Initialize the plugin
if err := gamePlugin.Init(engine.services); err != nil {
return nil, fmt.Errorf("failed to initialize game plugin: %w", err)
}
// Set up state manager
engine.stateManager = state.NewManager(gamePlugin.Detector())
// Set up default loot filter
if lootFilter := gamePlugin.DefaultLootFilter(); lootFilter != nil {
if ruleEngine, ok := lootFilter.(*loot.RuleEngine); ok {
engine.lootEngine = ruleEngine
}
}
return engine, nil
}
// Start begins the engine's main loop.
func (e *Engine) Start(ctx context.Context) error {
e.mu.Lock()
if e.running {
e.mu.Unlock()
return fmt.Errorf("engine is already running")
}
e.running = true
e.mu.Unlock()
log.Printf("Engine starting (dev mode: %v)", e.devMode)
ticker := time.NewTicker(33 * time.Millisecond) // ~30 FPS
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-e.stopChan:
return nil
case paused := <-e.pauseChan:
e.mu.Lock()
e.paused = paused
e.mu.Unlock()
if paused {
log.Println("Engine paused")
} else {
log.Println("Engine resumed")
}
case <-ticker.C:
if e.isPaused() {
continue
}
if err := e.processFrame(); err != nil {
log.Printf("Frame processing error: %v", err)
continue
}
}
}
}
// Stop stops the engine.
func (e *Engine) Stop() {
e.mu.Lock()
defer e.mu.Unlock()
if !e.running {
return
}
e.running = false
close(e.stopChan)
log.Println("Engine stopped")
}
// Pause pauses or resumes the engine.
func (e *Engine) Pause(paused bool) {
select {
case e.pauseChan <- paused:
default:
// Channel is full, already have a pending pause state
}
}
// Status returns the current engine status.
func (e *Engine) Status() EngineStatus {
e.mu.RLock()
defer e.mu.RUnlock()
return EngineStatus{
Running: e.running,
Paused: e.paused,
DevMode: e.devMode,
GameState: e.gameState,
Vitals: e.vitals,
FrameCount: e.frameCount,
ItemsFound: e.itemsFound,
RunCount: e.runCount,
Uptime: time.Since(e.startTime),
CaptureStats: e.captureManager.Stats(),
}
}
// CurrentFrame returns the most recent captured frame.
func (e *Engine) CurrentFrame() image.Image {
e.mu.RLock()
defer e.mu.RUnlock()
return e.currentFrame
}
// GamePlugin returns the active game plugin.
func (e *Engine) GamePlugin() plugin.Plugin {
return e.gamePlugin
}
// ResolutionRegistry returns the resolution registry.
func (e *Engine) ResolutionRegistry() *resolution.Registry {
return e.resolutionRegistry
}
// LootEngine returns the loot filter engine.
func (e *Engine) LootEngine() *loot.RuleEngine {
return e.lootEngine
}
// processFrame captures and analyzes a single frame.
func (e *Engine) processFrame() error {
// Capture frame
frame, err := e.captureManager.Capture()
if err != nil {
return fmt.Errorf("capture failed: %w", err)
}
e.mu.Lock()
e.currentFrame = frame
e.frameCount++
e.mu.Unlock()
// Update game state
gameState := e.stateManager.Update(frame)
// Read vitals if in-game
var vitals plugin.VitalStats
if gameState == plugin.StateInGame {
vitals = e.gamePlugin.Detector().ReadVitals(frame)
}
e.mu.Lock()
e.gameState = gameState
e.vitals = vitals
e.mu.Unlock()
// In dev mode, we don't perform any actions
if e.devMode {
return nil
}
// TODO: Implement bot actions based on state and detected items
return nil
}
// isPaused returns true if the engine is paused.
func (e *Engine) isPaused() bool {
e.mu.RLock()
defer e.mu.RUnlock()
return e.paused
}
// EngineStatus represents the current state of the engine.
type EngineStatus struct {
Running bool `json:"running"`
Paused bool `json:"paused"`
DevMode bool `json:"devMode"`
GameState plugin.GameState `json:"gameState"`
Vitals plugin.VitalStats `json:"vitals"`
FrameCount uint64 `json:"frameCount"`
ItemsFound int `json:"itemsFound"`
RunCount int `json:"runCount"`
Uptime time.Duration `json:"uptime"`
CaptureStats capture.Stats `json:"captureStats"`
}
// engineServices implements plugin.EngineServices.
type engineServices struct {
engine *Engine
}
// Capture returns the current screen frame.
func (s *engineServices) Capture() image.Image {
return s.engine.CurrentFrame()
}
// CaptureSource returns the active capture source.
func (s *engineServices) CaptureSource() capture.Source {
return s.engine.captureManager.Source()
}
// Resolution returns the current capture resolution.
func (s *engineServices) Resolution() (width, height int) {
return s.engine.captureManager.Size()
}
// Region returns a named screen region for the current game and resolution.
func (s *engineServices) Region(name string) image.Rectangle {
width, height := s.Resolution()
gameID := s.engine.gamePlugin.Info().ID
region, err := s.engine.resolutionRegistry.GetRegion(gameID, width, height, name)
if err != nil {
log.Printf("Warning: region %q not found: %v", name, err)
return image.Rectangle{}
}
return region
}
// Click sends a mouse click at the given position.
func (s *engineServices) Click(pos image.Point) {
if s.engine.devMode {
log.Printf("Dev mode: would click at (%d, %d)", pos.X, pos.Y)
return
}
// TODO: Implement actual mouse click
}
// MoveMouse moves the mouse to the given position.
func (s *engineServices) MoveMouse(pos image.Point) {
if s.engine.devMode {
log.Printf("Dev mode: would move mouse to (%d, %d)", pos.X, pos.Y)
return
}
// TODO: Implement actual mouse movement
}
// PressKey sends a key press.
func (s *engineServices) PressKey(key string) {
if s.engine.devMode {
log.Printf("Dev mode: would press key %q", key)
return
}
// TODO: Implement actual key press
}
// TypeText types text with human-like delays.
func (s *engineServices) TypeText(text string) {
if s.engine.devMode {
log.Printf("Dev mode: would type %q", text)
return
}
// TODO: Implement actual text typing
}
// Wait pauses for a human-like delay.
func (s *engineServices) Wait() {
time.Sleep(time.Duration(100+rand.Intn(200)) * time.Millisecond)
}
// WaitMs pauses for a specific duration with randomization.
func (s *engineServices) WaitMs(baseMs int, varianceMs int) {
variance := time.Duration(rand.Intn(varianceMs*2)-varianceMs) * time.Millisecond
delay := time.Duration(baseMs)*time.Millisecond + variance
time.Sleep(delay)
}
// Log logs a message associated with the plugin.
func (s *engineServices) Log(level string, msg string, args ...any) {
prefix := fmt.Sprintf("[%s] %s", s.engine.gamePlugin.Info().ID, level)
log.Printf(prefix+": "+msg, args...)
}
// Random number generator (TODO: use crypto/rand for better randomness)
var rand = struct {
Intn func(int) int
}{
Intn: func(n int) int {
return int(time.Now().UnixNano()) % n
},
}

View file

@ -1,12 +1,13 @@
// Package vision provides computer vision utilities for game screen analysis.
//
// Uses GoCV (OpenCV bindings for Go) for template matching, color detection,
// and contour analysis. Designed for high-throughput real-time analysis.
// Pure Go implementation without external dependencies like OpenCV.
// Designed for high-throughput real-time analysis of game screens.
package vision
import (
"image"
"image/color"
"math"
)
// Match represents a detected element on screen.
@ -31,6 +32,11 @@ type ColorRange struct {
UpperH, UpperS, UpperV int
}
// HSV represents a color in HSV color space.
type HSV struct {
H, S, V int
}
// Pipeline processes frames through a series of vision operations.
type Pipeline struct {
templates map[string]*Template
@ -58,31 +64,293 @@ func (p *Pipeline) LoadTemplate(name string, img image.Image) {
// FindTemplate searches for a template in the frame.
// Returns the best match above threshold, or nil.
// This is a simple implementation - could be improved with better algorithms.
func (p *Pipeline) FindTemplate(frame image.Image, templateName string) *Match {
// TODO: Implement with GoCV matchTemplate
// This is a stub — actual implementation needs gocv.MatchTemplate
template, exists := p.templates[templateName]
if !exists {
return nil
}
frameBounds := frame.Bounds()
templateBounds := template.Image.Bounds()
// Simple template matching by scanning every position
bestMatch := &Match{Confidence: 0}
for y := frameBounds.Min.Y; y <= frameBounds.Max.Y-templateBounds.Dy(); y++ {
for x := frameBounds.Min.X; x <= frameBounds.Max.X-templateBounds.Dx(); x++ {
confidence := p.compareAtPosition(frame, template.Image, x, y)
if confidence > bestMatch.Confidence {
bestMatch.Position = image.Point{X: x, Y: y}
bestMatch.BBox = image.Rect(x, y, x+templateBounds.Dx(), y+templateBounds.Dy())
bestMatch.Confidence = confidence
bestMatch.Label = templateName
}
}
}
if bestMatch.Confidence >= p.threshold {
return bestMatch
}
return nil
}
// FindAllTemplates finds all matches of a template above threshold.
func (p *Pipeline) FindAllTemplates(frame image.Image, templateName string) []Match {
// TODO: Implement with GoCV + NMS
// For simplicity, just return the best match
match := p.FindTemplate(frame, templateName)
if match != nil {
return []Match{*match}
}
return nil
}
// FindByColor detects regions matching an HSV color range.
func (p *Pipeline) FindByColor(frame image.Image, colorRange ColorRange, minArea int) []Match {
// TODO: Implement with GoCV inRange + findContours
return nil
bounds := frame.Bounds()
var matches []Match
// Simple blob detection by scanning for connected regions
visited := make(map[image.Point]bool)
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
pt := image.Point{X: x, Y: y}
if visited[pt] {
continue
}
c := frame.At(x, y)
hsv := RGBToHSV(c)
if p.colorInRange(hsv, colorRange) {
// Found a pixel in range, flood fill to find the blob
blob := p.floodFill(frame, pt, colorRange, visited)
if len(blob) >= minArea {
bbox := p.getBoundingBox(blob)
center := image.Point{
X: (bbox.Min.X + bbox.Max.X) / 2,
Y: (bbox.Min.Y + bbox.Max.Y) / 2,
}
matches = append(matches, Match{
Position: center,
BBox: bbox,
Confidence: 1.0, // Binary detection for color matching
Label: "color_match",
})
}
}
}
}
return matches
}
// ReadBarPercentage reads a horizontal bar's fill level (health, mana, xp).
func (p *Pipeline) ReadBarPercentage(frame image.Image, barRegion image.Rectangle, filledColor ColorRange) float64 {
// TODO: Implement — scan columns for filled color ratio
return 0.0
bounds := barRegion.Intersect(frame.Bounds())
if bounds.Empty() {
return 0.0
}
totalPixels := 0
filledPixels := 0
// Sample pixels across the width of the bar
centerY := (bounds.Min.Y + bounds.Max.Y) / 2
for x := bounds.Min.X; x < bounds.Max.X; x++ {
c := frame.At(x, centerY)
hsv := RGBToHSV(c)
totalPixels++
if p.colorInRange(hsv, filledColor) {
filledPixels++
}
}
if totalPixels == 0 {
return 0.0
}
return float64(filledPixels) / float64(totalPixels)
}
// GetPixelColor returns the color at a specific pixel.
func (p *Pipeline) GetPixelColor(frame image.Image, x, y int) color.Color {
return frame.At(x, y)
}
// GetPixelHSV returns the HSV values at a specific pixel.
func (p *Pipeline) GetPixelHSV(frame image.Image, x, y int) HSV {
c := frame.At(x, y)
return RGBToHSV(c)
}
// HasColorInRegion checks if any pixel in the region matches the color range.
func (p *Pipeline) HasColorInRegion(frame image.Image, region image.Rectangle, colorRange ColorRange) bool {
bounds := region.Intersect(frame.Bounds())
if bounds.Empty() {
return false
}
// Sample every few pixels for performance
step := 2
for y := bounds.Min.Y; y < bounds.Max.Y; y += step {
for x := bounds.Min.X; x < bounds.Max.X; x += step {
c := frame.At(x, y)
hsv := RGBToHSV(c)
if p.colorInRange(hsv, colorRange) {
return true
}
}
}
return false
}
// compareAtPosition compares template with frame at given position.
func (p *Pipeline) compareAtPosition(frame, template image.Image, frameX, frameY int) float64 {
templateBounds := template.Bounds()
totalPixels := 0
matchingPixels := 0
// Simple pixel-by-pixel comparison
for y := templateBounds.Min.Y; y < templateBounds.Max.Y; y++ {
for x := templateBounds.Min.X; x < templateBounds.Max.X; x++ {
frameColor := frame.At(frameX+x, frameY+y)
templateColor := template.At(x, y)
totalPixels++
if p.colorsMatch(frameColor, templateColor, 30) { // tolerance of 30
matchingPixels++
}
}
}
return float64(matchingPixels) / float64(totalPixels)
}
// colorsMatch checks if two colors are similar within tolerance.
func (p *Pipeline) colorsMatch(c1, c2 color.Color, tolerance int) bool {
r1, g1, b1, _ := c1.RGBA()
r2, g2, b2, _ := c2.RGBA()
// Convert from 16-bit to 8-bit
r1, g1, b1 = r1>>8, g1>>8, b1>>8
r2, g2, b2 = r2>>8, g2>>8, b2>>8
dr := int(r1) - int(r2)
dg := int(g1) - int(g2)
db := int(b1) - int(b2)
if dr < 0 { dr = -dr }
if dg < 0 { dg = -dg }
if db < 0 { db = -db }
return dr <= tolerance && dg <= tolerance && db <= tolerance
}
// colorInRange checks if HSV color is within range.
func (p *Pipeline) colorInRange(hsv HSV, colorRange ColorRange) bool {
return hsv.H >= colorRange.LowerH && hsv.H <= colorRange.UpperH &&
hsv.S >= colorRange.LowerS && hsv.S <= colorRange.UpperS &&
hsv.V >= colorRange.LowerV && hsv.V <= colorRange.UpperV
}
// floodFill finds connected pixels of the same color.
func (p *Pipeline) floodFill(frame image.Image, start image.Point, colorRange ColorRange, visited map[image.Point]bool) []image.Point {
bounds := frame.Bounds()
var blob []image.Point
stack := []image.Point{start}
for len(stack) > 0 {
pt := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if visited[pt] || !pt.In(bounds) {
continue
}
c := frame.At(pt.X, pt.Y)
hsv := RGBToHSV(c)
if !p.colorInRange(hsv, colorRange) {
continue
}
visited[pt] = true
blob = append(blob, pt)
// Add neighbors
neighbors := []image.Point{
{X: pt.X-1, Y: pt.Y},
{X: pt.X+1, Y: pt.Y},
{X: pt.X, Y: pt.Y-1},
{X: pt.X, Y: pt.Y+1},
}
stack = append(stack, neighbors...)
}
return blob
}
// getBoundingBox calculates the bounding box for a set of points.
func (p *Pipeline) getBoundingBox(points []image.Point) image.Rectangle {
if len(points) == 0 {
return image.Rectangle{}
}
minX, minY := points[0].X, points[0].Y
maxX, maxY := minX, minY
for _, pt := range points {
if pt.X < minX { minX = pt.X }
if pt.X > maxX { maxX = pt.X }
if pt.Y < minY { minY = pt.Y }
if pt.Y > maxY { maxY = pt.Y }
}
return image.Rect(minX, minY, maxX+1, maxY+1)
}
// RGBToHSV converts RGB color to HSV.
func RGBToHSV(c color.Color) HSV {
r, g, b, _ := c.RGBA()
// Convert from 16-bit to float [0,1]
rf := float64(r>>8) / 255.0
gf := float64(g>>8) / 255.0
bf := float64(b>>8) / 255.0
max := math.Max(rf, math.Max(gf, bf))
min := math.Min(rf, math.Min(gf, bf))
delta := max - min
// Value
v := max
// Saturation
var s float64
if max != 0 {
s = delta / max
}
// Hue
var h float64
if delta != 0 {
switch max {
case rf:
h = math.Mod((gf-bf)/delta, 6)
case gf:
h = (bf-rf)/delta + 2
case bf:
h = (rf-gf)/delta + 4
}
h *= 60
if h < 0 {
h += 360
}
}
return HSV{
H: int(h),
S: int(s * 255),
V: int(v * 255),
}
}