diff --git a/pkg/api/api.go b/pkg/api/api.go index b3dde63..36d692c 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -2,30 +2,49 @@ // // Endpoints: // GET /api/status — bot status, current state, routine, stats -// GET /api/config — current configuration -// PUT /api/config — update configuration -// POST /api/start — start bot with routine -// POST /api/stop — stop bot -// POST /api/pause — pause/resume +// GET /api/capture/frame — returns current frame as JPEG +// GET /api/capture/frame/annotated — frame with region overlays +// GET /api/capture/stats — capture performance stats +// POST /api/capture/upload — upload a screenshot for testing +// POST /api/capture/source — switch capture source +// GET /api/regions — list all defined regions +// GET /api/state — current detected game state + vitals +// GET /api/loot/rules — get current loot filter rules // GET /api/routines — list available routines -// GET /api/loot/rules — get loot filter rules -// PUT /api/loot/rules — update loot filter rules -// GET /api/stats — run statistics, items found, etc. -// WS /api/ws — real-time status stream +// GET /api/pixel — get pixel color at coordinates +// GET /api/config — current config +// GET /api/plugins — list loaded plugins and info +// WS /api/ws — WebSocket for real-time updates +// GET / — serve dev dashboard // // The API is served by the bot process itself (single binary). package api import ( + "bytes" "encoding/json" + "fmt" + "image" + "image/color" + "image/draw" + "image/jpeg" + "io" + "log" "net/http" + "os" + "path/filepath" + "strconv" "sync" + + "git.cloonar.com/openclawd/iso-bot/pkg/engine" + "git.cloonar.com/openclawd/iso-bot/pkg/engine/vision" ) // Status represents the current bot status. type Status struct { Running bool `json:"running"` Paused bool `json:"paused"` + DevMode bool `json:"devMode"` GameState string `json:"gameState"` Routine string `json:"routine,omitempty"` Phase string `json:"phase,omitempty"` @@ -37,19 +56,39 @@ type Status struct { ManaPct float64 `json:"manaPct"` } +// PixelInfo represents RGB and HSV values at a pixel. +type PixelInfo struct { + X int `json:"x"` + Y int `json:"y"` + RGB [3]int `json:"rgb"` + HSV [3]int `json:"hsv"` +} + +// RegionInfo represents a screen region. +type RegionInfo struct { + Name string `json:"name"` + X int `json:"x"` + Y int `json:"y"` + Width int `json:"width"` + Height int `json:"height"` +} + // Server provides the HTTP API and WebSocket endpoint. type Server struct { mu sync.RWMutex - status Status + engine *engine.Engine addr string mux *http.ServeMux + webRoot string } // NewServer creates an API server on the given address. -func NewServer(addr string) *Server { +func NewServer(addr string, eng *engine.Engine, webRoot string) *Server { s := &Server{ - addr: addr, - mux: http.NewServeMux(), + engine: eng, + addr: addr, + mux: http.NewServeMux(), + webRoot: webRoot, } s.registerRoutes() return s @@ -57,37 +96,427 @@ func NewServer(addr string) *Server { // Start begins serving the API. func (s *Server) Start() error { - return http.ListenAndServe(s.addr, s.mux) -} - -// UpdateStatus updates the bot status (called by the engine). -func (s *Server) UpdateStatus(status Status) { - s.mu.Lock() - defer s.mu.Unlock() - s.status = status - // TODO: Broadcast to WebSocket clients + return http.ListenAndServe(s.addr, s.enableCORS(s.mux)) } func (s *Server) registerRoutes() { + // API routes s.mux.HandleFunc("GET /api/status", s.handleStatus) + s.mux.HandleFunc("GET /api/capture/frame", s.handleCaptureFrame) + s.mux.HandleFunc("GET /api/capture/frame/annotated", s.handleCaptureFrameAnnotated) + s.mux.HandleFunc("GET /api/capture/stats", s.handleCaptureStats) + s.mux.HandleFunc("POST /api/capture/upload", s.handleCaptureUpload) + s.mux.HandleFunc("POST /api/capture/source", s.handleCaptureSource) + s.mux.HandleFunc("GET /api/regions", s.handleRegions) + s.mux.HandleFunc("GET /api/state", s.handleState) + s.mux.HandleFunc("GET /api/loot/rules", s.handleLootRules) + s.mux.HandleFunc("GET /api/routines", s.handleRoutines) + s.mux.HandleFunc("GET /api/pixel", s.handlePixel) + s.mux.HandleFunc("GET /api/config", s.handleConfig) + s.mux.HandleFunc("GET /api/plugins", s.handlePlugins) s.mux.HandleFunc("POST /api/start", s.handleStart) s.mux.HandleFunc("POST /api/stop", s.handleStop) - // TODO: Remaining routes + s.mux.HandleFunc("POST /api/pause", s.handlePause) + + // WebSocket endpoint + // s.mux.HandleFunc("/api/ws", s.handleWebSocket) + + // Serve static files for dev dashboard + s.mux.Handle("/", http.FileServer(http.Dir(s.webRoot))) +} + +func (s *Server) enableCORS(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") + + if r.Method == "OPTIONS" { + w.WriteHeader(http.StatusOK) + return + } + + next.ServeHTTP(w, r) + }) } func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { - s.mu.RLock() - defer s.mu.RUnlock() + status := s.engine.Status() + + response := Status{ + Running: status.Running, + Paused: status.Paused, + DevMode: status.DevMode, + GameState: string(status.GameState), + Uptime: status.Uptime.String(), + CaptureFPS: status.CaptureStats.FPS, + HealthPct: status.Vitals.HealthPct, + ManaPct: status.Vitals.ManaPct, + ItemsFound: status.ItemsFound, + RunCount: status.RunCount, + } + w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(s.status) + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleCaptureFrame(w http.ResponseWriter, r *http.Request) { + frame := s.engine.CurrentFrame() + if frame == nil { + http.Error(w, "No frame available", http.StatusNoContent) + return + } + + w.Header().Set("Content-Type", "image/jpeg") + + var buf bytes.Buffer + if err := jpeg.Encode(&buf, frame, &jpeg.Options{Quality: 85}); err != nil { + http.Error(w, "Failed to encode frame", http.StatusInternalServerError) + return + } + + w.Write(buf.Bytes()) +} + +func (s *Server) handleCaptureFrameAnnotated(w http.ResponseWriter, r *http.Request) { + frame := s.engine.CurrentFrame() + if frame == nil { + http.Error(w, "No frame available", http.StatusNoContent) + return + } + + // Create a copy of the frame to draw on + bounds := frame.Bounds() + annotated := image.NewRGBA(bounds) + draw.Draw(annotated, bounds, frame, bounds.Min, draw.Src) + + // Draw region overlays + regions := s.getRegions() + s.drawRegionOverlays(annotated, regions) + + w.Header().Set("Content-Type", "image/jpeg") + + var buf bytes.Buffer + if err := jpeg.Encode(&buf, annotated, &jpeg.Options{Quality: 85}); err != nil { + http.Error(w, "Failed to encode annotated frame", http.StatusInternalServerError) + return + } + + w.Write(buf.Bytes()) +} + +func (s *Server) handleCaptureStats(w http.ResponseWriter, r *http.Request) { + status := s.engine.Status() + stats := status.CaptureStats + + response := map[string]interface{}{ + "frameCount": stats.FrameCount, + "avgCaptureMs": stats.AvgCaptureMs, + "fps": stats.FPS, + "lastCapture": stats.LastCapture, + "backend": "file", // TODO: Get actual backend name + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleCaptureUpload(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + file, header, err := r.FormFile("file") + if err != nil { + http.Error(w, "Failed to read uploaded file", http.StatusBadRequest) + return + } + defer file.Close() + + // Save to temporary directory + tempDir := "/tmp/iso-bot-uploads" + os.MkdirAll(tempDir, 0755) + + filename := filepath.Join(tempDir, header.Filename) + out, err := os.Create(filename) + if err != nil { + http.Error(w, "Failed to save file", http.StatusInternalServerError) + return + } + defer out.Close() + + _, err = io.Copy(out, file) + if err != nil { + http.Error(w, "Failed to save file", http.StatusInternalServerError) + return + } + + response := map[string]string{ + "filename": filename, + "message": "File uploaded successfully. Use /api/capture/source to switch to it.", + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleCaptureSource(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + var req struct { + Type string `json:"type"` + Config map[string]interface{} `json:"config"` + } + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "Invalid JSON", http.StatusBadRequest) + return + } + + // TODO: Switch capture source + // This would require restarting the engine with a new source + response := map[string]string{ + "message": "Capture source switching not implemented yet", + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleRegions(w http.ResponseWriter, r *http.Request) { + regions := s.getRegions() + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(regions) +} + +func (s *Server) handleState(w http.ResponseWriter, r *http.Request) { + status := s.engine.Status() + + response := map[string]interface{}{ + "gameState": string(status.GameState), + "healthPct": status.Vitals.HealthPct, + "manaPct": status.Vitals.ManaPct, + "xpPct": status.Vitals.XPPct, + "frameCount": status.FrameCount, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleLootRules(w http.ResponseWriter, r *http.Request) { + lootEngine := s.engine.LootEngine() + if lootEngine == nil { + response := map[string]interface{}{ + "rules": []interface{}{}, + "count": 0, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + response := map[string]interface{}{ + "rules": lootEngine.Rules, + "count": len(lootEngine.Rules), + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleRoutines(w http.ResponseWriter, r *http.Request) { + routines := s.engine.GamePlugin().Routines() + + var response []map[string]interface{} + for _, routine := range routines { + response = append(response, map[string]interface{}{ + "name": routine.Name(), + "phase": routine.Phase(), + }) + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handlePixel(w http.ResponseWriter, r *http.Request) { + xStr := r.URL.Query().Get("x") + yStr := r.URL.Query().Get("y") + + x, err := strconv.Atoi(xStr) + if err != nil { + http.Error(w, "Invalid x coordinate", http.StatusBadRequest) + return + } + + y, err := strconv.Atoi(yStr) + if err != nil { + http.Error(w, "Invalid y coordinate", http.StatusBadRequest) + return + } + + frame := s.engine.CurrentFrame() + if frame == nil { + http.Error(w, "No frame available", http.StatusNoContent) + return + } + + bounds := frame.Bounds() + point := image.Point{X: x, Y: y} + if !point.In(bounds) { + http.Error(w, "Coordinates out of bounds", http.StatusBadRequest) + return + } + + c := frame.At(x, y) + red, green, blue, _ := c.RGBA() + // Convert to 8-bit + red, green, blue = red>>8, green>>8, blue>>8 + + hsv := vision.RGBToHSV(c) + + response := PixelInfo{ + X: x, + Y: y, + RGB: [3]int{int(red), int(green), int(blue)}, + HSV: [3]int{hsv.H, hsv.S, hsv.V}, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) { + // TODO: Return actual config + response := map[string]interface{}{ + "game": s.engine.GamePlugin().Info().ID, + "devMode": s.engine.Status().DevMode, + "resolution": fmt.Sprintf("%dx%d", 1920, 1080), // TODO: Get actual resolution + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handlePlugins(w http.ResponseWriter, r *http.Request) { + pluginInfo := s.engine.GamePlugin().Info() + + response := []map[string]interface{}{ + { + "id": pluginInfo.ID, + "name": pluginInfo.Name, + "version": pluginInfo.Version, + "description": pluginInfo.Description, + "resolution": fmt.Sprintf("%dx%d", pluginInfo.Resolution.X, pluginInfo.Resolution.Y), + }, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) } func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { - // TODO: Signal engine to start - w.WriteHeader(http.StatusAccepted) + // TODO: Start the engine + response := map[string]string{ + "message": "Start not implemented yet", + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) } func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { - // TODO: Signal engine to stop - w.WriteHeader(http.StatusAccepted) + s.engine.Stop() + response := map[string]string{ + "message": "Engine stopped", + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (s *Server) handlePause(w http.ResponseWriter, r *http.Request) { + var req struct { + Paused bool `json:"paused"` + } + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "Invalid JSON", http.StatusBadRequest) + return + } + + s.engine.Pause(req.Paused) + + action := "resumed" + if req.Paused { + action = "paused" + } + + response := map[string]string{ + "message": fmt.Sprintf("Engine %s", action), + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +// Helper functions + +func (s *Server) getRegions() []RegionInfo { + gameID := s.engine.GamePlugin().Info().ID + // TODO: Get actual current resolution + width, height := 1920, 1080 + + registry := s.engine.ResolutionRegistry() + profile, err := registry.Get(gameID, width, height) + if err != nil { + log.Printf("Failed to get resolution profile: %v", err) + return nil + } + + var regions []RegionInfo + for name, rect := range profile.Regions { + regions = append(regions, RegionInfo{ + Name: name, + X: rect.Min.X, + Y: rect.Min.Y, + Width: rect.Dx(), + Height: rect.Dy(), + }) + } + + return regions +} + +func (s *Server) drawRegionOverlays(img *image.RGBA, regions []RegionInfo) { + // Simple colored rectangles for region overlays + colors := []image.Uniform{ + {C: color.RGBA{255, 0, 0, 128}}, // Red + {C: color.RGBA{0, 255, 0, 128}}, // Green + {C: color.RGBA{0, 0, 255, 128}}, // Blue + {C: color.RGBA{255, 255, 0, 128}}, // Yellow + {C: color.RGBA{255, 0, 255, 128}}, // Magenta + {C: color.RGBA{0, 255, 255, 128}}, // Cyan + } + + for i, region := range regions { + color := &colors[i%len(colors)] + rect := image.Rect(region.X, region.Y, region.X+region.Width, region.Y+region.Height) + + // Draw border (simple approach - just draw a few pixel wide border) + borderWidth := 2 + + // Top border + draw.Draw(img, image.Rect(rect.Min.X, rect.Min.Y, rect.Max.X, rect.Min.Y+borderWidth), color, image.Point{}, draw.Over) + // Bottom border + draw.Draw(img, image.Rect(rect.Min.X, rect.Max.Y-borderWidth, rect.Max.X, rect.Max.Y), color, image.Point{}, draw.Over) + // Left border + draw.Draw(img, image.Rect(rect.Min.X, rect.Min.Y, rect.Min.X+borderWidth, rect.Max.Y), color, image.Point{}, draw.Over) + // Right border + draw.Draw(img, image.Rect(rect.Max.X-borderWidth, rect.Min.Y, rect.Max.X, rect.Max.Y), color, image.Point{}, draw.Over) + } } diff --git a/pkg/engine/capture/backends/registry.go b/pkg/engine/capture/backends/registry.go index 0facb96..760e836 100644 --- a/pkg/engine/capture/backends/registry.go +++ b/pkg/engine/capture/backends/registry.go @@ -129,14 +129,24 @@ func ParseBackendType(s string) (BackendType, error) { func GetDefault() *Registry { reg := NewRegistry() - // Register platform-specific backends - reg.Register(BackendWindowWin32, NewWin32Source) - reg.Register(BackendWindowX11, NewX11Source) - reg.Register(BackendWayland, NewWaylandSource) - reg.Register(BackendVNC, NewVNCSource) - reg.Register(BackendSpice, NewSpiceSource) - reg.Register(BackendMonitor, NewMonitorSource) + // Always register these core backends that work on all platforms reg.Register(BackendFile, NewFileSource) - + + // Platform-specific backends are registered via init() functions + // with appropriate build tags to avoid compilation errors + return reg +} + +// init function registers platform-specific backends. +// Each platform will have its own init function with build tags. +func init() { + defaultRegistry := GetDefault() + + // Register monitor backend (should work on most platforms) + if MonitorSourceAvailable() { + defaultRegistry.Register(BackendMonitor, NewMonitorSource) + } + + // Other backends registered via platform-specific init() functions } \ No newline at end of file diff --git a/pkg/engine/capture/backends/stubs.go b/pkg/engine/capture/backends/stubs.go new file mode 100644 index 0000000..b0d1a71 --- /dev/null +++ b/pkg/engine/capture/backends/stubs.go @@ -0,0 +1,15 @@ +// Helper functions for backends +package backends + +import "runtime" + +// MonitorSourceAvailable returns true if monitor capture is available on this platform. +func MonitorSourceAvailable() bool { + // Monitor capture should work on most platforms, but let's be conservative + switch runtime.GOOS { + case "windows", "linux", "darwin": + return true + default: + return false + } +} \ No newline at end of file diff --git a/pkg/engine/capture/capture.go b/pkg/engine/capture/capture.go index 7c3fa6a..6cd8b4c 100644 --- a/pkg/engine/capture/capture.go +++ b/pkg/engine/capture/capture.go @@ -97,3 +97,13 @@ func (m *Manager) Stats() Stats { func (m *Manager) Close() error { return m.source.Close() } + +// Source returns the underlying capture source. +func (m *Manager) Source() Source { + return m.source +} + +// Size returns the source dimensions. +func (m *Manager) Size() (width, height int) { + return m.source.Size() +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go new file mode 100644 index 0000000..d64962b --- /dev/null +++ b/pkg/engine/engine.go @@ -0,0 +1,345 @@ +// Package engine provides the core bot engine that wires all components together. +package engine + +import ( + "context" + "fmt" + "image" + "log" + "sync" + "time" + + "git.cloonar.com/openclawd/iso-bot/pkg/engine/capture" + "git.cloonar.com/openclawd/iso-bot/pkg/engine/resolution" + "git.cloonar.com/openclawd/iso-bot/pkg/engine/state" + "git.cloonar.com/openclawd/iso-bot/pkg/engine/loot" + "git.cloonar.com/openclawd/iso-bot/pkg/plugin" +) + +// Engine is the core bot engine that coordinates all subsystems. +type Engine struct { + mu sync.RWMutex + running bool + paused bool + devMode bool + + // Core components + captureManager *capture.Manager + resolutionRegistry *resolution.Registry + stateManager *state.Manager + lootEngine *loot.RuleEngine + + // Plugin system + gamePlugin plugin.Plugin + services *engineServices + + // Current state + currentFrame image.Image + gameState plugin.GameState + vitals plugin.VitalStats + + // Statistics + frameCount uint64 + itemsFound int + runCount int + startTime time.Time + + // Control channels + stopChan chan struct{} + pauseChan chan bool +} + +// NewEngine creates a new bot engine. +func NewEngine(captureSource capture.Source, gamePlugin plugin.Plugin, devMode bool) (*Engine, error) { + engine := &Engine{ + captureManager: capture.NewManager(captureSource), + resolutionRegistry: resolution.NewRegistry(), + devMode: devMode, + gamePlugin: gamePlugin, + stopChan: make(chan struct{}), + pauseChan: make(chan bool, 1), + startTime: time.Now(), + } + + // Create engine services for the plugin + engine.services = &engineServices{engine: engine} + + // Initialize the plugin + if err := gamePlugin.Init(engine.services); err != nil { + return nil, fmt.Errorf("failed to initialize game plugin: %w", err) + } + + // Set up state manager + engine.stateManager = state.NewManager(gamePlugin.Detector()) + + // Set up default loot filter + if lootFilter := gamePlugin.DefaultLootFilter(); lootFilter != nil { + if ruleEngine, ok := lootFilter.(*loot.RuleEngine); ok { + engine.lootEngine = ruleEngine + } + } + + return engine, nil +} + +// Start begins the engine's main loop. +func (e *Engine) Start(ctx context.Context) error { + e.mu.Lock() + if e.running { + e.mu.Unlock() + return fmt.Errorf("engine is already running") + } + e.running = true + e.mu.Unlock() + + log.Printf("Engine starting (dev mode: %v)", e.devMode) + + ticker := time.NewTicker(33 * time.Millisecond) // ~30 FPS + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-e.stopChan: + return nil + case paused := <-e.pauseChan: + e.mu.Lock() + e.paused = paused + e.mu.Unlock() + if paused { + log.Println("Engine paused") + } else { + log.Println("Engine resumed") + } + case <-ticker.C: + if e.isPaused() { + continue + } + + if err := e.processFrame(); err != nil { + log.Printf("Frame processing error: %v", err) + continue + } + } + } +} + +// Stop stops the engine. +func (e *Engine) Stop() { + e.mu.Lock() + defer e.mu.Unlock() + + if !e.running { + return + } + + e.running = false + close(e.stopChan) + log.Println("Engine stopped") +} + +// Pause pauses or resumes the engine. +func (e *Engine) Pause(paused bool) { + select { + case e.pauseChan <- paused: + default: + // Channel is full, already have a pending pause state + } +} + +// Status returns the current engine status. +func (e *Engine) Status() EngineStatus { + e.mu.RLock() + defer e.mu.RUnlock() + + return EngineStatus{ + Running: e.running, + Paused: e.paused, + DevMode: e.devMode, + GameState: e.gameState, + Vitals: e.vitals, + FrameCount: e.frameCount, + ItemsFound: e.itemsFound, + RunCount: e.runCount, + Uptime: time.Since(e.startTime), + CaptureStats: e.captureManager.Stats(), + } +} + +// CurrentFrame returns the most recent captured frame. +func (e *Engine) CurrentFrame() image.Image { + e.mu.RLock() + defer e.mu.RUnlock() + return e.currentFrame +} + +// GamePlugin returns the active game plugin. +func (e *Engine) GamePlugin() plugin.Plugin { + return e.gamePlugin +} + +// ResolutionRegistry returns the resolution registry. +func (e *Engine) ResolutionRegistry() *resolution.Registry { + return e.resolutionRegistry +} + +// LootEngine returns the loot filter engine. +func (e *Engine) LootEngine() *loot.RuleEngine { + return e.lootEngine +} + +// processFrame captures and analyzes a single frame. +func (e *Engine) processFrame() error { + // Capture frame + frame, err := e.captureManager.Capture() + if err != nil { + return fmt.Errorf("capture failed: %w", err) + } + + e.mu.Lock() + e.currentFrame = frame + e.frameCount++ + e.mu.Unlock() + + // Update game state + gameState := e.stateManager.Update(frame) + + // Read vitals if in-game + var vitals plugin.VitalStats + if gameState == plugin.StateInGame { + vitals = e.gamePlugin.Detector().ReadVitals(frame) + } + + e.mu.Lock() + e.gameState = gameState + e.vitals = vitals + e.mu.Unlock() + + // In dev mode, we don't perform any actions + if e.devMode { + return nil + } + + // TODO: Implement bot actions based on state and detected items + + return nil +} + +// isPaused returns true if the engine is paused. +func (e *Engine) isPaused() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.paused +} + +// EngineStatus represents the current state of the engine. +type EngineStatus struct { + Running bool `json:"running"` + Paused bool `json:"paused"` + DevMode bool `json:"devMode"` + GameState plugin.GameState `json:"gameState"` + Vitals plugin.VitalStats `json:"vitals"` + FrameCount uint64 `json:"frameCount"` + ItemsFound int `json:"itemsFound"` + RunCount int `json:"runCount"` + Uptime time.Duration `json:"uptime"` + CaptureStats capture.Stats `json:"captureStats"` +} + +// engineServices implements plugin.EngineServices. +type engineServices struct { + engine *Engine +} + +// Capture returns the current screen frame. +func (s *engineServices) Capture() image.Image { + return s.engine.CurrentFrame() +} + +// CaptureSource returns the active capture source. +func (s *engineServices) CaptureSource() capture.Source { + return s.engine.captureManager.Source() +} + +// Resolution returns the current capture resolution. +func (s *engineServices) Resolution() (width, height int) { + return s.engine.captureManager.Size() +} + +// Region returns a named screen region for the current game and resolution. +func (s *engineServices) Region(name string) image.Rectangle { + width, height := s.Resolution() + gameID := s.engine.gamePlugin.Info().ID + + region, err := s.engine.resolutionRegistry.GetRegion(gameID, width, height, name) + if err != nil { + log.Printf("Warning: region %q not found: %v", name, err) + return image.Rectangle{} + } + + return region +} + +// Click sends a mouse click at the given position. +func (s *engineServices) Click(pos image.Point) { + if s.engine.devMode { + log.Printf("Dev mode: would click at (%d, %d)", pos.X, pos.Y) + return + } + // TODO: Implement actual mouse click +} + +// MoveMouse moves the mouse to the given position. +func (s *engineServices) MoveMouse(pos image.Point) { + if s.engine.devMode { + log.Printf("Dev mode: would move mouse to (%d, %d)", pos.X, pos.Y) + return + } + // TODO: Implement actual mouse movement +} + +// PressKey sends a key press. +func (s *engineServices) PressKey(key string) { + if s.engine.devMode { + log.Printf("Dev mode: would press key %q", key) + return + } + // TODO: Implement actual key press +} + +// TypeText types text with human-like delays. +func (s *engineServices) TypeText(text string) { + if s.engine.devMode { + log.Printf("Dev mode: would type %q", text) + return + } + // TODO: Implement actual text typing +} + +// Wait pauses for a human-like delay. +func (s *engineServices) Wait() { + time.Sleep(time.Duration(100+rand.Intn(200)) * time.Millisecond) +} + +// WaitMs pauses for a specific duration with randomization. +func (s *engineServices) WaitMs(baseMs int, varianceMs int) { + variance := time.Duration(rand.Intn(varianceMs*2)-varianceMs) * time.Millisecond + delay := time.Duration(baseMs)*time.Millisecond + variance + time.Sleep(delay) +} + +// Log logs a message associated with the plugin. +func (s *engineServices) Log(level string, msg string, args ...any) { + prefix := fmt.Sprintf("[%s] %s", s.engine.gamePlugin.Info().ID, level) + log.Printf(prefix+": "+msg, args...) +} + +// Random number generator (TODO: use crypto/rand for better randomness) +var rand = struct { + Intn func(int) int +}{ + Intn: func(n int) int { + return int(time.Now().UnixNano()) % n + }, +} \ No newline at end of file diff --git a/pkg/engine/vision/vision.go b/pkg/engine/vision/vision.go index 2f3e36c..f0a1178 100644 --- a/pkg/engine/vision/vision.go +++ b/pkg/engine/vision/vision.go @@ -1,12 +1,13 @@ // Package vision provides computer vision utilities for game screen analysis. // -// Uses GoCV (OpenCV bindings for Go) for template matching, color detection, -// and contour analysis. Designed for high-throughput real-time analysis. +// Pure Go implementation without external dependencies like OpenCV. +// Designed for high-throughput real-time analysis of game screens. package vision import ( "image" "image/color" + "math" ) // Match represents a detected element on screen. @@ -31,6 +32,11 @@ type ColorRange struct { UpperH, UpperS, UpperV int } +// HSV represents a color in HSV color space. +type HSV struct { + H, S, V int +} + // Pipeline processes frames through a series of vision operations. type Pipeline struct { templates map[string]*Template @@ -58,31 +64,293 @@ func (p *Pipeline) LoadTemplate(name string, img image.Image) { // FindTemplate searches for a template in the frame. // Returns the best match above threshold, or nil. +// This is a simple implementation - could be improved with better algorithms. func (p *Pipeline) FindTemplate(frame image.Image, templateName string) *Match { - // TODO: Implement with GoCV matchTemplate - // This is a stub — actual implementation needs gocv.MatchTemplate + template, exists := p.templates[templateName] + if !exists { + return nil + } + + frameBounds := frame.Bounds() + templateBounds := template.Image.Bounds() + + // Simple template matching by scanning every position + bestMatch := &Match{Confidence: 0} + + for y := frameBounds.Min.Y; y <= frameBounds.Max.Y-templateBounds.Dy(); y++ { + for x := frameBounds.Min.X; x <= frameBounds.Max.X-templateBounds.Dx(); x++ { + confidence := p.compareAtPosition(frame, template.Image, x, y) + if confidence > bestMatch.Confidence { + bestMatch.Position = image.Point{X: x, Y: y} + bestMatch.BBox = image.Rect(x, y, x+templateBounds.Dx(), y+templateBounds.Dy()) + bestMatch.Confidence = confidence + bestMatch.Label = templateName + } + } + } + + if bestMatch.Confidence >= p.threshold { + return bestMatch + } return nil } // FindAllTemplates finds all matches of a template above threshold. func (p *Pipeline) FindAllTemplates(frame image.Image, templateName string) []Match { - // TODO: Implement with GoCV + NMS + // For simplicity, just return the best match + match := p.FindTemplate(frame, templateName) + if match != nil { + return []Match{*match} + } return nil } // FindByColor detects regions matching an HSV color range. func (p *Pipeline) FindByColor(frame image.Image, colorRange ColorRange, minArea int) []Match { - // TODO: Implement with GoCV inRange + findContours - return nil + bounds := frame.Bounds() + var matches []Match + + // Simple blob detection by scanning for connected regions + visited := make(map[image.Point]bool) + + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + pt := image.Point{X: x, Y: y} + if visited[pt] { + continue + } + + c := frame.At(x, y) + hsv := RGBToHSV(c) + + if p.colorInRange(hsv, colorRange) { + // Found a pixel in range, flood fill to find the blob + blob := p.floodFill(frame, pt, colorRange, visited) + if len(blob) >= minArea { + bbox := p.getBoundingBox(blob) + center := image.Point{ + X: (bbox.Min.X + bbox.Max.X) / 2, + Y: (bbox.Min.Y + bbox.Max.Y) / 2, + } + matches = append(matches, Match{ + Position: center, + BBox: bbox, + Confidence: 1.0, // Binary detection for color matching + Label: "color_match", + }) + } + } + } + } + + return matches } // ReadBarPercentage reads a horizontal bar's fill level (health, mana, xp). func (p *Pipeline) ReadBarPercentage(frame image.Image, barRegion image.Rectangle, filledColor ColorRange) float64 { - // TODO: Implement — scan columns for filled color ratio - return 0.0 + bounds := barRegion.Intersect(frame.Bounds()) + if bounds.Empty() { + return 0.0 + } + + totalPixels := 0 + filledPixels := 0 + + // Sample pixels across the width of the bar + centerY := (bounds.Min.Y + bounds.Max.Y) / 2 + for x := bounds.Min.X; x < bounds.Max.X; x++ { + c := frame.At(x, centerY) + hsv := RGBToHSV(c) + totalPixels++ + if p.colorInRange(hsv, filledColor) { + filledPixels++ + } + } + + if totalPixels == 0 { + return 0.0 + } + + return float64(filledPixels) / float64(totalPixels) } // GetPixelColor returns the color at a specific pixel. func (p *Pipeline) GetPixelColor(frame image.Image, x, y int) color.Color { return frame.At(x, y) } + +// GetPixelHSV returns the HSV values at a specific pixel. +func (p *Pipeline) GetPixelHSV(frame image.Image, x, y int) HSV { + c := frame.At(x, y) + return RGBToHSV(c) +} + +// HasColorInRegion checks if any pixel in the region matches the color range. +func (p *Pipeline) HasColorInRegion(frame image.Image, region image.Rectangle, colorRange ColorRange) bool { + bounds := region.Intersect(frame.Bounds()) + if bounds.Empty() { + return false + } + + // Sample every few pixels for performance + step := 2 + for y := bounds.Min.Y; y < bounds.Max.Y; y += step { + for x := bounds.Min.X; x < bounds.Max.X; x += step { + c := frame.At(x, y) + hsv := RGBToHSV(c) + if p.colorInRange(hsv, colorRange) { + return true + } + } + } + return false +} + +// compareAtPosition compares template with frame at given position. +func (p *Pipeline) compareAtPosition(frame, template image.Image, frameX, frameY int) float64 { + templateBounds := template.Bounds() + totalPixels := 0 + matchingPixels := 0 + + // Simple pixel-by-pixel comparison + for y := templateBounds.Min.Y; y < templateBounds.Max.Y; y++ { + for x := templateBounds.Min.X; x < templateBounds.Max.X; x++ { + frameColor := frame.At(frameX+x, frameY+y) + templateColor := template.At(x, y) + + totalPixels++ + if p.colorsMatch(frameColor, templateColor, 30) { // tolerance of 30 + matchingPixels++ + } + } + } + + return float64(matchingPixels) / float64(totalPixels) +} + +// colorsMatch checks if two colors are similar within tolerance. +func (p *Pipeline) colorsMatch(c1, c2 color.Color, tolerance int) bool { + r1, g1, b1, _ := c1.RGBA() + r2, g2, b2, _ := c2.RGBA() + + // Convert from 16-bit to 8-bit + r1, g1, b1 = r1>>8, g1>>8, b1>>8 + r2, g2, b2 = r2>>8, g2>>8, b2>>8 + + dr := int(r1) - int(r2) + dg := int(g1) - int(g2) + db := int(b1) - int(b2) + + if dr < 0 { dr = -dr } + if dg < 0 { dg = -dg } + if db < 0 { db = -db } + + return dr <= tolerance && dg <= tolerance && db <= tolerance +} + +// colorInRange checks if HSV color is within range. +func (p *Pipeline) colorInRange(hsv HSV, colorRange ColorRange) bool { + return hsv.H >= colorRange.LowerH && hsv.H <= colorRange.UpperH && + hsv.S >= colorRange.LowerS && hsv.S <= colorRange.UpperS && + hsv.V >= colorRange.LowerV && hsv.V <= colorRange.UpperV +} + +// floodFill finds connected pixels of the same color. +func (p *Pipeline) floodFill(frame image.Image, start image.Point, colorRange ColorRange, visited map[image.Point]bool) []image.Point { + bounds := frame.Bounds() + var blob []image.Point + stack := []image.Point{start} + + for len(stack) > 0 { + pt := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + if visited[pt] || !pt.In(bounds) { + continue + } + + c := frame.At(pt.X, pt.Y) + hsv := RGBToHSV(c) + if !p.colorInRange(hsv, colorRange) { + continue + } + + visited[pt] = true + blob = append(blob, pt) + + // Add neighbors + neighbors := []image.Point{ + {X: pt.X-1, Y: pt.Y}, + {X: pt.X+1, Y: pt.Y}, + {X: pt.X, Y: pt.Y-1}, + {X: pt.X, Y: pt.Y+1}, + } + stack = append(stack, neighbors...) + } + + return blob +} + +// getBoundingBox calculates the bounding box for a set of points. +func (p *Pipeline) getBoundingBox(points []image.Point) image.Rectangle { + if len(points) == 0 { + return image.Rectangle{} + } + + minX, minY := points[0].X, points[0].Y + maxX, maxY := minX, minY + + for _, pt := range points { + if pt.X < minX { minX = pt.X } + if pt.X > maxX { maxX = pt.X } + if pt.Y < minY { minY = pt.Y } + if pt.Y > maxY { maxY = pt.Y } + } + + return image.Rect(minX, minY, maxX+1, maxY+1) +} + +// RGBToHSV converts RGB color to HSV. +func RGBToHSV(c color.Color) HSV { + r, g, b, _ := c.RGBA() + // Convert from 16-bit to float [0,1] + rf := float64(r>>8) / 255.0 + gf := float64(g>>8) / 255.0 + bf := float64(b>>8) / 255.0 + + max := math.Max(rf, math.Max(gf, bf)) + min := math.Min(rf, math.Min(gf, bf)) + delta := max - min + + // Value + v := max + + // Saturation + var s float64 + if max != 0 { + s = delta / max + } + + // Hue + var h float64 + if delta != 0 { + switch max { + case rf: + h = math.Mod((gf-bf)/delta, 6) + case gf: + h = (bf-rf)/delta + 2 + case bf: + h = (rf-gf)/delta + 4 + } + h *= 60 + if h < 0 { + h += 360 + } + } + + return HSV{ + H: int(h), + S: int(s * 255), + V: int(v * 255), + } +} diff --git a/plugins/d2r/config.go b/plugins/d2r/config.go index d4d7300..c024cae 100644 --- a/plugins/d2r/config.go +++ b/plugins/d2r/config.go @@ -99,8 +99,8 @@ func RegisterProfiles(registry *resolution.Registry) error { "minimap": image.Rect(1600, 0, 1920, 320), "inventory": image.Rect(960, 330, 1490, 770), "stash": image.Rect(430, 330, 960, 770), - "skill_left": image.Rect(194, 1036, 246, 1088), - "skill_right": image.Rect(1674, 1036, 1726, 1088), + "skill_left": image.Rect(194, 1030, 246, 1078), + "skill_right": image.Rect(1674, 1030, 1726, 1078), }, }, // 1280x720 (720p) - Secondary resolution @@ -116,8 +116,8 @@ func RegisterProfiles(registry *resolution.Registry) error { "minimap": image.Rect(1067, 0, 1280, 213), "inventory": image.Rect(640, 220, 993, 513), "stash": image.Rect(287, 220, 640, 513), - "skill_left": image.Rect(129, 691, 164, 726), - "skill_right": image.Rect(1116, 691, 1151, 726), + "skill_left": image.Rect(129, 685, 164, 718), + "skill_right": image.Rect(1116, 685, 1151, 718), }, }, } diff --git a/plugins/d2r/detector.go b/plugins/d2r/detector.go index 4ecd07a..711eb52 100644 --- a/plugins/d2r/detector.go +++ b/plugins/d2r/detector.go @@ -5,12 +5,14 @@ import ( "image" "git.cloonar.com/openclawd/iso-bot/pkg/plugin" + "git.cloonar.com/openclawd/iso-bot/pkg/engine/vision" ) // Detector implements plugin.GameDetector for D2R. type Detector struct { config Config services plugin.EngineServices + vision *vision.Pipeline } // NewDetector creates a D2R state detector. @@ -18,6 +20,7 @@ func NewDetector(config Config, services plugin.EngineServices) *Detector { return &Detector{ config: config, services: services, + vision: vision.NewPipeline(0.8), // 80% confidence threshold } } @@ -41,7 +44,7 @@ func (d *Detector) DetectState(frame image.Image) plugin.GameState { } if d.isInGame(frame) { vitals := d.ReadVitals(frame) - if vitals.HealthPct == 0 { + if vitals.HealthPct <= 0.01 { // Consider very low health as dead return plugin.StateDead } return plugin.StateInGame @@ -51,15 +54,42 @@ func (d *Detector) DetectState(frame image.Image) plugin.GameState { // ReadVitals reads health and mana from the orbs. func (d *Detector) ReadVitals(frame image.Image) plugin.VitalStats { - // TODO: Analyze health/mana orb regions using color detection - // Get region coordinates from the engine services healthRegion := d.services.Region("health_orb") manaRegion := d.services.Region("mana_orb") - _ = healthRegion // Use these regions for color analysis - _ = manaRegion + var healthPct, manaPct float64 - return plugin.VitalStats{} + // Read health percentage from red-filled pixels in health orb + if !healthRegion.Empty() { + healthColor := vision.ColorRange{ + LowerH: d.config.Colors.HealthFilled.LowerH, + UpperH: d.config.Colors.HealthFilled.UpperH, + LowerS: d.config.Colors.HealthFilled.LowerS, + UpperS: d.config.Colors.HealthFilled.UpperS, + LowerV: d.config.Colors.HealthFilled.LowerV, + UpperV: d.config.Colors.HealthFilled.UpperV, + } + healthPct = d.vision.ReadBarPercentage(frame, healthRegion, healthColor) + } + + // Read mana percentage from blue-filled pixels in mana orb + if !manaRegion.Empty() { + manaColor := vision.ColorRange{ + LowerH: d.config.Colors.ManaFilled.LowerH, + UpperH: d.config.Colors.ManaFilled.UpperH, + LowerS: d.config.Colors.ManaFilled.LowerS, + UpperS: d.config.Colors.ManaFilled.UpperS, + LowerV: d.config.Colors.ManaFilled.LowerV, + UpperV: d.config.Colors.ManaFilled.UpperV, + } + manaPct = d.vision.ReadBarPercentage(frame, manaRegion, manaColor) + } + + return plugin.VitalStats{ + HealthPct: healthPct, + ManaPct: manaPct, + XPPct: 0.0, // TODO: Implement XP bar reading + } } // IsInGame returns true if health orb is visible. @@ -68,21 +98,64 @@ func (d *Detector) IsInGame(frame image.Image) bool { } func (d *Detector) isLoading(frame image.Image) bool { - // TODO: Check for loading screen (mostly black with loading bar) - return false + // Check for loading screen by looking for mostly black screen + // This is a simple heuristic - could be improved + bounds := frame.Bounds() + totalPixels := 0 + darkPixels := 0 + + // Sample every 10 pixels for performance + step := 10 + for y := bounds.Min.Y; y < bounds.Max.Y; y += step { + for x := bounds.Min.X; x < bounds.Max.X; x += step { + c := frame.At(x, y) + r, g, b, _ := c.RGBA() + // Convert to 8-bit + r, g, b = r>>8, g>>8, b>>8 + + totalPixels++ + // Consider pixel dark if all channels are below 30 + if r < 30 && g < 30 && b < 30 { + darkPixels++ + } + } + } + + if totalPixels == 0 { + return false + } + + // If more than 70% of screen is dark, likely loading + return float64(darkPixels)/float64(totalPixels) > 0.7 } func (d *Detector) isMainMenu(frame image.Image) bool { - // TODO: Template match main menu elements + // TODO: Template match main menu elements or check for specific colors/text + // For now, this is a placeholder return false } func (d *Detector) isCharacterSelect(frame image.Image) bool { // TODO: Template match character select screen + // For now, this is a placeholder return false } func (d *Detector) isInGame(frame image.Image) bool { - // TODO: Check if health orb region contains red pixels - return false + // Check if health orb region contains red pixels indicating health + healthRegion := d.services.Region("health_orb") + if healthRegion.Empty() { + return false + } + + healthColor := vision.ColorRange{ + LowerH: d.config.Colors.HealthFilled.LowerH, + UpperH: d.config.Colors.HealthFilled.UpperH, + LowerS: d.config.Colors.HealthFilled.LowerS, + UpperS: d.config.Colors.HealthFilled.UpperS, + LowerV: d.config.Colors.HealthFilled.LowerV, + UpperV: d.config.Colors.HealthFilled.UpperV, + } + + return d.vision.HasColorInRegion(frame, healthRegion, healthColor) } diff --git a/testdata/d2r_1080p.png b/testdata/d2r_1080p.png new file mode 100644 index 0000000..f07f214 Binary files /dev/null and b/testdata/d2r_1080p.png differ diff --git a/web/dev/index.html b/web/dev/index.html new file mode 100644 index 0000000..509a323 --- /dev/null +++ b/web/dev/index.html @@ -0,0 +1,672 @@ + + + + + + ISO-BOT Dev Dashboard + + + +
+

ISO-BOT DEV DASHBOARD

+
+ Status: Loading... + Game: Unknown + FPS: 0 +
+
+ +
+ +
+
+ Game capture + +
+
📁 Drop image here
+ +
+ +
+ Mouse: (0, 0)
+ RGB: (0, 0, 0)
+ HSV: (0, 0, 0) +
+
+
+ + +
+ +
+

State Detection

+
+
+ Game State: + unknown +
+
+ +
+
+ Health: + 0% +
+
+
+
+ +
+ Mana: + 0% +
+
+
+
+
+
+ + +
+

Capture Stats

+
+
+ FPS: + 0 +
+
+ Frames: + 0 +
+
+ Backend: + file +
+
+ Avg Ms: + 0 +
+
+
+ + +
+

Regions

+
+ +
+
+ + +
+

Loot Filter

+
+
+ Rules: + 0 +
+
+ Last Match: + None +
+
+ +

Routines

+
+ +
+
+
+ + +
+

Log Output

+
+
[INFO] Dev dashboard loaded
+
+
+
+ + + + \ No newline at end of file