openclawd/iso-bot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix dashboard + vision: region checkboxes, upload auto-switch, state detection, analyze endpoint

Browse source 
- Region checkboxes now control which overlays are drawn (?regions= param)
- Upload auto-switches capture source (no restart needed)
- State detection: check in-game BEFORE loading, require 90% dark for loading
- Tighten HSV ranges: health H 0-30 S 50+ V 30+, mana H 200-240 S 40+ V 20+
- Add /api/analyze endpoint with per-region color analysis
This commit is contained in:
Hoid 2026-02-14 11:27:46 +00:00
parent 67f2e5536a
commit a665253d4d
5 changed files with 277 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
debug Executable file
View file

Binary file not shown.

236
pkg/api/api.go
View file

@ -34,9 +34,11 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"git.cloonar.com/openclawd/iso-bot/pkg/engine"
"git.cloonar.com/openclawd/iso-bot/pkg/engine/capture/backends"
"git.cloonar.com/openclawd/iso-bot/pkg/engine/vision"
)
@ -73,6 +75,33 @@ type RegionInfo struct {
Height int `json:"height"`
}
// HSVColor represents an HSV color value.
type HSVColor struct {
H int `json:"h"`
S int `json:"s"`
V int `json:"v"`
}
// RegionAnalysis represents analysis results for a specific region.
type RegionAnalysis struct {
Name string `json:"name"`
DominantHSV HSVColor `json:"dominantHSV"`
MatchPercent float64 `json:"matchPercent"`
}
// AnalysisResponse represents the response from /api/analyze endpoint.
type AnalysisResponse struct {
GameState string `json:"gameState"`
Vitals VitalsInfo `json:"vitals"`
Regions []RegionAnalysis `json:"regions"`
}
// VitalsInfo represents vital statistics.
type VitalsInfo struct {
HealthPct float64 `json:"healthPct"`
ManaPct float64 `json:"manaPct"`
}
// Server provides the HTTP API and WebSocket endpoint.
type Server struct {
mu sync.RWMutex
@ -114,6 +143,7 @@ func (s *Server) registerRoutes() {
s.mux.HandleFunc("GET /api/pixel", s.handlePixel)
s.mux.HandleFunc("GET /api/config", s.handleConfig)
s.mux.HandleFunc("GET /api/plugins", s.handlePlugins)
s.mux.HandleFunc("GET /api/analyze", s.handleAnalyze)
s.mux.HandleFunc("POST /api/start", s.handleStart)
s.mux.HandleFunc("POST /api/stop", s.handleStop)
s.mux.HandleFunc("POST /api/pause", s.handlePause)
@ -184,24 +214,45 @@ func (s *Server) handleCaptureFrameAnnotated(w http.ResponseWriter, r *http.Requ
http.Error(w, "No frame available", http.StatusNoContent)
return
}
// Create a copy of the frame to draw on
bounds := frame.Bounds()
annotated := image.NewRGBA(bounds)
draw.Draw(annotated, bounds, frame, bounds.Min, draw.Src)
// Draw region overlays
// Get regions to draw (filtered by query parameter if present)
regions := s.getRegions()
// Filter regions by query parameter if provided
if r.URL.Query().Has("regions") {
regionsParam := r.URL.Query().Get("regions")
if regionsParam == "" {
regions = nil // Empty param = no overlays
} else {
regionNames := strings.Split(regionsParam, ",")
filteredRegions := make([]RegionInfo, 0)
for _, region := range regions {
for _, name := range regionNames {
if strings.TrimSpace(name) == region.Name {
filteredRegions = append(filteredRegions, region)
break
}
}
}
regions = filteredRegions
}
}
s.drawRegionOverlays(annotated, regions)
w.Header().Set("Content-Type", "image/jpeg")
var buf bytes.Buffer
if err := jpeg.Encode(&buf, annotated, &jpeg.Options{Quality: 85}); err != nil {
http.Error(w, "Failed to encode annotated frame", http.StatusInternalServerError)
return
}
w.Write(buf.Bytes())
}
@ -251,13 +302,43 @@ func (s *Server) handleCaptureUpload(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Failed to save file", http.StatusInternalServerError)
return
}
// TODO: Auto-switch capture source to uploaded file
// For now, just inform user they need to restart with the new file
// Auto-switch capture source to uploaded file
fileConfig := map[string]interface{}{
"path": filename,
"type": "image",
}
newSource, err := backends.NewFileSource(fileConfig)
if err != nil {
log.Printf("Failed to create file source: %v", err)
response := map[string]string{
"filename": filename,
"message": fmt.Sprintf("File uploaded to %s but failed to switch capture source: %v", filename, err),
"autoSwitch": "failed",
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
return
}
err = s.engine.SetCaptureSource(newSource)
if err != nil {
log.Printf("Failed to set capture source: %v", err)
response := map[string]string{
"filename": filename,
"message": fmt.Sprintf("File uploaded to %s but failed to switch capture source: %v", filename, err),
"autoSwitch": "failed",
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
return
}
response := map[string]string{
"filename": filename,
"message": fmt.Sprintf("File uploaded to %s. Restart with --capture-file to use it.", filename),
"autoSwitch": "false", // Feature not implemented yet
"filename": filename,
"message": fmt.Sprintf("File uploaded to %s and capture source switched successfully", filename),
"autoSwitch": "true",
}
w.Header().Set("Content-Type", "application/json")
@ -449,23 +530,57 @@ func (s *Server) handlePause(w http.ResponseWriter, r *http.Request) {
var req struct {
Paused bool `json:"paused"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "Invalid JSON", http.StatusBadRequest)
return
}
s.engine.Pause(req.Paused)
action := "resumed"
if req.Paused {
action = "paused"
}
response := map[string]string{
"message": fmt.Sprintf("Engine %s", action),
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *Server) handleAnalyze(w http.ResponseWriter, r *http.Request) {
frame := s.engine.CurrentFrame()
if frame == nil {
http.Error(w, "No frame available", http.StatusNoContent)
return
}
status := s.engine.Status()
// Get vitals
vitals := VitalsInfo{
HealthPct: status.Vitals.HealthPct,
ManaPct: status.Vitals.ManaPct,
}
// Get regions and analyze each one
regions := s.getRegions()
regionAnalyses := make([]RegionAnalysis, 0, len(regions))
for _, region := range regions {
analysis := s.analyzeRegion(frame, region)
regionAnalyses = append(regionAnalyses, analysis)
}
response := AnalysisResponse{
GameState: string(status.GameState),
Vitals: vitals,
Regions: regionAnalyses,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
@ -498,6 +613,93 @@ func (s *Server) getRegions() []RegionInfo {
return regions
}
func (s *Server) analyzeRegion(frame image.Image, region RegionInfo) RegionAnalysis {
rect := image.Rect(region.X, region.Y, region.X+region.Width, region.Y+region.Height)
bounds := rect.Intersect(frame.Bounds())
if bounds.Empty() {
return RegionAnalysis{
Name: region.Name,
DominantHSV: HSVColor{H: 0, S: 0, V: 0},
MatchPercent: 0.0,
}
}
// Calculate dominant HSV by averaging all pixels in region
var totalH, totalS, totalV int64
var pixelCount int64
// Also count matches for the region's expected color range
var matchingPixels int64
expectedColorRange := s.getExpectedColorRange(region.Name)
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
c := frame.At(x, y)
hsv := vision.RGBToHSV(c)
totalH += int64(hsv.H)
totalS += int64(hsv.S)
totalV += int64(hsv.V)
pixelCount++
// Check if this pixel matches the expected color range
if expectedColorRange != nil && s.colorInRange(hsv, *expectedColorRange) {
matchingPixels++
}
}
}
dominantHSV := HSVColor{H: 0, S: 0, V: 0}
matchPercent := 0.0
if pixelCount > 0 {
dominantHSV = HSVColor{
H: int(totalH / pixelCount),
S: int(totalS / pixelCount),
V: int(totalV / pixelCount),
}
if expectedColorRange != nil {
matchPercent = float64(matchingPixels) / float64(pixelCount) * 100.0
}
}
return RegionAnalysis{
Name: region.Name,
DominantHSV: dominantHSV,
MatchPercent: matchPercent,
}
}
func (s *Server) getExpectedColorRange(regionName string) *vision.ColorRange {
// Get D2R config to determine expected color ranges for each region type
// For now, we'll handle the main regions: health_orb and mana_orb
switch regionName {
case "health_orb":
return &vision.ColorRange{
LowerH: 0, UpperH: 30,
LowerS: 50, UpperS: 255,
LowerV: 30, UpperV: 255,
}
case "mana_orb":
return &vision.ColorRange{
LowerH: 200, UpperH: 240,
LowerS: 40, UpperS: 255,
LowerV: 20, UpperV: 255,
}
default:
// For other regions, we don't have specific expected color ranges
return nil
}
}
func (s *Server) colorInRange(hsv vision.HSV, colorRange vision.ColorRange) bool {
return hsv.H >= colorRange.LowerH && hsv.H <= colorRange.UpperH &&
hsv.S >= colorRange.LowerS && hsv.S <= colorRange.UpperS &&
hsv.V >= colorRange.LowerV && hsv.V <= colorRange.UpperV
}
func (s *Server) drawRegionOverlays(img *image.RGBA, regions []RegionInfo) {
// Simple colored rectangles for region overlays
colors := []image.Uniform{

10
plugins/d2r/config.go
View file

@ -56,11 +56,11 @@ type Config struct {
func DefaultConfig() Config {
return Config{
Colors: Colors{
// Updated ranges based on real D2R screenshot analysis
// Health orb - includes the actual colors found (olive/brown when low, reds when high)
HealthFilled: HSVRange{0, 30, 10, 100, 255, 255}, // Wide range: reds through yellows/browns
// Mana orb - includes the actual colors found (dark/brown when low, blues when high)
ManaFilled: HSVRange{40, 20, 10, 250, 255, 255}, // Browns through blues
// Tightened HSV ranges based on real D2R screenshot calibration
// Health orb is RED: H 0-30 (covering red wrapping), S 50-255, V 30-255
HealthFilled: HSVRange{0, 50, 30, 30, 255, 255},
// Mana orb is BLUE: H 200-240, S 40-255, V 20-255
ManaFilled: HSVRange{200, 40, 20, 240, 255, 255},
ItemUnique: HSVRange{15, 100, 180, 30, 255, 255},
ItemSet: HSVRange{35, 100, 150, 55, 255, 255},
ItemRare: HSVRange{15, 50, 200, 25, 150, 255},

60
plugins/d2r/detector.go
View file

@ -27,12 +27,19 @@ func NewDetector(config Config, services plugin.EngineServices) *Detector {
// DetectState analyzes a screenshot and returns the current game state.
func (d *Detector) DetectState(frame image.Image) plugin.GameState {
// Priority-based detection:
// 1. Check for loading screen
// 2. Check for main menu
// 3. Check for character select
// 4. Check for in-game (health orb visible)
// 1. Check for in-game first (health orb visible)
// 2. Check for loading screen (but only if not in-game)
// 3. Check for main menu
// 4. Check for character select
// 5. Check for death screen
if d.isInGame(frame) {
vitals := d.ReadVitals(frame)
if vitals.HealthPct <= 0.01 { // Consider very low health as dead
return plugin.StateDead
}
return plugin.StateInGame
}
if d.isLoading(frame) {
return plugin.StateLoading
}
@ -42,13 +49,6 @@ func (d *Detector) DetectState(frame image.Image) plugin.GameState {
if d.isCharacterSelect(frame) {
return plugin.StateCharacterSelect
}
if d.isInGame(frame) {
vitals := d.ReadVitals(frame)
if vitals.HealthPct <= 0.01 { // Consider very low health as dead
return plugin.StateDead
}
return plugin.StateInGame
}
return plugin.StateUnknown
}
@ -98,12 +98,12 @@ func (d *Detector) IsInGame(frame image.Image) bool {
}
func (d *Detector) isLoading(frame image.Image) bool {
// Check for loading screen by looking for mostly black screen
// This is a simple heuristic - could be improved
// Check for loading screen by looking for mostly black screen (>90%)
// AND ensuring no health orb colors are present in the health orb region
bounds := frame.Bounds()
totalPixels := 0
darkPixels := 0
// Sample every 10 pixels for performance
step := 10
for y := bounds.Min.Y; y < bounds.Max.Y; y += step {
@ -112,7 +112,7 @@ func (d *Detector) isLoading(frame image.Image) bool {
r, g, b, _ := c.RGBA()
// Convert to 8-bit
r, g, b = r>>8, g>>8, b>>8
totalPixels++
// Consider pixel dark if all channels are below 30
if r < 30 && g < 30 && b < 30 {
@ -120,13 +120,35 @@ func (d *Detector) isLoading(frame image.Image) bool {
}
}
}
if totalPixels == 0 {
return false
}
// If more than 70% of screen is dark, likely loading
return float64(darkPixels)/float64(totalPixels) > 0.7
// Require more than 90% dark screen
darkPercentage := float64(darkPixels) / float64(totalPixels)
if darkPercentage <= 0.9 {
return false
}
// Additional check: ensure no health colors in health orb region
healthRegion := d.services.Region("health_orb")
if !healthRegion.Empty() {
healthColor := vision.ColorRange{
LowerH: d.config.Colors.HealthFilled.LowerH,
UpperH: d.config.Colors.HealthFilled.UpperH,
LowerS: d.config.Colors.HealthFilled.LowerS,
UpperS: d.config.Colors.HealthFilled.UpperS,
LowerV: d.config.Colors.HealthFilled.LowerV,
UpperV: d.config.Colors.HealthFilled.UpperV,
}
// If health colors are detected in health orb region, it's not loading
if d.vision.HasColorInRegion(frame, healthRegion, healthColor) {
return false
}
}
return true
}
func (d *Detector) isMainMenu(frame image.Image) bool {

18
web/dev/index.html
View file

@ -539,17 +539,23 @@
// Update capture image with overlays
async function updateCaptureImage() {
try {
const response = await fetch('/api/capture/frame/annotated');
let url = '/api/capture/frame/annotated';
// Always pass regions param to control overlays (empty = no overlays)
const regionNames = Array.from(visibleRegions).join(',');
url += `?regions=${encodeURIComponent(regionNames)}`;
const response = await fetch(url);
const blob = await response.blob();
const url = URL.createObjectURL(blob);
const imageUrl = URL.createObjectURL(blob);
// Clean up previous URL
if (captureImage.src && captureImage.src.startsWith('blob:')) {
URL.revokeObjectURL(captureImage.src);
}
captureImage.src = url;
captureImage.src = imageUrl;
} catch (error) {
addLogLine('ERROR', `Failed to update capture image: ${error.message}`);
}