Fix dashboard + vision: region checkboxes, upload auto-switch, state detection, analyze endpoint
- Region checkboxes now control which overlays are drawn (?regions= param) - Upload auto-switches capture source (no restart needed) - State detection: check in-game BEFORE loading, require 90% dark for loading - Tighten HSV ranges: health H 0-30 S 50+ V 30+, mana H 200-240 S 40+ V 20+ - Add /api/analyze endpoint with per-region color analysis
This commit is contained in:
Hoid
parent
67f2e5536a
commit
a665253d4d
5 changed files with 277 additions and 47 deletions
236
pkg/api/api.go
236
pkg/api/api.go
|
|
@ -34,9 +34,11 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.cloonar.com/openclawd/iso-bot/pkg/engine"
|
||||
"git.cloonar.com/openclawd/iso-bot/pkg/engine/capture/backends"
|
||||
"git.cloonar.com/openclawd/iso-bot/pkg/engine/vision"
|
||||
)
|
||||
|
||||
|
|
@ -73,6 +75,33 @@ type RegionInfo struct {
|
|||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
// HSVColor represents an HSV color value.
|
||||
type HSVColor struct {
|
||||
H int `json:"h"`
|
||||
S int `json:"s"`
|
||||
V int `json:"v"`
|
||||
}
|
||||
|
||||
// RegionAnalysis represents analysis results for a specific region.
|
||||
type RegionAnalysis struct {
|
||||
Name string `json:"name"`
|
||||
DominantHSV HSVColor `json:"dominantHSV"`
|
||||
MatchPercent float64 `json:"matchPercent"`
|
||||
}
|
||||
|
||||
// AnalysisResponse represents the response from /api/analyze endpoint.
|
||||
type AnalysisResponse struct {
|
||||
GameState string `json:"gameState"`
|
||||
Vitals VitalsInfo `json:"vitals"`
|
||||
Regions []RegionAnalysis `json:"regions"`
|
||||
}
|
||||
|
||||
// VitalsInfo represents vital statistics.
|
||||
type VitalsInfo struct {
|
||||
HealthPct float64 `json:"healthPct"`
|
||||
ManaPct float64 `json:"manaPct"`
|
||||
}
|
||||
|
||||
// Server provides the HTTP API and WebSocket endpoint.
|
||||
type Server struct {
|
||||
mu sync.RWMutex
|
||||
|
|
@ -114,6 +143,7 @@ func (s *Server) registerRoutes() {
|
|||
s.mux.HandleFunc("GET /api/pixel", s.handlePixel)
|
||||
s.mux.HandleFunc("GET /api/config", s.handleConfig)
|
||||
s.mux.HandleFunc("GET /api/plugins", s.handlePlugins)
|
||||
s.mux.HandleFunc("GET /api/analyze", s.handleAnalyze)
|
||||
s.mux.HandleFunc("POST /api/start", s.handleStart)
|
||||
s.mux.HandleFunc("POST /api/stop", s.handleStop)
|
||||
s.mux.HandleFunc("POST /api/pause", s.handlePause)
|
||||
|
|
@ -184,24 +214,45 @@ func (s *Server) handleCaptureFrameAnnotated(w http.ResponseWriter, r *http.Requ
|
|||
http.Error(w, "No frame available", http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
// Create a copy of the frame to draw on
|
||||
bounds := frame.Bounds()
|
||||
annotated := image.NewRGBA(bounds)
|
||||
draw.Draw(annotated, bounds, frame, bounds.Min, draw.Src)
|
||||
|
||||
// Draw region overlays
|
||||
|
||||
// Get regions to draw (filtered by query parameter if present)
|
||||
regions := s.getRegions()
|
||||
|
||||
// Filter regions by query parameter if provided
|
||||
if r.URL.Query().Has("regions") {
|
||||
regionsParam := r.URL.Query().Get("regions")
|
||||
if regionsParam == "" {
|
||||
regions = nil // Empty param = no overlays
|
||||
} else {
|
||||
regionNames := strings.Split(regionsParam, ",")
|
||||
filteredRegions := make([]RegionInfo, 0)
|
||||
for _, region := range regions {
|
||||
for _, name := range regionNames {
|
||||
if strings.TrimSpace(name) == region.Name {
|
||||
filteredRegions = append(filteredRegions, region)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
regions = filteredRegions
|
||||
}
|
||||
}
|
||||
|
||||
s.drawRegionOverlays(annotated, regions)
|
||||
|
||||
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, annotated, &jpeg.Options{Quality: 85}); err != nil {
|
||||
http.Error(w, "Failed to encode annotated frame", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
w.Write(buf.Bytes())
|
||||
}
|
||||
|
||||
|
|
@ -251,13 +302,43 @@ func (s *Server) handleCaptureUpload(w http.ResponseWriter, r *http.Request) {
|
|||
http.Error(w, "Failed to save file", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: Auto-switch capture source to uploaded file
|
||||
// For now, just inform user they need to restart with the new file
|
||||
|
||||
// Auto-switch capture source to uploaded file
|
||||
fileConfig := map[string]interface{}{
|
||||
"path": filename,
|
||||
"type": "image",
|
||||
}
|
||||
|
||||
newSource, err := backends.NewFileSource(fileConfig)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create file source: %v", err)
|
||||
response := map[string]string{
|
||||
"filename": filename,
|
||||
"message": fmt.Sprintf("File uploaded to %s but failed to switch capture source: %v", filename, err),
|
||||
"autoSwitch": "failed",
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(response)
|
||||
return
|
||||
}
|
||||
|
||||
err = s.engine.SetCaptureSource(newSource)
|
||||
if err != nil {
|
||||
log.Printf("Failed to set capture source: %v", err)
|
||||
response := map[string]string{
|
||||
"filename": filename,
|
||||
"message": fmt.Sprintf("File uploaded to %s but failed to switch capture source: %v", filename, err),
|
||||
"autoSwitch": "failed",
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(response)
|
||||
return
|
||||
}
|
||||
|
||||
response := map[string]string{
|
||||
"filename": filename,
|
||||
"message": fmt.Sprintf("File uploaded to %s. Restart with --capture-file to use it.", filename),
|
||||
"autoSwitch": "false", // Feature not implemented yet
|
||||
"filename": filename,
|
||||
"message": fmt.Sprintf("File uploaded to %s and capture source switched successfully", filename),
|
||||
"autoSwitch": "true",
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
|
@ -449,23 +530,57 @@ func (s *Server) handlePause(w http.ResponseWriter, r *http.Request) {
|
|||
var req struct {
|
||||
Paused bool `json:"paused"`
|
||||
}
|
||||
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "Invalid JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
s.engine.Pause(req.Paused)
|
||||
|
||||
|
||||
action := "resumed"
|
||||
if req.Paused {
|
||||
action = "paused"
|
||||
}
|
||||
|
||||
|
||||
response := map[string]string{
|
||||
"message": fmt.Sprintf("Engine %s", action),
|
||||
}
|
||||
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}
|
||||
|
||||
func (s *Server) handleAnalyze(w http.ResponseWriter, r *http.Request) {
|
||||
frame := s.engine.CurrentFrame()
|
||||
if frame == nil {
|
||||
http.Error(w, "No frame available", http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
status := s.engine.Status()
|
||||
|
||||
// Get vitals
|
||||
vitals := VitalsInfo{
|
||||
HealthPct: status.Vitals.HealthPct,
|
||||
ManaPct: status.Vitals.ManaPct,
|
||||
}
|
||||
|
||||
// Get regions and analyze each one
|
||||
regions := s.getRegions()
|
||||
regionAnalyses := make([]RegionAnalysis, 0, len(regions))
|
||||
|
||||
for _, region := range regions {
|
||||
analysis := s.analyzeRegion(frame, region)
|
||||
regionAnalyses = append(regionAnalyses, analysis)
|
||||
}
|
||||
|
||||
response := AnalysisResponse{
|
||||
GameState: string(status.GameState),
|
||||
Vitals: vitals,
|
||||
Regions: regionAnalyses,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}
|
||||
|
|
@ -498,6 +613,93 @@ func (s *Server) getRegions() []RegionInfo {
|
|||
return regions
|
||||
}
|
||||
|
||||
func (s *Server) analyzeRegion(frame image.Image, region RegionInfo) RegionAnalysis {
|
||||
rect := image.Rect(region.X, region.Y, region.X+region.Width, region.Y+region.Height)
|
||||
bounds := rect.Intersect(frame.Bounds())
|
||||
|
||||
if bounds.Empty() {
|
||||
return RegionAnalysis{
|
||||
Name: region.Name,
|
||||
DominantHSV: HSVColor{H: 0, S: 0, V: 0},
|
||||
MatchPercent: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate dominant HSV by averaging all pixels in region
|
||||
var totalH, totalS, totalV int64
|
||||
var pixelCount int64
|
||||
|
||||
// Also count matches for the region's expected color range
|
||||
var matchingPixels int64
|
||||
expectedColorRange := s.getExpectedColorRange(region.Name)
|
||||
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
c := frame.At(x, y)
|
||||
hsv := vision.RGBToHSV(c)
|
||||
|
||||
totalH += int64(hsv.H)
|
||||
totalS += int64(hsv.S)
|
||||
totalV += int64(hsv.V)
|
||||
pixelCount++
|
||||
|
||||
// Check if this pixel matches the expected color range
|
||||
if expectedColorRange != nil && s.colorInRange(hsv, *expectedColorRange) {
|
||||
matchingPixels++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dominantHSV := HSVColor{H: 0, S: 0, V: 0}
|
||||
matchPercent := 0.0
|
||||
|
||||
if pixelCount > 0 {
|
||||
dominantHSV = HSVColor{
|
||||
H: int(totalH / pixelCount),
|
||||
S: int(totalS / pixelCount),
|
||||
V: int(totalV / pixelCount),
|
||||
}
|
||||
|
||||
if expectedColorRange != nil {
|
||||
matchPercent = float64(matchingPixels) / float64(pixelCount) * 100.0
|
||||
}
|
||||
}
|
||||
|
||||
return RegionAnalysis{
|
||||
Name: region.Name,
|
||||
DominantHSV: dominantHSV,
|
||||
MatchPercent: matchPercent,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) getExpectedColorRange(regionName string) *vision.ColorRange {
|
||||
// Get D2R config to determine expected color ranges for each region type
|
||||
// For now, we'll handle the main regions: health_orb and mana_orb
|
||||
switch regionName {
|
||||
case "health_orb":
|
||||
return &vision.ColorRange{
|
||||
LowerH: 0, UpperH: 30,
|
||||
LowerS: 50, UpperS: 255,
|
||||
LowerV: 30, UpperV: 255,
|
||||
}
|
||||
case "mana_orb":
|
||||
return &vision.ColorRange{
|
||||
LowerH: 200, UpperH: 240,
|
||||
LowerS: 40, UpperS: 255,
|
||||
LowerV: 20, UpperV: 255,
|
||||
}
|
||||
default:
|
||||
// For other regions, we don't have specific expected color ranges
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) colorInRange(hsv vision.HSV, colorRange vision.ColorRange) bool {
|
||||
return hsv.H >= colorRange.LowerH && hsv.H <= colorRange.UpperH &&
|
||||
hsv.S >= colorRange.LowerS && hsv.S <= colorRange.UpperS &&
|
||||
hsv.V >= colorRange.LowerV && hsv.V <= colorRange.UpperV
|
||||
}
|
||||
|
||||
func (s *Server) drawRegionOverlays(img *image.RGBA, regions []RegionInfo) {
|
||||
// Simple colored rectangles for region overlays
|
||||
colors := []image.Uniform{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue