work on first boss
BIN
assets/black-cathedral-well.png
Normal file
|
After Width: | Height: | Size: 16 KiB |
BIN
assets/new.png
Normal file
|
After Width: | Height: | Size: 7.4 KiB |
BIN
assets/unique-boss-present.png
Normal file
|
After Width: | Height: | Size: 17 KiB |
BIN
debug_loot_capture.png
Normal file
|
After Width: | Height: | Size: 8.3 MiB |
BIN
debug_loot_detected.png
Normal file
|
After Width: | Height: | Size: 6.8 MiB |
BIN
debug_loot_dilated.png
Normal file
|
After Width: | Height: | Size: 297 KiB |
BIN
debug_loot_edges.png
Normal file
|
After Width: | Height: | Size: 639 KiB |
|
|
@ -15,7 +15,10 @@ public class BossRunExecutor : GameExecutor
|
|||
private static readonly string BlackCathedralTemplate = Path.Combine("assets", "black-cathedral.png");
|
||||
private static readonly string InvitationTemplate = Path.Combine("assets", "invitation.png");
|
||||
private static readonly string CathedralDoorTemplate = Path.Combine("assets", "black-cathedral-door.png");
|
||||
private static readonly string CathedralWellTemplate = Path.Combine("assets", "black-cathedral-well.png");
|
||||
private static readonly string ReturnTheRingTemplate = Path.Combine("assets", "return-the-ring.png");
|
||||
private static readonly string BossHealthbarTemplate = Path.Combine("assets", "unique-boss-present.png");
|
||||
private static readonly string NewInstanceTemplate = Path.Combine("assets", "new.png");
|
||||
|
||||
private BossRunState _state = BossRunState.Idle;
|
||||
private readonly IClientLogWatcher _logWatcher;
|
||||
|
|
@ -53,7 +56,6 @@ public class BossRunExecutor : GameExecutor
|
|||
public async Task RunBossLoop()
|
||||
{
|
||||
_stopped = false;
|
||||
_bossDetector.SetBoss("kulemak");
|
||||
Log.Information("Starting boss run loop ({Count} invitations)", _config.Kulemak.InvitationCount);
|
||||
|
||||
if (!await Prepare())
|
||||
|
|
@ -193,6 +195,7 @@ public class BossRunExecutor : GameExecutor
|
|||
|
||||
// Close stash
|
||||
await _game.PressEscape();
|
||||
_inventory.ResetStashTabState();
|
||||
await Helpers.Sleep(Delays.PostEscape);
|
||||
|
||||
Log.Information("Preparation complete");
|
||||
|
|
@ -214,7 +217,7 @@ public class BossRunExecutor : GameExecutor
|
|||
Log.Error("Could not find Waypoint nameplate");
|
||||
return false;
|
||||
}
|
||||
await Helpers.Sleep(1000);
|
||||
await Helpers.Sleep(500);
|
||||
|
||||
// Template match well-of-souls.png and click
|
||||
var match = await _screen.TemplateMatch(WellOfSoulsTemplate);
|
||||
|
|
@ -256,32 +259,24 @@ public class BossRunExecutor : GameExecutor
|
|||
|
||||
// Hover first so the game registers the target, then use invitation
|
||||
await _game.MoveMouseTo(x, y);
|
||||
await Helpers.Sleep(500);
|
||||
await Helpers.Sleep(200);
|
||||
await _game.CtrlLeftClickAt(x, y);
|
||||
await Helpers.Sleep(1000);
|
||||
await Helpers.Sleep(500);
|
||||
|
||||
// Find "NEW" text — pick the leftmost instance
|
||||
var ocr = await _screen.Ocr();
|
||||
var newWords = ocr.Lines
|
||||
.SelectMany(l => l.Words)
|
||||
.Where(w => w.Text.Equals("NEW", StringComparison.OrdinalIgnoreCase)
|
||||
|| w.Text.Equals("New", StringComparison.Ordinal))
|
||||
.OrderBy(w => w.X)
|
||||
.ToList();
|
||||
|
||||
if (newWords.Count == 0)
|
||||
// Find "NEW" button via template match — pick the leftmost
|
||||
var matches = await _screen.TemplateMatchAll(NewInstanceTemplate);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
Log.Error("Could not find 'NEW' text for instance selection");
|
||||
Log.Error("Could not find 'NEW' template for instance selection");
|
||||
return false;
|
||||
}
|
||||
|
||||
var target = newWords[0];
|
||||
var clickX = target.X + target.Width / 2;
|
||||
var clickY = target.Y + target.Height / 2;
|
||||
Log.Information("Found {Count} 'NEW' matches, clicking leftmost at ({X},{Y})", newWords.Count, clickX, clickY);
|
||||
await _game.MoveMouseTo(clickX, clickY);
|
||||
await Helpers.Sleep(500);
|
||||
await _game.LeftClickAt(clickX, clickY);
|
||||
var target = matches.OrderBy(m => m.X).First();
|
||||
Log.Information("Found {Count} 'NEW' matches, clicking leftmost at ({X},{Y}) conf={Conf:F3}",
|
||||
matches.Count, target.X, target.Y, target.Confidence);
|
||||
await _game.MoveMouseTo(target.X, target.Y);
|
||||
await Helpers.Sleep(150);
|
||||
await _game.LeftClickAt(target.X, target.Y);
|
||||
|
||||
// Wait for area transition into boss arena
|
||||
var arrived = await _inventory.WaitForAreaTransition(_config.TravelTimeoutMs);
|
||||
|
|
@ -296,13 +291,26 @@ public class BossRunExecutor : GameExecutor
|
|||
return true;
|
||||
}
|
||||
|
||||
private void StartBossDetection()
|
||||
{
|
||||
_bossDetector.SetBoss("kulemak");
|
||||
_bossDetector.Enabled = true;
|
||||
Log.Information("Boss detection started");
|
||||
}
|
||||
|
||||
private void StopBossDetection()
|
||||
{
|
||||
_bossDetector.Enabled = false;
|
||||
Log.Information("Boss detection stopped");
|
||||
}
|
||||
|
||||
private async Task Fight()
|
||||
{
|
||||
SetState(BossRunState.Fighting);
|
||||
Log.Information("Fight phase starting");
|
||||
|
||||
// Wait for arena to settle
|
||||
await Helpers.Sleep(6000);
|
||||
await Helpers.Sleep(5500);
|
||||
if (_stopped) return;
|
||||
|
||||
// Find and click the cathedral door
|
||||
|
|
@ -317,12 +325,14 @@ public class BossRunExecutor : GameExecutor
|
|||
await _game.LeftClickAt(door.X, door.Y);
|
||||
|
||||
// Wait for cathedral interior to load
|
||||
await Helpers.Sleep(12000);
|
||||
await Helpers.Sleep(14000);
|
||||
if (_stopped) return;
|
||||
|
||||
StartBossDetection();
|
||||
|
||||
// Walk to fight area (world coords)
|
||||
const double fightWorldX = -454;
|
||||
const double fightWorldY = -332;
|
||||
var fightWorldX = -454.0;
|
||||
var fightWorldY = -332.0;
|
||||
const double wellWorldX = -496;
|
||||
const double wellWorldY = -378;
|
||||
|
||||
|
|
@ -335,14 +345,21 @@ public class BossRunExecutor : GameExecutor
|
|||
if (_stopped) return;
|
||||
Log.Information("=== Boss phase {Phase}/4 ===", phase);
|
||||
|
||||
await AttackBossUntilGone();
|
||||
var lastBossPos = await AttackBossUntilGone();
|
||||
if (_stopped) return;
|
||||
|
||||
// Walk to well and click it
|
||||
// Update fight area to where the boss was last seen
|
||||
if (lastBossPos != null)
|
||||
{
|
||||
fightWorldX = lastBossPos.Value.X;
|
||||
fightWorldY = lastBossPos.Value.Y;
|
||||
Log.Information("Fight area updated to ({X:F0},{Y:F0})", fightWorldX, fightWorldY);
|
||||
}
|
||||
|
||||
// Walk to well and click the closest match to screen center
|
||||
Log.Information("Phase {Phase} done, walking to well", phase);
|
||||
await WalkToWorldPosition(wellWorldX, wellWorldY);
|
||||
// Click at screen center (well should be near character)
|
||||
await _game.LeftClickAt(1280, 720);
|
||||
await ClickClosestTemplateToCenter(CathedralWellTemplate);
|
||||
await Helpers.Sleep(2000);
|
||||
|
||||
// Walk back to fight position for next phase
|
||||
|
|
@ -396,70 +413,208 @@ public class BossRunExecutor : GameExecutor
|
|||
await Helpers.Sleep(500);
|
||||
await AttackAtPosition(1280, 720, 7000);
|
||||
|
||||
StopBossDetection();
|
||||
Log.Information("Fight complete");
|
||||
}
|
||||
|
||||
private async Task AttackBossUntilGone(int timeoutMs = 120_000)
|
||||
/// <summary>
|
||||
/// Check top-of-screen region for the unique boss healthbar frame.
|
||||
/// Uses lower threshold (0.5) to tolerate partial overlay from YOLO bounding boxes.
|
||||
/// </summary>
|
||||
private async Task<bool> IsBossAlive()
|
||||
{
|
||||
// Move mouse to screen center initially
|
||||
await _game.MoveMouseFast(1280, 720);
|
||||
var topRegion = new Region(800, 0, 960, 120);
|
||||
var matches = await _screen.TemplateMatchAll(BossHealthbarTemplate, topRegion, threshold: 0.5);
|
||||
return matches.Count > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wait for the boss healthbar to appear (boss spawns/becomes active).
|
||||
/// </summary>
|
||||
private async Task<bool> WaitForBossSpawn(int timeoutMs = 30_000)
|
||||
{
|
||||
Log.Information("Waiting for boss healthbar to appear...");
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
const int screenCx = 1280;
|
||||
const int screenCy = 720;
|
||||
|
||||
while (sw.ElapsedMilliseconds < timeoutMs)
|
||||
{
|
||||
if (_stopped) return false;
|
||||
|
||||
if (await IsBossAlive())
|
||||
{
|
||||
Log.Information("Boss healthbar detected after {Ms}ms", sw.ElapsedMilliseconds);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Attack at center while waiting — boss often spawns on top of us
|
||||
var targetX = screenCx + Rng.Next(-30, 31);
|
||||
var targetY = screenCy + Rng.Next(-30, 31);
|
||||
await _game.MoveMouseFast(targetX, targetY);
|
||||
|
||||
_game.LeftMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(20, 35));
|
||||
_game.LeftMouseUp();
|
||||
_game.RightMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(20, 35));
|
||||
_game.RightMouseUp();
|
||||
|
||||
await Helpers.Sleep(Rng.Next(50, 80));
|
||||
}
|
||||
|
||||
Log.Warning("WaitForBossSpawn timed out after {Ms}ms", timeoutMs);
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wait for boss to spawn, then attack until healthbar disappears.
|
||||
/// Returns the last world position where YOLO spotted the boss, or null.
|
||||
/// </summary>
|
||||
private async Task<(double X, double Y)?> AttackBossUntilGone(int timeoutMs = 120_000)
|
||||
{
|
||||
// Wait for boss to actually appear before attacking
|
||||
if (!await WaitForBossSpawn())
|
||||
return null;
|
||||
|
||||
const int screenCx = 1280;
|
||||
const int screenCy = 720;
|
||||
|
||||
// Attack target — defaults to screen center, updated by YOLO
|
||||
var atkX = screenCx;
|
||||
var atkY = screenCy;
|
||||
(double X, double Y)? lastBossWorldPos = null;
|
||||
|
||||
await _game.MoveMouseFast(atkX, atkY);
|
||||
await Helpers.Sleep(200);
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var consecutiveMisses = 0;
|
||||
var lastCheckMs = 0L;
|
||||
var holding = false;
|
||||
var manaStableCount = 0;
|
||||
|
||||
while (sw.ElapsedMilliseconds < timeoutMs)
|
||||
Log.Information("Boss is alive, engaging");
|
||||
|
||||
try
|
||||
{
|
||||
if (_stopped) return;
|
||||
|
||||
var snapshot = _bossDetector.Latest;
|
||||
if (snapshot.Bosses.Count > 0)
|
||||
while (sw.ElapsedMilliseconds < timeoutMs)
|
||||
{
|
||||
consecutiveMisses = 0;
|
||||
var boss = snapshot.Bosses[0];
|
||||
if (_stopped) return lastBossWorldPos;
|
||||
|
||||
// Check mana before attacking
|
||||
var hud = _hudReader.Current;
|
||||
if (hud.ManaPct < 0.80f)
|
||||
// Update attack target from YOLO when available
|
||||
var snapshot = _bossDetector.Latest;
|
||||
if (snapshot.Bosses.Count > 0)
|
||||
{
|
||||
await Helpers.Sleep(200);
|
||||
continue;
|
||||
var boss = snapshot.Bosses[0];
|
||||
atkX = boss.Cx;
|
||||
atkY = boss.Cy;
|
||||
|
||||
// Convert boss screen offset to world position
|
||||
// Scale: sqrt(72²+65²)/835 ≈ 0.116 minimap px per screen px
|
||||
const double screenToWorld = 97.0 / 835.0;
|
||||
var wp = _nav.WorldPosition;
|
||||
var bossWorldX = wp.X + (boss.Cx - screenCx) * screenToWorld;
|
||||
var bossWorldY = wp.Y + (boss.Cy - screenCy) * screenToWorld;
|
||||
lastBossWorldPos = (bossWorldX, bossWorldY);
|
||||
}
|
||||
|
||||
// Move to boss and attack
|
||||
var targetX = boss.Cx + Rng.Next(-10, 11);
|
||||
var targetY = boss.Cy + Rng.Next(-10, 11);
|
||||
await _game.MoveMouseFast(targetX, targetY);
|
||||
// Re-check healthbar every ~1.5s, need 5 consecutive misses (~7.5s) to confirm phase end
|
||||
if (sw.ElapsedMilliseconds - lastCheckMs > 1500)
|
||||
{
|
||||
var bossAlive = await IsBossAlive();
|
||||
lastCheckMs = sw.ElapsedMilliseconds;
|
||||
|
||||
_game.LeftMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(30, 50));
|
||||
_game.LeftMouseUp();
|
||||
await Helpers.Sleep(Rng.Next(20, 40));
|
||||
_game.RightMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(30, 50));
|
||||
_game.RightMouseUp();
|
||||
if (!bossAlive)
|
||||
{
|
||||
consecutiveMisses++;
|
||||
Log.Information("Healthbar not found ({Misses}/5 consecutive misses, {Ms}ms elapsed)",
|
||||
consecutiveMisses, sw.ElapsedMilliseconds);
|
||||
if (consecutiveMisses >= 5)
|
||||
{
|
||||
Log.Information("Boss phase over after {Ms}ms", sw.ElapsedMilliseconds);
|
||||
return lastBossWorldPos;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (consecutiveMisses > 0)
|
||||
Log.Information("Healthbar found again, resetting miss count (was {Misses})", consecutiveMisses);
|
||||
consecutiveMisses = 0;
|
||||
}
|
||||
}
|
||||
|
||||
await Helpers.Sleep(Rng.Next(100, 150));
|
||||
// Check mana to decide hold vs click
|
||||
var mana = _hudReader.Current.ManaPct;
|
||||
|
||||
if (!holding)
|
||||
{
|
||||
// Ramp up: click cycle until mana is stable above 80%
|
||||
if (mana >= 0.80f)
|
||||
manaStableCount++;
|
||||
else
|
||||
manaStableCount = 0;
|
||||
|
||||
var targetX = atkX + Rng.Next(-30, 31);
|
||||
var targetY = atkY + Rng.Next(-30, 31);
|
||||
await _game.MoveMouseFast(targetX, targetY);
|
||||
|
||||
_game.LeftMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(20, 35));
|
||||
_game.LeftMouseUp();
|
||||
_game.RightMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(20, 35));
|
||||
_game.RightMouseUp();
|
||||
|
||||
await Helpers.Sleep(Rng.Next(50, 80));
|
||||
|
||||
// After 5 consecutive checks above 80%, switch to hold
|
||||
if (manaStableCount >= 5)
|
||||
{
|
||||
Log.Information("Mana stable at {Mana:P0}, switching to hold attack", mana);
|
||||
_game.LeftMouseDown();
|
||||
_game.RightMouseDown();
|
||||
holding = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Holding: nudge mouse toward boss and monitor mana
|
||||
var targetX = atkX + Rng.Next(-30, 31);
|
||||
var targetY = atkY + Rng.Next(-30, 31);
|
||||
await _game.MoveMouseFast(targetX, targetY);
|
||||
|
||||
if (mana < 0.30f)
|
||||
{
|
||||
Log.Information("Mana dropped to {Mana:P0}, releasing to recover", mana);
|
||||
_game.LeftMouseUp();
|
||||
_game.RightMouseUp();
|
||||
holding = false;
|
||||
manaStableCount = 0;
|
||||
}
|
||||
|
||||
await Helpers.Sleep(Rng.Next(80, 120));
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs);
|
||||
return lastBossWorldPos;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (holding)
|
||||
{
|
||||
consecutiveMisses++;
|
||||
if (consecutiveMisses >= 15)
|
||||
{
|
||||
Log.Information("Boss gone after {Ms}ms ({Misses} consecutive misses)",
|
||||
sw.ElapsedMilliseconds, consecutiveMisses);
|
||||
return;
|
||||
}
|
||||
await Helpers.Sleep(200);
|
||||
_game.LeftMouseUp();
|
||||
_game.RightMouseUp();
|
||||
}
|
||||
}
|
||||
|
||||
Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs);
|
||||
}
|
||||
|
||||
private async Task AttackAtPosition(int x, int y, int durationMs)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
while (sw.ElapsedMilliseconds < durationMs)
|
||||
{
|
||||
if (_stopped) return;
|
||||
|
|
@ -469,17 +624,44 @@ public class BossRunExecutor : GameExecutor
|
|||
await _game.MoveMouseFast(targetX, targetY);
|
||||
|
||||
_game.LeftMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(30, 50));
|
||||
await Helpers.Sleep(Rng.Next(20, 35));
|
||||
_game.LeftMouseUp();
|
||||
await Helpers.Sleep(Rng.Next(20, 40));
|
||||
_game.RightMouseDown();
|
||||
await Helpers.Sleep(Rng.Next(30, 50));
|
||||
await Helpers.Sleep(Rng.Next(20, 35));
|
||||
_game.RightMouseUp();
|
||||
|
||||
await Helpers.Sleep(Rng.Next(100, 150));
|
||||
await Helpers.Sleep(Rng.Next(50, 80));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Find all template matches and click the one closest to screen center.
|
||||
/// </summary>
|
||||
private async Task ClickClosestTemplateToCenter(string templatePath)
|
||||
{
|
||||
const int screenCx = 2560 / 2;
|
||||
const int screenCy = 1440 / 2;
|
||||
|
||||
var matches = await _screen.TemplateMatchAll(templatePath);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
Log.Warning("No matches found for {Template}, clicking screen center", Path.GetFileName(templatePath));
|
||||
await _game.LeftClickAt(screenCx, screenCy);
|
||||
return;
|
||||
}
|
||||
|
||||
var closest = matches.OrderBy(m =>
|
||||
{
|
||||
var dx = m.X - screenCx;
|
||||
var dy = m.Y - screenCy;
|
||||
return dx * dx + dy * dy;
|
||||
}).First();
|
||||
|
||||
Log.Information("Clicking closest match at ({X},{Y}) conf={Conf:F3} (of {Count} matches)",
|
||||
closest.X, closest.Y, closest.Confidence, matches.Count);
|
||||
await _game.LeftClickAt(closest.X, closest.Y);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Walk to a world position using WASD keys, checking minimap position each iteration.
|
||||
/// </summary>
|
||||
|
|
@ -487,8 +669,12 @@ public class BossRunExecutor : GameExecutor
|
|||
{
|
||||
Log.Information("Walking to world ({X:F0},{Y:F0})", worldX, worldY);
|
||||
|
||||
const int screenCx = 1280;
|
||||
const int screenCy = 720;
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var heldKeys = new HashSet<int>();
|
||||
var lastBlinkMs = -2300L; // allow immediate first blink
|
||||
|
||||
try
|
||||
{
|
||||
|
|
@ -512,6 +698,19 @@ public class BossRunExecutor : GameExecutor
|
|||
var dirX = dx / len;
|
||||
var dirY = dy / len;
|
||||
|
||||
// Blink toward destination every ~2.3s ± 0.3s
|
||||
var blinkCooldown = 2300 + Rng.Next(-300, 301);
|
||||
if (sw.ElapsedMilliseconds - lastBlinkMs >= blinkCooldown)
|
||||
{
|
||||
// Move mouse in the travel direction so blink goes the right way
|
||||
var blinkX = screenCx + (int)(dirX * 400);
|
||||
var blinkY = screenCy + (int)(dirY * 400);
|
||||
await _game.MoveMouseFast(blinkX, blinkY);
|
||||
await Helpers.Sleep(30);
|
||||
await _game.PressKey(InputSender.VK.SPACE);
|
||||
lastBlinkMs = sw.ElapsedMilliseconds;
|
||||
}
|
||||
|
||||
// Map direction to WASD keys
|
||||
var wanted = new HashSet<int>();
|
||||
if (dirY < -0.3) wanted.Add(InputSender.VK.W); // up
|
||||
|
|
@ -556,13 +755,13 @@ public class BossRunExecutor : GameExecutor
|
|||
|
||||
// Walk away from loot (hold S briefly)
|
||||
await _game.KeyDown(InputSender.VK.S);
|
||||
await Helpers.Sleep(1000);
|
||||
await Helpers.Sleep(500);
|
||||
await _game.KeyUp(InputSender.VK.S);
|
||||
await Helpers.Sleep(300);
|
||||
await Helpers.Sleep(200);
|
||||
|
||||
// Press + to open portal
|
||||
await _game.PressPlus();
|
||||
await Helpers.Sleep(1500);
|
||||
await Helpers.Sleep(800);
|
||||
|
||||
// Find "The Ardura Caravan" and click it
|
||||
var caravanPos = await _inventory.FindAndClickNameplate("The Ardura Caravan", maxRetries: 5, retryDelayMs: 1500);
|
||||
|
|
@ -638,6 +837,7 @@ public class BossRunExecutor : GameExecutor
|
|||
|
||||
// Close stash
|
||||
await _game.PressEscape();
|
||||
_inventory.ResetStashTabState();
|
||||
await Helpers.Sleep(Delays.PostEscape);
|
||||
|
||||
Log.Information("Loot stored");
|
||||
|
|
|
|||
|
|
@ -76,7 +76,6 @@ public class BotOrchestrator : IAsyncDisposable
|
|||
EnemyDetector = new EnemyDetector();
|
||||
EnemyDetector.Enabled = true;
|
||||
BossDetector = new BossDetector();
|
||||
BossDetector.Enabled = true;
|
||||
FrameSaver = new FrameSaver();
|
||||
|
||||
// Register on shared pipeline
|
||||
|
|
|
|||
|
|
@ -38,13 +38,15 @@ public abstract class GameExecutor
|
|||
// ------ Loot pickup ------
|
||||
|
||||
// Tiers to skip (noise, low-value, or hidden by filter)
|
||||
private static readonly HashSet<string> SkipTiers = ["unknown", "gold"];
|
||||
private static readonly HashSet<string> SkipTiers = ["gold"];
|
||||
|
||||
public async Task Loot()
|
||||
{
|
||||
Log.Information("Starting loot pickup");
|
||||
|
||||
const int maxRounds = 5;
|
||||
var totalPicked = 0;
|
||||
|
||||
for (var round = 0; round < maxRounds; round++)
|
||||
{
|
||||
if (_stopped) return;
|
||||
|
|
@ -53,49 +55,58 @@ public abstract class GameExecutor
|
|||
_game.MoveMouseInstant(0, 1440);
|
||||
await Helpers.Sleep(100);
|
||||
|
||||
// Hold Alt to ensure all labels are visible, then capture
|
||||
// Hold Alt, capture, detect
|
||||
await _game.KeyDown(InputSender.VK.MENU);
|
||||
await Helpers.Sleep(250);
|
||||
var capture = _screen.CaptureRawBitmap();
|
||||
|
||||
// Detect magenta-bordered labels directly (no diff needed)
|
||||
using var capture = _screen.CaptureRawBitmap();
|
||||
var labels = _screen.DetectLootLabels(capture, capture);
|
||||
capture.Dispose();
|
||||
|
||||
// Filter out noise and unwanted tiers
|
||||
var pickups = labels.Where(l => !SkipTiers.Contains(l.Tier)).ToList();
|
||||
|
||||
if (pickups.Count == 0)
|
||||
{
|
||||
await _game.KeyUp(InputSender.VK.MENU);
|
||||
Log.Information("No loot labels in round {Round} (total detected: {Total}, filtered: {Filtered})",
|
||||
round + 1, labels.Count, labels.Count - pickups.Count);
|
||||
Log.Information("No loot labels in round {Round} (picked {Total} total)", round + 1, totalPicked);
|
||||
break;
|
||||
}
|
||||
|
||||
Log.Information("Round {Round}: {Count} loot labels ({Skipped} skipped)",
|
||||
round + 1, pickups.Count, labels.Count - pickups.Count);
|
||||
|
||||
foreach (var skip in labels.Where(l => SkipTiers.Contains(l.Tier)))
|
||||
Log.Debug("Skipped: tier={Tier} color=({R},{G},{B}) at ({X},{Y})",
|
||||
skip.Tier, skip.AvgR, skip.AvgG, skip.AvgB, skip.CenterX, skip.CenterY);
|
||||
|
||||
// Click each label center (Alt still held so labels visible)
|
||||
// Click all detected labels (positions are stable)
|
||||
foreach (var label in pickups)
|
||||
{
|
||||
if (_stopped) break;
|
||||
|
||||
Log.Information("Picking up: tier={Tier} color=({R},{G},{B}) at ({X},{Y})",
|
||||
label.Tier, label.AvgR, label.AvgG, label.AvgB, label.CenterX, label.CenterY);
|
||||
await _game.LeftClickAt(label.CenterX, label.CenterY);
|
||||
await Helpers.Sleep(300);
|
||||
totalPicked++;
|
||||
await Helpers.Sleep(200);
|
||||
}
|
||||
|
||||
// Quick check: capture small region around each clicked label to see if
|
||||
// new labels appeared underneath. If none changed, we're done.
|
||||
await Helpers.Sleep(300);
|
||||
_game.MoveMouseInstant(0, 1440);
|
||||
await Helpers.Sleep(100);
|
||||
|
||||
using var recheck = _screen.CaptureRawBitmap();
|
||||
var newLabels = _screen.DetectLootLabels(recheck, recheck);
|
||||
var newPickups = newLabels.Where(l => !SkipTiers.Contains(l.Tier)).ToList();
|
||||
|
||||
await _game.KeyUp(InputSender.VK.MENU);
|
||||
await Helpers.Sleep(500);
|
||||
|
||||
if (newPickups.Count == 0)
|
||||
{
|
||||
Log.Information("Quick recheck: no new labels, done (picked {Total} total)", totalPicked);
|
||||
break;
|
||||
}
|
||||
|
||||
Log.Information("Quick recheck: {Count} new labels appeared, continuing", newPickups.Count);
|
||||
await Helpers.Sleep(300);
|
||||
}
|
||||
|
||||
Log.Information("Loot pickup complete");
|
||||
Log.Information("Loot pickup complete ({Count} items)", totalPicked);
|
||||
}
|
||||
|
||||
// ------ Recovery ------
|
||||
|
|
|
|||
|
|
@ -20,5 +20,6 @@ public interface IInventoryManager
|
|||
Task<bool> SalvageItems(List<PlacedItem> items);
|
||||
(bool[,] Grid, List<PlacedItem> Items, int Free) GetInventoryState();
|
||||
Task ClickStashTab(StashTabInfo tab, StashTabInfo? parentFolder = null);
|
||||
void ResetStashTabState();
|
||||
Task GrabItemsFromStash(string layoutName, int maxItems, string? templatePath = null);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ public class InventoryManager : IInventoryManager
|
|||
|
||||
private bool _atOwnHideout = true;
|
||||
private string _sellerAccount = "";
|
||||
private string? _currentFolder;
|
||||
private string? _currentTab;
|
||||
private readonly IGameController _game;
|
||||
private readonly IScreenReader _screen;
|
||||
private readonly IClientLogWatcher _logWatcher;
|
||||
|
|
@ -344,7 +346,17 @@ public class InventoryManager : IInventoryManager
|
|||
|
||||
public async Task ClickStashTab(StashTabInfo tab, StashTabInfo? parentFolder = null)
|
||||
{
|
||||
if (parentFolder != null)
|
||||
var folderName = parentFolder?.Name;
|
||||
var tabName = tab.Name;
|
||||
|
||||
// Already on the right folder and tab — skip
|
||||
if (_currentFolder == folderName && _currentTab == tabName)
|
||||
{
|
||||
Log.Debug("Already on tab '{Tab}' (folder '{Folder}'), skipping", tabName, folderName ?? "none");
|
||||
return;
|
||||
}
|
||||
|
||||
if (parentFolder != null && _currentFolder != folderName)
|
||||
{
|
||||
Log.Information("Clicking folder '{Folder}' at ({X},{Y})", parentFolder.Name, parentFolder.ClickX, parentFolder.ClickY);
|
||||
await _game.LeftClickAt(parentFolder.ClickX, parentFolder.ClickY);
|
||||
|
|
@ -354,6 +366,15 @@ public class InventoryManager : IInventoryManager
|
|||
Log.Information("Clicking tab '{Tab}' at ({X},{Y})", tab.Name, tab.ClickX, tab.ClickY);
|
||||
await _game.LeftClickAt(tab.ClickX, tab.ClickY);
|
||||
await Helpers.Sleep(Delays.PostStashOpen);
|
||||
|
||||
_currentFolder = folderName;
|
||||
_currentTab = tabName;
|
||||
}
|
||||
|
||||
public void ResetStashTabState()
|
||||
{
|
||||
_currentFolder = null;
|
||||
_currentTab = null;
|
||||
}
|
||||
|
||||
public async Task GrabItemsFromStash(string layoutName, int maxItems, string? templatePath = null)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ public interface IScreenReader : IDisposable
|
|||
Task Snapshot();
|
||||
Task<DiffOcrResponse> DiffOcr(string? savePath = null, Region? region = null);
|
||||
Task<TemplateMatchResult?> TemplateMatch(string templatePath, Region? region = null);
|
||||
Task<List<TemplateMatchResult>> TemplateMatchAll(string templatePath, Region? region = null, double threshold = 0.7);
|
||||
Task<OcrResponse> NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);
|
||||
void SetLootBaseline(System.Drawing.Bitmap frame);
|
||||
List<LootLabel> DetectLootLabels(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);
|
||||
|
|
|
|||
|
|
@ -74,7 +74,17 @@ class PythonOcrBridge : IDisposable
|
|||
{
|
||||
_proc!.StandardInput.WriteLine(json);
|
||||
_proc.StandardInput.Flush();
|
||||
responseLine = _proc.StandardOutput.ReadLine()
|
||||
|
||||
// Read with timeout to prevent indefinite hang
|
||||
var readTask = Task.Run(() => _proc.StandardOutput.ReadLine());
|
||||
if (!readTask.Wait(TimeSpan.FromSeconds(15)))
|
||||
{
|
||||
Log.Warning("Python OCR daemon timed out after 15s, restarting");
|
||||
try { _proc.Kill(); } catch { /* best effort */ }
|
||||
_proc = null;
|
||||
throw new TimeoutException("Python OCR daemon timed out");
|
||||
}
|
||||
responseLine = readTask.Result
|
||||
?? throw new Exception("Python daemon returned null");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -52,21 +52,28 @@ public class ScreenReader : IScreenReader
|
|||
|
||||
public Task<OcrResponse> Ocr(Region? region = null, string? preprocess = null)
|
||||
{
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
using var bitmap = ScreenCapture.CaptureOrLoad(null, region);
|
||||
|
||||
OcrResponse result;
|
||||
if (preprocess == "tophat")
|
||||
{
|
||||
using var processed = ImagePreprocessor.PreprocessForOcr(bitmap);
|
||||
return Task.FromResult(_pythonBridge.OcrFromBitmap(processed));
|
||||
result = _pythonBridge.OcrFromBitmap(processed);
|
||||
}
|
||||
|
||||
if (preprocess == "clahe")
|
||||
else if (preprocess == "clahe")
|
||||
{
|
||||
using var processed = ImagePreprocessor.PreprocessClahe(bitmap);
|
||||
return Task.FromResult(_pythonBridge.OcrFromBitmap(processed));
|
||||
result = _pythonBridge.OcrFromBitmap(processed);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = _pythonBridge.OcrFromBitmap(bitmap);
|
||||
}
|
||||
|
||||
return Task.FromResult(_pythonBridge.OcrFromBitmap(bitmap));
|
||||
var allText = string.Join(" | ", result.Lines.Select(l => l.Text));
|
||||
Log.Information("OCR completed in {Ms}ms ({Lines} lines): {Text}", sw.ElapsedMilliseconds, result.Lines.Count, allText);
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public async Task<(int X, int Y)?> FindTextOnScreen(string searchText, bool fuzzy = false)
|
||||
|
|
@ -170,6 +177,13 @@ public class ScreenReader : IScreenReader
|
|||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task<List<TemplateMatchResult>> TemplateMatchAll(string templatePath, Region? region = null, double threshold = 0.7)
|
||||
{
|
||||
var results = _templateMatch.MatchAll(templatePath, region, threshold);
|
||||
Log.Information("TemplateMatchAll: {Count} matches for {Template}", results.Count, Path.GetFileName(templatePath));
|
||||
return Task.FromResult(results);
|
||||
}
|
||||
|
||||
// -- Save --
|
||||
|
||||
public Task SaveScreenshot(string path)
|
||||
|
|
@ -229,8 +243,9 @@ public class ScreenReader : IScreenReader
|
|||
var allLines = new List<OcrLine>();
|
||||
var allText = new List<string>();
|
||||
|
||||
foreach (var box in boxes)
|
||||
for (int bi = 0; bi < boxes.Count; bi++)
|
||||
{
|
||||
var box = boxes[bi];
|
||||
// Pad the crop slightly
|
||||
int pad = 4;
|
||||
int cx = Math.Max(0, box.X - pad);
|
||||
|
|
@ -239,7 +254,19 @@ public class ScreenReader : IScreenReader
|
|||
int ch = Math.Min(h - cy, box.Height + pad * 2);
|
||||
|
||||
using var crop = current.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb);
|
||||
var ocrResult = _pythonBridge.OcrFromBitmap(crop);
|
||||
var clusterSw = System.Diagnostics.Stopwatch.StartNew();
|
||||
OcrResponse ocrResult;
|
||||
try
|
||||
{
|
||||
ocrResult = _pythonBridge.OcrFromBitmap(crop);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
Log.Warning("NameplateDiffOcr: cluster {I}/{Count} OCR timed out, skipping", bi + 1, boxes.Count);
|
||||
continue;
|
||||
}
|
||||
Log.Debug("NameplateDiffOcr: cluster {I}/{Count} ({W}x{H}) OCR took {Ms}ms",
|
||||
bi + 1, boxes.Count, cw, ch, clusterSw.ElapsedMilliseconds);
|
||||
|
||||
// Offset word coordinates to screen space
|
||||
foreach (var line in ocrResult.Lines)
|
||||
|
|
@ -322,78 +349,301 @@ public class ScreenReader : IScreenReader
|
|||
return boxes;
|
||||
}
|
||||
|
||||
// -- Loot label detection (magenta background) --
|
||||
// -- Loot label detection (Canny edge + contour) --
|
||||
//
|
||||
// All loot labels: white border, magenta (255,0,255) background, black text.
|
||||
// Magenta never appears in the game world → detect directly, no diff needed.
|
||||
// Finds rectangular contours from Canny edges, filters by shape
|
||||
// (aspect ratio, size, rectangularity) and content (label interior
|
||||
// must have visible color/brightness, unlike the dark game world).
|
||||
// Single frame — no diff needed, no custom filter colors required.
|
||||
|
||||
public void SetLootBaseline(Bitmap frame) { }
|
||||
|
||||
// Detection parameters
|
||||
// -- Loot detection constants --
|
||||
private const int CannyLow = 20, CannyHigh = 80;
|
||||
// Shape constraints
|
||||
private const int LabelMinW = 80, LabelMaxW = 500;
|
||||
private const int LabelMinH = 15, LabelMaxH = 100;
|
||||
private const double LabelMinAspect = 1.3, LabelMaxAspect = 10.0;
|
||||
// Strict pass: well-formed rectangle contours
|
||||
private const double MinRectangularity = 0.5;
|
||||
private const float StrictMinBS = 200f;
|
||||
private const float StrictMinEdgeDensity = 25f;
|
||||
// Relaxed pass: any contour bbox in play area (catches VFX-broken borders)
|
||||
private const int RelaxedMinW = 100;
|
||||
private const float RelaxedMinBS = 250f;
|
||||
private const float RelaxedMinEdgeDensity = 25f;
|
||||
private const double UiMarginTop = 0.08;
|
||||
private const double UiMarginBottom = 0.82;
|
||||
// Post-processing
|
||||
private const int MergeGap = 30;
|
||||
private const int MergeYTolerance = 15;
|
||||
private const double NmsIouThresh = 0.4;
|
||||
|
||||
/// <summary>
|
||||
/// Two-pass loot label detection:
|
||||
/// 1. Strict: polygon-approximated rectangle contours (high precision)
|
||||
/// 2. Relaxed: any contour bbox in play area (catches VFX-broken borders)
|
||||
/// Results merged, horizontal fragments joined, then NMS.
|
||||
/// </summary>
|
||||
public List<LootLabel> DetectLootLabels(Bitmap reference, Bitmap current)
|
||||
{
|
||||
using var mat = BitmapConverter.ToMat(current);
|
||||
if (mat.Channels() == 4)
|
||||
Cv2.CvtColor(mat, mat, ColorConversionCodes.BGRA2BGR);
|
||||
|
||||
// Mask magenta background pixels (BGR: B≈255, G≈0, R≈255)
|
||||
using var mask = new Mat();
|
||||
Cv2.InRange(mat, new Scalar(200, 0, 200), new Scalar(255, 60, 255), mask);
|
||||
int imgH = mat.Height, imgW = mat.Width;
|
||||
int playTop = (int)(imgH * UiMarginTop);
|
||||
int playBot = (int)(imgH * UiMarginBottom);
|
||||
|
||||
// Morph close fills text gaps within a label
|
||||
// Height=2 bridges line gaps within multi-line labels but not between separate labels
|
||||
using var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(12, 2));
|
||||
using var closed = new Mat();
|
||||
Cv2.MorphologyEx(mask, closed, MorphTypes.Close, kernel);
|
||||
using var gray = new Mat();
|
||||
Cv2.CvtColor(mat, gray, ColorConversionCodes.BGR2GRAY);
|
||||
|
||||
// Save debug images
|
||||
using var hsv = new Mat();
|
||||
Cv2.CvtColor(mat, hsv, ColorConversionCodes.BGR2HSV);
|
||||
|
||||
// Edge detection
|
||||
using var edges = new Mat();
|
||||
Cv2.Canny(gray, edges, CannyLow, CannyHigh);
|
||||
|
||||
using var dilateKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3));
|
||||
using var dilated = new Mat();
|
||||
Cv2.Dilate(edges, dilated, dilateKernel, iterations: 1);
|
||||
|
||||
Cv2.FindContours(dilated, out var contours, out _, RetrievalModes.Tree, ContourApproximationModes.ApproxSimple);
|
||||
|
||||
var strict = new List<LabelCandidate>();
|
||||
var relaxed = new List<LabelCandidate>();
|
||||
|
||||
foreach (var contour in contours)
|
||||
{
|
||||
var box = Cv2.BoundingRect(contour);
|
||||
|
||||
// Common shape gate
|
||||
if (box.Width <= LabelMinW || box.Width >= LabelMaxW) continue;
|
||||
if (box.Height <= LabelMinH || box.Height >= LabelMaxH) continue;
|
||||
double aspect = (double)box.Width / Math.Max(box.Height, 1);
|
||||
if (aspect <= LabelMinAspect || aspect >= LabelMaxAspect) continue;
|
||||
|
||||
// Content metrics
|
||||
using var roiHsv = new Mat(hsv, box);
|
||||
var meanHsv = Cv2.Mean(roiHsv);
|
||||
float meanVal = (float)meanHsv[2];
|
||||
float meanSat = (float)meanHsv[1];
|
||||
float bs = meanVal + meanSat;
|
||||
|
||||
using var roiGray = new Mat(gray, box);
|
||||
using var roiEdges = new Mat();
|
||||
Cv2.Canny(roiGray, roiEdges, 50, 150);
|
||||
float ed = (float)Cv2.Mean(roiEdges)[0];
|
||||
|
||||
// Strict pass: well-formed polygon (4-8 vertices)
|
||||
var peri = Cv2.ArcLength(contour, true);
|
||||
var approx = Cv2.ApproxPolyDP(contour, 0.02 * peri, true);
|
||||
|
||||
if (approx.Length >= 4 && approx.Length <= 8)
|
||||
{
|
||||
double contourArea = Cv2.ContourArea(approx);
|
||||
double rect = contourArea / Math.Max(box.Width * box.Height, 1);
|
||||
if (rect >= MinRectangularity && bs >= StrictMinBS && ed >= StrictMinEdgeDensity)
|
||||
strict.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat));
|
||||
}
|
||||
|
||||
// Relaxed pass: any contour bbox in play area
|
||||
bool inPlay = box.Y > playTop && box.Y + box.Height < playBot;
|
||||
if (inPlay && box.Width >= RelaxedMinW && bs >= RelaxedMinBS && ed >= RelaxedMinEdgeDensity)
|
||||
relaxed.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat));
|
||||
}
|
||||
|
||||
// Merge strict + relaxed (strict wins on overlap)
|
||||
var merged = new List<LabelCandidate>(strict);
|
||||
foreach (var rlb in relaxed)
|
||||
{
|
||||
if (!OverlapsAny(rlb, strict, 0.3))
|
||||
merged.Add(rlb);
|
||||
}
|
||||
|
||||
// Join horizontal fragments
|
||||
merged = MergeHorizontal(merged, MergeGap, MergeYTolerance);
|
||||
|
||||
// Build LootLabels with color classification
|
||||
var scored = new List<(LootLabel Label, float Score)>();
|
||||
foreach (var c in merged)
|
||||
{
|
||||
var (avgR, avgG, avgB) = SampleLabelColor(mat, c.X, c.Y, c.W, c.H);
|
||||
var tier = LootColorClassifier.Classify(avgR, avgG, avgB);
|
||||
int cx = c.X + c.W / 2;
|
||||
int cy = c.Y + c.H / 2;
|
||||
scored.Add((new LootLabel(cx, cy, c.W, c.H, tier, avgR, avgG, avgB),
|
||||
c.MeanBrightness + c.MeanSaturation));
|
||||
}
|
||||
|
||||
// Final NMS
|
||||
var labels = NmsLootLabels(scored, NmsIouThresh);
|
||||
labels.Sort((a, b) => a.CenterY.CompareTo(b.CenterY));
|
||||
|
||||
// Debug images
|
||||
try
|
||||
{
|
||||
Cv2.ImWrite("debug_loot_mask.png", mask);
|
||||
Cv2.ImWrite("debug_loot_closed.png", closed);
|
||||
current.Save("debug_loot_capture.png", System.Drawing.Imaging.ImageFormat.Png);
|
||||
Log.Information("Saved debug images: debug_loot_mask.png, debug_loot_closed.png, debug_loot_capture.png");
|
||||
Cv2.ImWrite("debug_loot_edges.png", edges);
|
||||
Cv2.ImWrite("debug_loot_dilated.png", dilated);
|
||||
using var debugMat = mat.Clone();
|
||||
foreach (var label in labels)
|
||||
Cv2.Rectangle(debugMat,
|
||||
new Rect(label.CenterX - label.Width / 2, label.CenterY - label.Height / 2, label.Width, label.Height),
|
||||
new Scalar(0, 255, 0), 2);
|
||||
Cv2.ImWrite("debug_loot_detected.png", debugMat);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning(ex, "Failed to save debug images");
|
||||
}
|
||||
|
||||
Cv2.FindContours(closed, out var contours, out _,
|
||||
RetrievalModes.External, ContourApproximationModes.ApproxSimple);
|
||||
|
||||
Log.Information("DetectLootLabels: {N} magenta contours", contours.Length);
|
||||
|
||||
const int minW = 40, maxW = 600;
|
||||
const int minH = 8, maxH = 100;
|
||||
const double minAspect = 1.5;
|
||||
int yMax = mat.Height - 210;
|
||||
|
||||
var labels = new List<LootLabel>();
|
||||
foreach (var contour in contours)
|
||||
{
|
||||
var box = Cv2.BoundingRect(contour);
|
||||
double aspect = box.Height > 0 ? (double)box.Width / box.Height : 0;
|
||||
|
||||
if (box.Width < minW || box.Width > maxW ||
|
||||
box.Height < minH || box.Height > maxH ||
|
||||
aspect < minAspect ||
|
||||
box.Y < 65 || box.Y + box.Height > yMax)
|
||||
{
|
||||
Log.Information("Rejected contour: ({X},{Y}) {W}x{H} aspect={Aspect:F1} yMax={YMax}",
|
||||
box.X, box.Y, box.Width, box.Height, aspect, yMax);
|
||||
continue;
|
||||
}
|
||||
|
||||
int cx = box.X + box.Width / 2;
|
||||
int cy = box.Y + box.Height / 2;
|
||||
|
||||
Log.Information("Label at ({X},{Y}) {W}x{H}", box.X, box.Y, box.Width, box.Height);
|
||||
labels.Add(new LootLabel(cx, cy, box.Width, box.Height, "loot", 255, 0, 255));
|
||||
}
|
||||
Log.Information("DetectLootLabels: strict={Strict} relaxed={Relaxed} merged={Merged} final={Final}",
|
||||
strict.Count, relaxed.Count, merged.Count, labels.Count);
|
||||
foreach (var label in labels)
|
||||
Log.Information(" Label ({X},{Y}) {W}x{H} color=({R},{G},{B}) tier={Tier}",
|
||||
label.CenterX - label.Width / 2, label.CenterY - label.Height / 2,
|
||||
label.Width, label.Height, label.AvgR, label.AvgG, label.AvgB, label.Tier);
|
||||
|
||||
return labels;
|
||||
}
|
||||
|
||||
// -- Loot detection helpers --
|
||||
|
||||
private record struct LabelCandidate(int X, int Y, int W, int H, float MeanBrightness, float MeanSaturation);
|
||||
|
||||
private static (byte R, byte G, byte B) SampleLabelColor(Mat mat, int x, int y, int w, int h)
|
||||
{
|
||||
var roiX = x + w / 4;
|
||||
var roiY = y + h / 4;
|
||||
var roiW = Math.Min(mat.Cols - roiX, w / 2);
|
||||
var roiH = Math.Min(mat.Rows - roiY, h / 2);
|
||||
if (roiW <= 0 || roiH <= 0) return (0, 0, 0);
|
||||
using var roi = new Mat(mat, new Rect(roiX, roiY, roiW, roiH));
|
||||
var mean = Cv2.Mean(roi);
|
||||
return ((byte)mean[2], (byte)mean[1], (byte)mean[0]);
|
||||
}
|
||||
|
||||
private static bool OverlapsAny(LabelCandidate label, List<LabelCandidate> others, double iouThresh)
|
||||
{
|
||||
foreach (var o in others)
|
||||
{
|
||||
int ix1 = Math.Max(label.X, o.X), iy1 = Math.Max(label.Y, o.Y);
|
||||
int ix2 = Math.Min(label.X + label.W, o.X + o.W);
|
||||
int iy2 = Math.Min(label.Y + label.H, o.Y + o.H);
|
||||
int inter = Math.Max(0, ix2 - ix1) * Math.Max(0, iy2 - iy1);
|
||||
int union = label.W * label.H + o.W * o.H - inter;
|
||||
if (inter / (double)Math.Max(union, 1) > iouThresh)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge labels that sit side-by-side on the same line.
|
||||
/// </summary>
|
||||
private static List<LabelCandidate> MergeHorizontal(List<LabelCandidate> labels, int gap, int yTol)
|
||||
{
|
||||
if (labels.Count < 2) return labels;
|
||||
|
||||
var used = new bool[labels.Count];
|
||||
var indices = Enumerable.Range(0, labels.Count)
|
||||
.OrderBy(i => labels[i].Y).ThenBy(i => labels[i].X).ToList();
|
||||
var result = new List<LabelCandidate>();
|
||||
|
||||
for (int ii = 0; ii < indices.Count; ii++)
|
||||
{
|
||||
int i = indices[ii];
|
||||
if (used[i]) continue;
|
||||
used[i] = true;
|
||||
|
||||
var a = labels[i];
|
||||
int gx1 = a.X, gy1 = a.Y, gx2 = a.X + a.W, gy2 = a.Y + a.H;
|
||||
double briArea = a.MeanBrightness * a.W * a.H;
|
||||
double satArea = a.MeanSaturation * a.W * a.H;
|
||||
int totalArea = a.W * a.H;
|
||||
|
||||
bool changed = true;
|
||||
while (changed)
|
||||
{
|
||||
changed = false;
|
||||
for (int jj = 0; jj < indices.Count; jj++)
|
||||
{
|
||||
int j = indices[jj];
|
||||
if (used[j]) continue;
|
||||
var b = labels[j];
|
||||
|
||||
double cyA = (gy1 + gy2) / 2.0;
|
||||
double cyB = b.Y + b.H / 2.0;
|
||||
if (Math.Abs(cyA - cyB) > yTol) continue;
|
||||
|
||||
int hGap = Math.Max(b.X - gx2, gx1 - (b.X + b.W));
|
||||
if (hGap > gap) continue;
|
||||
|
||||
int bArea = b.W * b.H;
|
||||
gx1 = Math.Min(gx1, b.X);
|
||||
gy1 = Math.Min(gy1, b.Y);
|
||||
gx2 = Math.Max(gx2, b.X + b.W);
|
||||
gy2 = Math.Max(gy2, b.Y + b.H);
|
||||
briArea += b.MeanBrightness * bArea;
|
||||
satArea += b.MeanSaturation * bArea;
|
||||
totalArea += bArea;
|
||||
used[j] = true;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
int w = gx2 - gx1, h = gy2 - gy1;
|
||||
float bri = (float)(briArea / Math.Max(totalArea, 1));
|
||||
float sat = (float)(satArea / Math.Max(totalArea, 1));
|
||||
result.Add(new LabelCandidate(gx1, gy1, w, h, bri, sat));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<LootLabel> NmsLootLabels(List<(LootLabel Label, float Score)> candidates, double iouThresh)
|
||||
{
|
||||
if (candidates.Count == 0) return [];
|
||||
|
||||
candidates.Sort((a, b) => b.Score.CompareTo(a.Score));
|
||||
|
||||
var keep = new List<LootLabel>();
|
||||
var suppressed = new bool[candidates.Count];
|
||||
|
||||
for (int i = 0; i < candidates.Count; i++)
|
||||
{
|
||||
if (suppressed[i]) continue;
|
||||
keep.Add(candidates[i].Label);
|
||||
|
||||
var a = candidates[i].Label;
|
||||
int ax1 = a.CenterX - a.Width / 2, ay1 = a.CenterY - a.Height / 2;
|
||||
int ax2 = ax1 + a.Width, ay2 = ay1 + a.Height;
|
||||
int areaA = a.Width * a.Height;
|
||||
|
||||
for (int j = i + 1; j < candidates.Count; j++)
|
||||
{
|
||||
if (suppressed[j]) continue;
|
||||
var b = candidates[j].Label;
|
||||
int bx1 = b.CenterX - b.Width / 2, by1 = b.CenterY - b.Height / 2;
|
||||
int bx2 = bx1 + b.Width, by2 = by1 + b.Height;
|
||||
int areaB = b.Width * b.Height;
|
||||
|
||||
int ix1 = Math.Max(ax1, bx1), iy1 = Math.Max(ay1, by1);
|
||||
int ix2 = Math.Min(ax2, bx2), iy2 = Math.Min(ay2, by2);
|
||||
int inter = Math.Max(0, ix2 - ix1) * Math.Max(0, iy2 - iy1);
|
||||
double iou = inter / (double)(areaA + areaB - inter + 1);
|
||||
|
||||
if (iou >= iouThresh)
|
||||
suppressed[j] = true;
|
||||
}
|
||||
}
|
||||
|
||||
return keep;
|
||||
}
|
||||
|
||||
public void Dispose() => _pythonBridge.Dispose();
|
||||
|
||||
// -- OCR text matching --
|
||||
|
|
|
|||
|
|
@ -50,6 +50,65 @@ class TemplateMatchHandler
|
|||
return best;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Find all matches above threshold, suppressing overlapping detections.
|
||||
/// </summary>
|
||||
public List<TemplateMatchResult> MatchAll(string templatePath, Region? region = null,
|
||||
double threshold = 0.7)
|
||||
{
|
||||
if (!System.IO.File.Exists(templatePath))
|
||||
throw new FileNotFoundException($"Template file not found: {templatePath}");
|
||||
|
||||
using var screenshot = ScreenCapture.CaptureOrLoad(null, region);
|
||||
using var screenMat = BitmapConverter.ToMat(screenshot);
|
||||
using var template = Cv2.ImRead(templatePath, ImreadModes.Color);
|
||||
|
||||
if (template.Empty())
|
||||
throw new InvalidOperationException($"Failed to load template image: {templatePath}");
|
||||
|
||||
using var screenBgr = new Mat();
|
||||
if (screenMat.Channels() == 4)
|
||||
Cv2.CvtColor(screenMat, screenBgr, ColorConversionCodes.BGRA2BGR);
|
||||
else
|
||||
screenMat.CopyTo(screenBgr);
|
||||
|
||||
if (template.Rows > screenBgr.Rows || template.Cols > screenBgr.Cols)
|
||||
return [];
|
||||
|
||||
using var result = new Mat();
|
||||
Cv2.MatchTemplate(screenBgr, template, result, TemplateMatchModes.CCoeffNormed);
|
||||
|
||||
var offsetX = region?.X ?? 0;
|
||||
var offsetY = region?.Y ?? 0;
|
||||
var matches = new List<TemplateMatchResult>();
|
||||
|
||||
// Find all peaks above threshold using non-maximum suppression
|
||||
while (true)
|
||||
{
|
||||
Cv2.MinMaxLoc(result, out _, out double maxVal, out _, out OpenCvSharp.Point maxLoc);
|
||||
if (maxVal < threshold) break;
|
||||
|
||||
matches.Add(new TemplateMatchResult
|
||||
{
|
||||
X = offsetX + maxLoc.X + template.Cols / 2,
|
||||
Y = offsetY + maxLoc.Y + template.Rows / 2,
|
||||
Width = template.Cols,
|
||||
Height = template.Rows,
|
||||
Confidence = maxVal,
|
||||
});
|
||||
|
||||
// Suppress this region so we find the next match
|
||||
var suppressX = Math.Max(0, maxLoc.X - template.Cols / 2);
|
||||
var suppressY = Math.Max(0, maxLoc.Y - template.Rows / 2);
|
||||
var suppressW = Math.Min(result.Cols - suppressX, template.Cols);
|
||||
var suppressH = Math.Min(result.Rows - suppressY, template.Rows);
|
||||
using var roi = new Mat(result, new Rect(suppressX, suppressY, suppressW, suppressH));
|
||||
roi.SetTo(new Scalar(0));
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
private static TemplateMatchResult? MatchAtScale(Mat screen, Mat template,
|
||||
Region? region, double scale, double threshold)
|
||||
{
|
||||
|
|
|
|||