diff --git a/assets/resurrect.png b/assets/resurrect.png new file mode 100644 index 0000000..1df4480 Binary files /dev/null and b/assets/resurrect.png differ diff --git a/assets/unique-boss-present2.png b/assets/unique-boss-present2.png new file mode 100644 index 0000000..0f1af73 Binary files /dev/null and b/assets/unique-boss-present2.png differ diff --git a/assets/unique-boss-present3.png b/assets/unique-boss-present3.png new file mode 100644 index 0000000..834eaa5 Binary files /dev/null and b/assets/unique-boss-present3.png differ diff --git a/debug_loot_capture.png b/debug_loot_capture.png index 9a258ba..8fac337 100644 Binary files a/debug_loot_capture.png and b/debug_loot_capture.png differ diff --git a/debug_loot_detected.png b/debug_loot_detected.png index 1737e86..15f766c 100644 Binary files a/debug_loot_detected.png and b/debug_loot_detected.png differ diff --git a/debug_loot_dilated.png b/debug_loot_dilated.png index 0d5e137..7cd9e4d 100644 Binary files a/debug_loot_dilated.png and b/debug_loot_dilated.png differ diff --git a/debug_loot_edges.png b/debug_loot_edges.png index 6cf2a5a..2639463 100644 Binary files a/debug_loot_edges.png and b/debug_loot_edges.png differ diff --git a/src/Poe2Trade.Bot/BossRunExecutor.cs b/src/Poe2Trade.Bot/BossRunExecutor.cs index 2eb98d7..f97b103 100644 --- a/src/Poe2Trade.Bot/BossRunExecutor.cs +++ b/src/Poe2Trade.Bot/BossRunExecutor.cs @@ -18,13 +18,16 @@ public class BossRunExecutor : GameExecutor private static readonly string CathedralWellTemplate = Path.Combine("assets", "black-cathedral-well.png"); private static readonly string ReturnTheRingTemplate = Path.Combine("assets", "return-the-ring.png"); private static readonly string BossHealthbarTemplate = Path.Combine("assets", "unique-boss-present.png"); + private static readonly string BossHealthbarTemplate2 = Path.Combine("assets", "unique-boss-present2.png"); + private static readonly string BossHealthbarTemplate3 = Path.Combine("assets", "unique-boss-present3.png"); private static readonly string NewInstanceTemplate = Path.Combine("assets", "new.png"); + private static readonly string ResurrectTemplate = Path.Combine("assets", "resurrect.png"); private BossRunState _state = BossRunState.Idle; private readonly IClientLogWatcher _logWatcher; private readonly BossDetector _bossDetector; - private readonly HudReader _hudReader; private readonly NavigationExecutor _nav; + private readonly CombatManager _combat; public event Action? StateChanged; @@ -35,7 +38,7 @@ public class BossRunExecutor : GameExecutor { _logWatcher = logWatcher; _bossDetector = bossDetector; - _hudReader = hudReader; + _combat = new CombatManager(game, hudReader, new FlaskManager(game, hudReader)); _nav = nav; } @@ -56,8 +59,10 @@ public class BossRunExecutor : GameExecutor public async Task RunBossLoop() { _stopped = false; - Log.Information("Starting boss run loop ({Count} invitations)", _config.Kulemak.InvitationCount); + var runCount = _config.Kulemak.RunCount; + Log.Information("Starting boss run loop ({Count} runs)", runCount); + // First run: deposit inventory and grab 1 invitation if (!await Prepare()) { SetState(BossRunState.Failed); @@ -67,11 +72,11 @@ public class BossRunExecutor : GameExecutor } var completed = 0; - for (var i = 0; i < _config.Kulemak.InvitationCount; i++) + for (var i = 0; i < runCount; i++) { if (_stopped) break; - Log.Information("=== Boss run {N}/{Total} ===", i + 1, _config.Kulemak.InvitationCount); + Log.Information("=== Boss run {N}/{Total} ===", i + 1, runCount); if (!await TravelToZone()) { @@ -113,13 +118,14 @@ public class BossRunExecutor : GameExecutor } if (_stopped) break; - await StoreLoot(); + bool isLastRun = i == runCount - 1 || _stopped; + await StoreLoot(grabInvitation: !isLastRun); completed++; if (_stopped) break; } - Log.Information("Boss run loop finished: {Completed}/{Total} runs completed", completed, _config.Kulemak.InvitationCount); + Log.Information("Boss run loop finished: {Completed}/{Total} runs completed", completed, runCount); SetState(BossRunState.Complete); await Helpers.Sleep(1000); SetState(BossRunState.Idle); @@ -186,7 +192,7 @@ public class BossRunExecutor : GameExecutor (false, false) => "stash12", }; - await _inventory.GrabItemsFromStash(layoutName, _config.Kulemak.InvitationCount, InvitationTemplate); + await _inventory.GrabItemsFromStash(layoutName, 1, InvitationTemplate); } else { @@ -310,7 +316,7 @@ public class BossRunExecutor : GameExecutor Log.Information("Fight phase starting"); // Wait for arena to settle - await Helpers.Sleep(5500); + await Helpers.Sleep(4500); if (_stopped) return; // Find and click the cathedral door @@ -336,7 +342,7 @@ public class BossRunExecutor : GameExecutor const double wellWorldX = -496; const double wellWorldY = -378; - await WalkToWorldPosition(fightWorldX, fightWorldY); + await WalkToWorldPosition(fightWorldX, fightWorldY, cancelWhen: IsBossAlive); if (_stopped) return; // 3x fight-then-well loop @@ -356,14 +362,17 @@ public class BossRunExecutor : GameExecutor Log.Information("Fight area updated to ({X:F0},{Y:F0})", fightWorldX, fightWorldY); } + // Wait for death animation before looking for well + await Helpers.Sleep(3000); + // Walk to well and click the closest match to screen center Log.Information("Phase {Phase} done, walking to well", phase); await WalkToWorldPosition(wellWorldX, wellWorldY); await ClickClosestTemplateToCenter(CathedralWellTemplate); - await Helpers.Sleep(2000); + await Helpers.Sleep(200); // Walk back to fight position for next phase - await WalkToWorldPosition(fightWorldX, fightWorldY); + await WalkToWorldPosition(fightWorldX, fightWorldY, cancelWhen: IsBossAlive); } // 4th fight - no well after @@ -372,7 +381,10 @@ public class BossRunExecutor : GameExecutor await AttackBossUntilGone(); if (_stopped) return; - // Return the ring + // Walk to ring area and return it + await WalkToWorldPosition(-440, -330); + if (_stopped) return; + Log.Information("Looking for Return the Ring..."); var ring = await _screen.TemplateMatch(ReturnTheRingTemplate); if (ring == null) @@ -419,13 +431,39 @@ public class BossRunExecutor : GameExecutor /// /// Check top-of-screen region for the unique boss healthbar frame. - /// Uses lower threshold (0.5) to tolerate partial overlay from YOLO bounding boxes. + /// Healthbar spans (750,16) to (1818,112). Uses lower threshold (0.5) to tolerate YOLO overlay. /// private async Task IsBossAlive() { - var topRegion = new Region(800, 0, 960, 120); - var matches = await _screen.TemplateMatchAll(BossHealthbarTemplate, topRegion, threshold: 0.5); - return matches.Count > 0; + var topRegion = new Region(750, 16, 1068, 96); + + // Check all three healthbar templates — boss is alive if ANY matches + var m1 = await _screen.TemplateMatchAll(BossHealthbarTemplate, topRegion, threshold: 0.5); + if (m1.Count > 0) return true; + + var m2 = await _screen.TemplateMatchAll(BossHealthbarTemplate2, topRegion, threshold: 0.5); + if (m2.Count > 0) return true; + + var m3 = await _screen.TemplateMatchAll(BossHealthbarTemplate3, topRegion, threshold: 0.5); + if (m3.Count > 0) return true; + + return false; + } + + /// + /// Check for the "Resurrect at Checkpoint" button — means we died. + /// If found, click it, wait for respawn, and return true. + /// + private async Task CheckDeath() + { + var match = await _screen.TemplateMatch(ResurrectTemplate); + if (match == null) return false; + + Log.Warning("Death detected! Clicking resurrect at ({X},{Y})", match.X, match.Y); + await _combat.ReleaseAll(); + await _game.LeftClickAt(match.X, match.Y); + await Helpers.Sleep(3000); // wait for respawn + loading + return true; } /// @@ -435,13 +473,13 @@ public class BossRunExecutor : GameExecutor { Log.Information("Waiting for boss healthbar to appear..."); var sw = Stopwatch.StartNew(); - - const int screenCx = 1280; - const int screenCy = 720; + var atkX = 1280; + var atkY = 720; while (sw.ElapsedMilliseconds < timeoutMs) { if (_stopped) return false; + if (await CheckDeath()) continue; if (await IsBossAlive()) { @@ -449,19 +487,14 @@ public class BossRunExecutor : GameExecutor return true; } - // Attack at center while waiting — boss often spawns on top of us - var targetX = screenCx + Rng.Next(-30, 31); - var targetY = screenCy + Rng.Next(-30, 31); - await _game.MoveMouseFast(targetX, targetY); - - _game.LeftMouseDown(); - await Helpers.Sleep(Rng.Next(20, 35)); - _game.LeftMouseUp(); - _game.RightMouseDown(); - await Helpers.Sleep(Rng.Next(20, 35)); - _game.RightMouseUp(); - - await Helpers.Sleep(Rng.Next(50, 80)); + // Attack toward YOLO-detected boss if available, otherwise last known position + var snapshot = _bossDetector.Latest; + if (snapshot.Bosses.Count > 0) + { + atkX = snapshot.Bosses[0].Cx; + atkY = snapshot.Bosses[0].Cy; + } + await _combat.Tick(atkX, atkY); } Log.Warning("WaitForBossSpawn timed out after {Ms}ms", timeoutMs); @@ -490,11 +523,9 @@ public class BossRunExecutor : GameExecutor await Helpers.Sleep(200); var sw = Stopwatch.StartNew(); - var consecutiveMisses = 0; var lastCheckMs = 0L; - var holding = false; - var manaStableCount = 0; + await _combat.Reset(); Log.Information("Boss is alive, engaging"); try @@ -520,82 +551,22 @@ public class BossRunExecutor : GameExecutor lastBossWorldPos = (bossWorldX, bossWorldY); } - // Re-check healthbar every ~1.5s, need 5 consecutive misses (~7.5s) to confirm phase end - if (sw.ElapsedMilliseconds - lastCheckMs > 1500) + // Re-check healthbar every ~0.5s, first miss = phase over + if (sw.ElapsedMilliseconds - lastCheckMs > 500) { + if (await CheckDeath()) continue; + var bossAlive = await IsBossAlive(); lastCheckMs = sw.ElapsedMilliseconds; if (!bossAlive) { - consecutiveMisses++; - Log.Information("Healthbar not found ({Misses}/5 consecutive misses, {Ms}ms elapsed)", - consecutiveMisses, sw.ElapsedMilliseconds); - if (consecutiveMisses >= 5) - { - Log.Information("Boss phase over after {Ms}ms", sw.ElapsedMilliseconds); - return lastBossWorldPos; - } - } - else - { - if (consecutiveMisses > 0) - Log.Information("Healthbar found again, resetting miss count (was {Misses})", consecutiveMisses); - consecutiveMisses = 0; + Log.Information("Healthbar not found, boss phase over after {Ms}ms", sw.ElapsedMilliseconds); + return lastBossWorldPos; } } - // Check mana to decide hold vs click - var mana = _hudReader.Current.ManaPct; - - if (!holding) - { - // Ramp up: click cycle until mana is stable above 80% - if (mana >= 0.80f) - manaStableCount++; - else - manaStableCount = 0; - - var targetX = atkX + Rng.Next(-30, 31); - var targetY = atkY + Rng.Next(-30, 31); - await _game.MoveMouseFast(targetX, targetY); - - _game.LeftMouseDown(); - await Helpers.Sleep(Rng.Next(20, 35)); - _game.LeftMouseUp(); - _game.RightMouseDown(); - await Helpers.Sleep(Rng.Next(20, 35)); - _game.RightMouseUp(); - - await Helpers.Sleep(Rng.Next(50, 80)); - - // After 5 consecutive checks above 80%, switch to hold - if (manaStableCount >= 5) - { - Log.Information("Mana stable at {Mana:P0}, switching to hold attack", mana); - _game.LeftMouseDown(); - _game.RightMouseDown(); - holding = true; - } - } - else - { - // Holding: nudge mouse toward boss and monitor mana - var targetX = atkX + Rng.Next(-30, 31); - var targetY = atkY + Rng.Next(-30, 31); - await _game.MoveMouseFast(targetX, targetY); - - if (mana < 0.30f) - { - Log.Information("Mana dropped to {Mana:P0}, releasing to recover", mana); - _game.LeftMouseUp(); - _game.RightMouseUp(); - holding = false; - manaStableCount = 0; - } - - await Helpers.Sleep(Rng.Next(80, 120)); - } + await _combat.Tick(atkX, atkY); } Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs); @@ -603,34 +574,26 @@ public class BossRunExecutor : GameExecutor } finally { - if (holding) - { - _game.LeftMouseUp(); - _game.RightMouseUp(); - } + await _combat.ReleaseAll(); } } private async Task AttackAtPosition(int x, int y, int durationMs) { + await _combat.Reset(); var sw = Stopwatch.StartNew(); - - while (sw.ElapsedMilliseconds < durationMs) + try { - if (_stopped) return; - - var targetX = x + Rng.Next(-20, 21); - var targetY = y + Rng.Next(-20, 21); - await _game.MoveMouseFast(targetX, targetY); - - _game.LeftMouseDown(); - await Helpers.Sleep(Rng.Next(20, 35)); - _game.LeftMouseUp(); - _game.RightMouseDown(); - await Helpers.Sleep(Rng.Next(20, 35)); - _game.RightMouseUp(); - - await Helpers.Sleep(Rng.Next(50, 80)); + while (sw.ElapsedMilliseconds < durationMs) + { + if (_stopped) return; + if (await CheckDeath()) continue; + await _combat.Tick(x, y, jitter: 20); + } + } + finally + { + await _combat.ReleaseAll(); } } @@ -665,7 +628,8 @@ public class BossRunExecutor : GameExecutor /// /// Walk to a world position using WASD keys, checking minimap position each iteration. /// - private async Task WalkToWorldPosition(double worldX, double worldY, int timeoutMs = 10000, double arrivalDist = 15) + private async Task WalkToWorldPosition(double worldX, double worldY, int timeoutMs = 10000, + double arrivalDist = 15, Func>? cancelWhen = null) { Log.Information("Walking to world ({X:F0},{Y:F0})", worldX, worldY); @@ -682,6 +646,12 @@ public class BossRunExecutor : GameExecutor { if (_stopped) break; + if (cancelWhen != null && await cancelWhen()) + { + Log.Information("Walk cancelled early (cancel condition met)"); + break; + } + var pos = _nav.WorldPosition; var dx = worldX - pos.X; var dy = worldY - pos.Y; @@ -795,7 +765,7 @@ public class BossRunExecutor : GameExecutor return true; } - private async Task StoreLoot() + private async Task StoreLoot(bool grabInvitation = false) { SetState(BossRunState.StoringLoot); Log.Information("Storing loot"); @@ -812,12 +782,11 @@ public class BossRunExecutor : GameExecutor } await Helpers.Sleep(Delays.PostStashOpen); - // Click loot tab + // Click loot tab and deposit all inventory items var (lootTab, lootFolder) = ResolveTabPath(_config.Kulemak.LootTabPath); if (lootTab != null) await _inventory.ClickStashTab(lootTab, lootFolder); - // Deposit all inventory items var scanResult = await _screen.Grid.Scan("inventory"); if (scanResult.Occupied.Count > 0) { @@ -835,6 +804,25 @@ public class BossRunExecutor : GameExecutor await Helpers.Sleep(Delays.PostEscape); } + // Grab 1 invitation for the next run while stash is still open + if (grabInvitation) + { + var (invTab, invFolder) = ResolveTabPath(_config.Kulemak.InvitationTabPath); + if (invTab != null) + { + await _inventory.ClickStashTab(invTab, invFolder); + var layoutName = (invTab.GridCols == 24, invFolder != null) switch + { + (true, true) => "stash24_folder", + (true, false) => "stash24", + (false, true) => "stash12_folder", + (false, false) => "stash12", + }; + await _inventory.GrabItemsFromStash(layoutName, 1, InvitationTemplate); + Log.Information("Grabbed 1 invitation for next run"); + } + } + // Close stash await _game.PressEscape(); _inventory.ResetStashTabState(); diff --git a/src/Poe2Trade.Bot/BotOrchestrator.cs b/src/Poe2Trade.Bot/BotOrchestrator.cs index 45ec69c..138918f 100644 --- a/src/Poe2Trade.Bot/BotOrchestrator.cs +++ b/src/Poe2Trade.Bot/BotOrchestrator.cs @@ -46,6 +46,7 @@ public class BotOrchestrator : IAsyncDisposable public EnemyDetector EnemyDetector { get; } public BossDetector BossDetector { get; } public FrameSaver FrameSaver { get; } + public LootDebugDetector LootDebugDetector { get; } public BossRunExecutor BossRunExecutor { get; } private readonly Dictionary _scrapExecutors = new(); @@ -77,6 +78,7 @@ public class BotOrchestrator : IAsyncDisposable EnemyDetector.Enabled = true; BossDetector = new BossDetector(); FrameSaver = new FrameSaver(); + LootDebugDetector = new LootDebugDetector(screen); // Register on shared pipeline pipelineService.Pipeline.AddConsumer(minimapCapture); diff --git a/src/Poe2Trade.Bot/CombatManager.cs b/src/Poe2Trade.Bot/CombatManager.cs new file mode 100644 index 0000000..a37d2e7 --- /dev/null +++ b/src/Poe2Trade.Bot/CombatManager.cs @@ -0,0 +1,154 @@ +using System.Diagnostics; +using Poe2Trade.Core; +using Poe2Trade.Game; +using Poe2Trade.Screen; +using Serilog; + +namespace Poe2Trade.Bot; + +/// +/// Manages the attack state machine (click → hold) with mana monitoring and flask usage. +/// Call each combat loop iteration, between phases, +/// and when done. +/// +public class CombatManager +{ + private static readonly Random Rng = new(); + + // Orbit: cycle W→D→S→A to dodge in a small circle + private static readonly int[] OrbitKeys = + [InputSender.VK.W, InputSender.VK.D, InputSender.VK.S, InputSender.VK.A]; + private const int OrbitStepMs = 400; // time per direction + + private readonly IGameController _game; + private readonly HudReader _hudReader; + private readonly FlaskManager _flasks; + + private bool _holding; + private int _manaStableCount; + private readonly Stopwatch _orbitSw = Stopwatch.StartNew(); + private int _orbitIndex = -1; + private long _lastOrbitMs; + + public bool IsHolding => _holding; + + public CombatManager(IGameController game, HudReader hudReader, FlaskManager flasks) + { + _game = game; + _hudReader = hudReader; + _flasks = flasks; + } + + /// + /// One combat iteration: flask check, mana-based click/hold, mouse jitter toward target. + /// + public async Task Tick(int x, int y, int jitter = 30) + { + await _flasks.Tick(); + await UpdateOrbit(); + + var mana = _hudReader.Current.ManaPct; + + if (!_holding) + { + if (mana >= 0.80f) + _manaStableCount++; + else + _manaStableCount = 0; + + var targetX = x + Rng.Next(-jitter, jitter + 1); + var targetY = y + Rng.Next(-jitter, jitter + 1); + await _game.MoveMouseFast(targetX, targetY); + + _game.LeftMouseDown(); + await Helpers.Sleep(Rng.Next(20, 35)); + _game.LeftMouseUp(); + _game.RightMouseDown(); + await Helpers.Sleep(Rng.Next(20, 35)); + _game.RightMouseUp(); + + await Helpers.Sleep(Rng.Next(50, 80)); + + if (_manaStableCount >= 5) + { + Log.Information("Mana stable at {Mana:P0}, switching to hold attack", mana); + _game.LeftMouseDown(); + _game.RightMouseDown(); + _holding = true; + } + } + else + { + var targetX = x + Rng.Next(-jitter, jitter + 1); + var targetY = y + Rng.Next(-jitter, jitter + 1); + await _game.MoveMouseFast(targetX, targetY); + + if (mana < 0.30f) + { + Log.Information("Mana dropped to {Mana:P0}, releasing to recover", mana); + _game.LeftMouseUp(); + _game.RightMouseUp(); + _holding = false; + _manaStableCount = 0; + } + + await Helpers.Sleep(Rng.Next(80, 120)); + } + } + + /// + /// Cycle WASD directions to orbit in a small circle while attacking. + /// + private async Task UpdateOrbit() + { + var now = _orbitSw.ElapsedMilliseconds; + if (now - _lastOrbitMs < OrbitStepMs) return; + _lastOrbitMs = now; + + // Release previous direction + if (_orbitIndex >= 0) + await _game.KeyUp(OrbitKeys[_orbitIndex]); + + // Advance to next direction + _orbitIndex = (_orbitIndex + 1) % OrbitKeys.Length; + await _game.KeyDown(OrbitKeys[_orbitIndex]); + } + + private async Task ReleaseOrbit() + { + if (_orbitIndex >= 0) + { + await _game.KeyUp(OrbitKeys[_orbitIndex]); + _orbitIndex = -1; + } + } + + /// + /// Reset state for a new combat phase (releases held buttons if any). + /// + public async Task Reset() + { + if (_holding) + { + _game.LeftMouseUp(); + _game.RightMouseUp(); + } + _holding = false; + _manaStableCount = 0; + await ReleaseOrbit(); + } + + /// + /// Release any held mouse buttons and movement keys. Call in finally blocks. + /// + public async Task ReleaseAll() + { + if (_holding) + { + _game.LeftMouseUp(); + _game.RightMouseUp(); + _holding = false; + } + await ReleaseOrbit(); + } +} diff --git a/src/Poe2Trade.Bot/FlaskManager.cs b/src/Poe2Trade.Bot/FlaskManager.cs new file mode 100644 index 0000000..70b0649 --- /dev/null +++ b/src/Poe2Trade.Bot/FlaskManager.cs @@ -0,0 +1,56 @@ +using System.Diagnostics; +using Poe2Trade.Game; +using Poe2Trade.Screen; +using Serilog; + +namespace Poe2Trade.Bot; + +/// +/// Monitors life/mana and presses flask keys when they drop below thresholds. +/// Call from any combat loop iteration. +/// +public class FlaskManager +{ + private const float LifeThreshold = 0.50f; + private const float ManaThreshold = 0.50f; + private const long CooldownMs = 4000; + + // VK codes for number keys 1 and 2 + private const int VK_1 = 0x31; + private const int VK_2 = 0x32; + + private readonly IGameController _game; + private readonly HudReader _hudReader; + private readonly Stopwatch _sw = Stopwatch.StartNew(); + private long _lastLifeFlaskMs = -CooldownMs; + private long _lastManaFlaskMs = -CooldownMs; + + public FlaskManager(IGameController game, HudReader hudReader) + { + _game = game; + _hudReader = hudReader; + } + + /// + /// Check life/mana and press flask keys if needed. Call every loop iteration. + /// + public async Task Tick() + { + var hud = _hudReader.Current; + var now = _sw.ElapsedMilliseconds; + + if (hud.LifePct < LifeThreshold && now - _lastLifeFlaskMs >= CooldownMs) + { + Log.Debug("Life flask: {Life:P0} < {Threshold:P0}", hud.LifePct, LifeThreshold); + await _game.PressKey(VK_1); + _lastLifeFlaskMs = now; + } + + if (hud.ManaPct < ManaThreshold && now - _lastManaFlaskMs >= CooldownMs) + { + Log.Debug("Mana flask: {Mana:P0} < {Threshold:P0}", hud.ManaPct, ManaThreshold); + await _game.PressKey(VK_2); + _lastManaFlaskMs = now; + } + } +} diff --git a/src/Poe2Trade.Core/ConfigStore.cs b/src/Poe2Trade.Core/ConfigStore.cs index da09e1c..6d27a86 100644 --- a/src/Poe2Trade.Core/ConfigStore.cs +++ b/src/Poe2Trade.Core/ConfigStore.cs @@ -44,6 +44,7 @@ public class KulemakSettings public string InvitationTabPath { get; set; } = ""; public string LootTabPath { get; set; } = ""; public int InvitationCount { get; set; } = 15; + public int RunCount { get; set; } = 15; } public class ConfigStore diff --git a/src/Poe2Trade.Core/Delays.cs b/src/Poe2Trade.Core/Delays.cs index 31632b2..a0b7bee 100644 --- a/src/Poe2Trade.Core/Delays.cs +++ b/src/Poe2Trade.Core/Delays.cs @@ -3,7 +3,7 @@ namespace Poe2Trade.Core; public static class Delays { public const int PostFocus = 300; - public const int PostTravel = 1500; + public const int PostTravel = 3000; public const int PostStashOpen = 1000; public const int ClickInterval = 150; public const int PostEscape = 500; diff --git a/src/Poe2Trade.Screen/LootDebugDetector.cs b/src/Poe2Trade.Screen/LootDebugDetector.cs new file mode 100644 index 0000000..fbf7a28 --- /dev/null +++ b/src/Poe2Trade.Screen/LootDebugDetector.cs @@ -0,0 +1,71 @@ +using Serilog; + +namespace Poe2Trade.Screen; + +/// +/// Debug-only: periodically captures the screen, runs loot label detection, +/// and exposes the latest results for overlay rendering. +/// +public class LootDebugDetector : IDisposable +{ + private readonly IScreenReader _screen; + private volatile List _latest = []; + private Timer? _timer; + private volatile bool _enabled; + private int _running; // guard against overlapping ticks + + public LootDebugDetector(IScreenReader screen) + { + _screen = screen; + } + + public IReadOnlyList Latest => _latest; + + public bool Enabled + { + get => _enabled; + set + { + if (_enabled == value) return; + _enabled = value; + if (value) + _timer = new Timer(_ => Tick(), null, 0, 500); + else + { + _timer?.Dispose(); + _timer = null; + _latest = []; + } + } + } + + private void Tick() + { + if (!_enabled) return; + if (Interlocked.CompareExchange(ref _running, 1, 0) != 0) return; + + try + { + using var frame = _screen.CaptureRawBitmap(); + var labels = _screen.DetectLootLabels(frame, frame); + _latest = labels; + if (labels.Count > 0) + Log.Information("[LootDebug] Detected {Count} labels", labels.Count); + } + catch (Exception ex) + { + Log.Warning("[LootDebug] Detection failed: {Error}", ex.Message); + _latest = []; + } + finally + { + Interlocked.Exchange(ref _running, 0); + } + } + + public void Dispose() + { + _timer?.Dispose(); + _timer = null; + } +} diff --git a/src/Poe2Trade.Screen/ScreenReader.cs b/src/Poe2Trade.Screen/ScreenReader.cs index 74445ef..1647133 100644 --- a/src/Poe2Trade.Screen/ScreenReader.cs +++ b/src/Poe2Trade.Screen/ScreenReader.cs @@ -202,11 +202,20 @@ public class ScreenReader : IScreenReader public Bitmap CaptureRawBitmap() => ScreenCapture.CaptureOrLoad(null, null); + // Nameplate search region — skip top HUD, bottom bar, and side margins + private const int NpTop = 120, NpBottom = 1080, NpMargin = 300; + public Task NameplateDiffOcr(Bitmap reference, Bitmap current) { int w = Math.Min(reference.Width, current.Width); int h = Math.Min(reference.Height, current.Height); + // Clamp search region to image bounds + int scanY0 = Math.Min(NpTop, h); + int scanY1 = Math.Min(NpBottom, h); + int scanX0 = Math.Min(NpMargin, w); + int scanX1 = Math.Max(scanX0, w - NpMargin); + var refData = reference.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); byte[] refPx = new byte[refData.Stride * h]; @@ -218,74 +227,100 @@ public class ScreenReader : IScreenReader current.UnlockBits(curData); // Build a binary mask of pixels that got significantly brighter (nameplates are bright text) + // Only scan within the play-area region to skip UI and reduce work const int brightThresh = 30; - bool[] mask = new bool[w * h]; - Parallel.For(0, h, y => + int scanW = scanX1 - scanX0; + int scanH = scanY1 - scanY0; + bool[] mask = new bool[scanW * scanH]; + Parallel.For(0, scanH, sy => { + int y = sy + scanY0; int rowOff = y * stride; - for (int x = 0; x < w; x++) + for (int sx = 0; sx < scanW; sx++) { + int x = sx + scanX0; int i = rowOff + x * 4; int brighter = (curPx[i] - refPx[i]) + (curPx[i + 1] - refPx[i + 1]) + (curPx[i + 2] - refPx[i + 2]); if (brighter > brightThresh) - mask[y * w + x] = true; + mask[sy * scanW + sx] = true; } }); // Find connected clusters via row-scan: collect bounding boxes of bright regions - var boxes = FindBrightClusters(mask, w, h, minWidth: 40, minHeight: 10, maxGap: 8); + var boxes = FindBrightClusters(mask, scanW, scanH, minWidth: 40, minHeight: 10, maxGap: 8); + + // Offset cluster boxes back to full-image coordinates + for (int i = 0; i < boxes.Count; i++) + { + var b = boxes[i]; + boxes[i] = new Rectangle(b.X + scanX0, b.Y + scanY0, b.Width, b.Height); + } Log.Information("NameplateDiff: found {Count} bright clusters", boxes.Count); if (boxes.Count == 0) return Task.FromResult(new OcrResponse { Text = "", Lines = [] }); - // OCR each cluster crop, accumulate results with screen-space coordinates - var allLines = new List(); - var allText = new List(); + // Collect valid cluster crops and stitch into a single image for one OCR call + const int pad = 4; + const int sep = 20; // black separator between crops to prevent cross-detection + var crops = new List<(int screenX, int screenY, int cropW, int cropH, int stitchY)>(); - for (int bi = 0; bi < boxes.Count; bi++) + int maxCropW = 0; + int totalH = 0; + foreach (var box in boxes) { - var box = boxes[bi]; - // Pad the crop slightly - int pad = 4; int cx = Math.Max(0, box.X - pad); int cy = Math.Max(0, box.Y - pad); int cw = Math.Min(w - cx, box.Width + pad * 2); int ch = Math.Min(h - cy, box.Height + pad * 2); - using var crop = current.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb); - var clusterSw = System.Diagnostics.Stopwatch.StartNew(); - OcrResponse ocrResult; - try - { - ocrResult = _pythonBridge.OcrFromBitmap(crop); - } - catch (TimeoutException) - { - Log.Warning("NameplateDiffOcr: cluster {I}/{Count} OCR timed out, skipping", bi + 1, boxes.Count); - continue; - } - Log.Debug("NameplateDiffOcr: cluster {I}/{Count} ({W}x{H}) OCR took {Ms}ms", - bi + 1, boxes.Count, cw, ch, clusterSw.ElapsedMilliseconds); + crops.Add((cx, cy, cw, ch, totalH)); + maxCropW = Math.Max(maxCropW, cw); + totalH += ch + sep; + } - // Offset word coordinates to screen space - foreach (var line in ocrResult.Lines) + if (crops.Count == 0) + return Task.FromResult(new OcrResponse { Text = "", Lines = [] }); + + totalH -= sep; // no separator after last crop + + // Stitch all crops vertically into one image + using var stitched = new Bitmap(maxCropW, totalH, PixelFormat.Format32bppArgb); + using (var g = System.Drawing.Graphics.FromImage(stitched)) + { + g.Clear(System.Drawing.Color.Black); + foreach (var (sx, sy, cw, ch, sY) in crops) + g.DrawImage(current, new Rectangle(0, sY, cw, ch), new Rectangle(sx, sy, cw, ch), GraphicsUnit.Pixel); + } + + // Single OCR call for all clusters + var ocrSw = System.Diagnostics.Stopwatch.StartNew(); + OcrResponse ocrResult; + try + { + ocrResult = _pythonBridge.OcrFromBitmap(stitched); + } + catch (TimeoutException) + { + Log.Warning("NameplateDiffOcr: batch OCR timed out ({Count} clusters)", crops.Count); + return Task.FromResult(new OcrResponse { Text = "", Lines = [] }); + } + Log.Information("NameplateDiffOcr: batch OCR {Count} clusters in {Ms}ms", + crops.Count, ocrSw.ElapsedMilliseconds); + + // Map OCR results back to screen coordinates + foreach (var line in ocrResult.Lines) + { + foreach (var word in line.Words) { - foreach (var word in line.Words) - { - word.X += cx; - word.Y += cy; - } - allLines.Add(line); - allText.Add(line.Text); + // Find which crop this word belongs to by Y position + var crop = crops.Last(c => word.Y >= c.stitchY); + word.X += crop.screenX; + word.Y = word.Y - crop.stitchY + crop.screenY; } } - return Task.FromResult(new OcrResponse - { - Text = string.Join("\n", allText), - Lines = allLines, - }); + return Task.FromResult(ocrResult); } private static List FindBrightClusters(bool[] mask, int w, int h, int minWidth, int minHeight, int maxGap) @@ -358,21 +393,29 @@ public class ScreenReader : IScreenReader public void SetLootBaseline(Bitmap frame) { } - // Detection parameters // -- Loot detection constants -- private const int CannyLow = 20, CannyHigh = 80; - // Shape constraints - private const int LabelMinW = 80, LabelMaxW = 500; + // Shape constraints (passes 1 & 2) + private const int LabelMinW = 100, LabelMaxW = 500; private const int LabelMinH = 15, LabelMaxH = 100; private const double LabelMinAspect = 1.3, LabelMaxAspect = 10.0; - // Strict pass: well-formed rectangle contours - private const double MinRectangularity = 0.5; - private const float StrictMinBS = 200f; - private const float StrictMinEdgeDensity = 25f; - // Relaxed pass: any contour bbox in play area (catches VFX-broken borders) + // Pass 1: strict (well-formed bordered rectangles) + private const double MinRectangularity = 0.7; + private const double StrictMinBS = 255; + // Pass 2: relaxed (play-area contours, VFX-tolerant) private const int RelaxedMinW = 100; - private const float RelaxedMinBS = 250f; - private const float RelaxedMinEdgeDensity = 25f; + private const double RelaxedMinBS = 265; + private const double RelaxedBrightPctThreshold = 8; + private const double RelaxedBgDarkPctThreshold = 50; + private const double MaxGreenPct = 5; + // Pass 3: yellow text clusters (borderless labels) + private const int YellowHueMin = 10, YellowHueMax = 35; + private const int YellowMinSat = 120, YellowMinVal = 120; + private const double YellowTextPctThreshold = 25; + private const int TextClusterMinWidth = 100; + private const double TextClusterMinAspect = 1.5; + private const double TextClusterContainmentThreshold = 0.5; + // Play area bounds private const double UiMarginTop = 0.08; private const double UiMarginBottom = 0.82; // Post-processing @@ -381,9 +424,10 @@ public class ScreenReader : IScreenReader private const double NmsIouThresh = 0.4; /// - /// Two-pass loot label detection: - /// 1. Strict: polygon-approximated rectangle contours (high precision) - /// 2. Relaxed: any contour bbox in play area (catches VFX-broken borders) + /// Three-pass loot label detection: + /// 1. Strict: polygon-approximated rectangle contours (bordered labels) + /// 2. Relaxed: contour bbox with label-like content OR bright text on dark background + /// 3. Yellow text clusters: morphological detection of gold/yellow text without background box /// Results merged, horizontal fragments joined, then NMS. /// public List DetectLootLabels(Bitmap reference, Bitmap current) @@ -402,57 +446,59 @@ public class ScreenReader : IScreenReader using var hsv = new Mat(); Cv2.CvtColor(mat, hsv, ColorConversionCodes.BGR2HSV); - // Edge detection + // Split HSV channels once for reuse + Cv2.Split(hsv, out Mat[] hsvChannels); + using var hChan = hsvChannels[0]; + using var sChan = hsvChannels[1]; + using var vChan = hsvChannels[2]; + + // ── Passes 1 & 2: Edge-based detection ── using var edges = new Mat(); Cv2.Canny(gray, edges, CannyLow, CannyHigh); - using var dilateKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3)); - using var dilated = new Mat(); - Cv2.Dilate(edges, dilated, dilateKernel, iterations: 1); + Cv2.Dilate(edges, edges, dilateKernel, iterations: 1); - Cv2.FindContours(dilated, out var contours, out _, RetrievalModes.Tree, ContourApproximationModes.ApproxSimple); + Cv2.FindContours(edges, out var contours, out _, RetrievalModes.Tree, ContourApproximationModes.ApproxSimple); var strict = new List(); var relaxed = new List(); foreach (var contour in contours) { - var box = Cv2.BoundingRect(contour); + var bbox = Cv2.BoundingRect(contour); + int x = bbox.X, y = bbox.Y, w = bbox.Width, h = bbox.Height; // Common shape gate - if (box.Width <= LabelMinW || box.Width >= LabelMaxW) continue; - if (box.Height <= LabelMinH || box.Height >= LabelMaxH) continue; - double aspect = (double)box.Width / Math.Max(box.Height, 1); - if (aspect <= LabelMinAspect || aspect >= LabelMaxAspect) continue; + if (w < LabelMinW || w > LabelMaxW || h < LabelMinH || h > LabelMaxH) continue; + double aspect = (double)w / Math.Max(h, 1); + if (aspect < LabelMinAspect || aspect > LabelMaxAspect) continue; - // Content metrics - using var roiHsv = new Mat(hsv, box); - var meanHsv = Cv2.Mean(roiHsv); - float meanVal = (float)meanHsv[2]; - float meanSat = (float)meanHsv[1]; - float bs = meanVal + meanSat; - - using var roiGray = new Mat(gray, box); - using var roiEdges = new Mat(); - Cv2.Canny(roiGray, roiEdges, 50, 150); - float ed = (float)Cv2.Mean(roiEdges)[0]; - - // Strict pass: well-formed polygon (4-8 vertices) - var peri = Cv2.ArcLength(contour, true); - var approx = Cv2.ApproxPolyDP(contour, 0.02 * peri, true); + // Content metrics (mean brightness + saturation) + using var roiV = new Mat(vChan, bbox); + using var roiS = new Mat(sChan, bbox); + double meanVal = Cv2.Mean(roiV).Val0; + double meanSat = Cv2.Mean(roiS).Val0; + double bs = meanVal + meanSat; + // Pass 1: strict – well-formed polygon (4-8 vertices) + var approx = Cv2.ApproxPolyDP(contour, Cv2.ArcLength(contour, true) * 0.02, true); if (approx.Length >= 4 && approx.Length <= 8) { double contourArea = Cv2.ContourArea(approx); - double rect = contourArea / Math.Max(box.Width * box.Height, 1); - if (rect >= MinRectangularity && bs >= StrictMinBS && ed >= StrictMinEdgeDensity) - strict.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat)); + double rectangularity = contourArea / Math.Max(w * h, 1); + if (rectangularity >= MinRectangularity && bs >= StrictMinBS) + strict.Add(new LabelCandidate(x, y, w, h, (float)meanVal, (float)meanSat)); } - // Relaxed pass: any contour bbox in play area - bool inPlay = box.Y > playTop && box.Y + box.Height < playBot; - if (inPlay && box.Width >= RelaxedMinW && bs >= RelaxedMinBS && ed >= RelaxedMinEdgeDensity) - relaxed.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat)); + // Pass 2: relaxed – play area, bs OR bright-on-dark + bool inPlay = y > playTop && (y + h) < playBot; + if (!inPlay || w < RelaxedMinW) continue; + + bool passesBs = bs >= RelaxedMinBS; + bool passesTextOnDark = !passesBs && CheckBrightTextOnDark(mat, vChan, sChan, bbox); + + if (passesBs || passesTextOnDark) + relaxed.Add(new LabelCandidate(x, y, w, h, (float)meanVal, (float)meanSat)); } // Merge strict + relaxed (strict wins on overlap) @@ -466,6 +512,14 @@ public class ScreenReader : IScreenReader // Join horizontal fragments merged = MergeHorizontal(merged, MergeGap, MergeYTolerance); + // ── Pass 3: Yellow text cluster detection (borderless labels) ── + var textClusters = DetectYellowTextClusters(mat, hChan, sChan, vChan, playTop, playBot); + foreach (var tc in textClusters) + { + if (!ContainedByAny(tc, merged, TextClusterContainmentThreshold)) + merged.Add(tc); + } + // Build LootLabels with color classification var scored = new List<(LootLabel Label, float Score)>(); foreach (var c in merged) @@ -487,7 +541,6 @@ public class ScreenReader : IScreenReader { current.Save("debug_loot_capture.png", System.Drawing.Imaging.ImageFormat.Png); Cv2.ImWrite("debug_loot_edges.png", edges); - Cv2.ImWrite("debug_loot_dilated.png", dilated); using var debugMat = mat.Clone(); foreach (var label in labels) Cv2.Rectangle(debugMat, @@ -500,8 +553,8 @@ public class ScreenReader : IScreenReader Log.Warning(ex, "Failed to save debug images"); } - Log.Information("DetectLootLabels: strict={Strict} relaxed={Relaxed} merged={Merged} final={Final}", - strict.Count, relaxed.Count, merged.Count, labels.Count); + Log.Information("DetectLootLabels: strict={Strict} relaxed={Relaxed} yellow={Yellow} final={Final}", + strict.Count, relaxed.Count, textClusters.Count, labels.Count); foreach (var label in labels) Log.Information(" Label ({X},{Y}) {W}x{H} color=({R},{G},{B}) tier={Tier}", label.CenterX - label.Width / 2, label.CenterY - label.Height / 2, @@ -526,81 +579,271 @@ public class ScreenReader : IScreenReader return ((byte)mean[2], (byte)mean[1], (byte)mean[0]); } + /// + /// Pass 2 helper: verify bright+saturated text on dark background with green fire rejection. + /// + private bool CheckBrightTextOnDark(Mat bgrImage, Mat vChan, Mat sChan, Rect bbox) + { + int area = bbox.Width * bbox.Height; + if (area == 0) return false; + + using var roiV = new Mat(vChan, bbox); + using var roiS = new Mat(sChan, bbox); + + // Bright + saturated pixels (the text) + using var brightMask = new Mat(); + using var satMask = new Mat(); + Cv2.Threshold(roiV, brightMask, 150, 255, ThresholdTypes.Binary); + Cv2.Threshold(roiS, satMask, 100, 255, ThresholdTypes.Binary); + using var brightSat = new Mat(); + Cv2.BitwiseAnd(brightMask, satMask, brightSat); + double brightPct = (double)Cv2.CountNonZero(brightSat) / area * 100; + + if (brightPct < RelaxedBrightPctThreshold) + return false; + + // Background darkness: non-text pixels should be dark + using var textMask = new Mat(); + using var bgMask = new Mat(); + Cv2.Threshold(roiV, textMask, 120, 255, ThresholdTypes.Binary); + using var textSatMask = new Mat(); + Cv2.Threshold(roiS, textSatMask, 100, 255, ThresholdTypes.Binary); + Cv2.BitwiseAnd(textMask, textSatMask, textMask); + Cv2.BitwiseNot(textMask, bgMask); + + int bgCount = Cv2.CountNonZero(bgMask); + if (bgCount == 0) return false; + + using var bgV = new Mat(); + roiV.CopyTo(bgV, bgMask); + using var darkBg = new Mat(); + Cv2.Threshold(bgV, darkBg, 40, 255, ThresholdTypes.BinaryInv); + Cv2.BitwiseAnd(darkBg, bgMask, darkBg); + double bgDarkPct = (double)Cv2.CountNonZero(darkBg) / bgCount * 100; + + if (bgDarkPct < RelaxedBgDarkPctThreshold) + return false; + + // Green fire rejection + return !IsGreenDominant(bgrImage, bbox, area); + } + + /// + /// Pass 3: detect gold/yellow text clusters without background boxes (normal rarity items). + /// Uses HSV thresholding, green fire subtraction, and morphological grouping. + /// + private List DetectYellowTextClusters( + Mat bgrImage, Mat hChan, Mat sChan, Mat vChan, + int playTop, int playBot) + { + var results = new List(); + + // Build yellow text mask: H:10-35, S>120, V>120 + using var hMin = new Mat(); + using var hMax = new Mat(); + using var sThresh = new Mat(); + using var vThresh = new Mat(); + Cv2.Threshold(hChan, hMin, YellowHueMin - 1, 255, ThresholdTypes.Binary); + Cv2.Threshold(hChan, hMax, YellowHueMax, 255, ThresholdTypes.BinaryInv); + Cv2.Threshold(sChan, sThresh, YellowMinSat - 1, 255, ThresholdTypes.Binary); + Cv2.Threshold(vChan, vThresh, YellowMinVal - 1, 255, ThresholdTypes.Binary); + + using var yellowMask = new Mat(); + Cv2.BitwiseAnd(hMin, hMax, yellowMask); + Cv2.BitwiseAnd(yellowMask, sThresh, yellowMask); + Cv2.BitwiseAnd(yellowMask, vThresh, yellowMask); + + // Subtract green fire pixels + SubtractGreenFire(bgrImage, yellowMask); + + // Morphological grouping + using var kH = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(25, 1)); + using var dilated = new Mat(); + Cv2.Dilate(yellowMask, dilated, kH, iterations: 1); + + using var kV = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(1, 8)); + using var closed = new Mat(); + Cv2.MorphologyEx(dilated, closed, MorphTypes.Close, kV); + + using var kO = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(40, 5)); + using var cleaned = new Mat(); + Cv2.MorphologyEx(closed, cleaned, MorphTypes.Open, kO); + + // Find and filter text clusters + Cv2.FindContours(cleaned, out var textContours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple); + + foreach (var contour in textContours) + { + var bbox = Cv2.BoundingRect(contour); + int x = bbox.X, y = bbox.Y, w = bbox.Width, h = bbox.Height; + + if (w < TextClusterMinWidth || h < LabelMinH || h > 120) continue; + double aspect = (double)w / Math.Max(h, 1); + if (aspect < TextClusterMinAspect) continue; + + bool inPlay = y > playTop && (y + h) < playBot; + if (!inPlay) continue; + + // Verify yellow text density in the bounding box + using var roiYellow = new Mat(yellowMask, bbox); + double yellowPct = (double)Cv2.CountNonZero(roiYellow) / (w * h) * 100; + + if (yellowPct >= YellowTextPctThreshold) + results.Add(new LabelCandidate(x, y, w, h, (float)(yellowPct * 10), 0)); + } + + return results; + } + + /// Check if region is green-fire dominant (G > R+15, G > B+15, G > 80). + private static bool IsGreenDominant(Mat bgrImage, Rect bbox, int area) + { + using var roiBgr = new Mat(bgrImage, bbox); + Cv2.Split(roiBgr, out Mat[] bgr); + try + { + using var gMinusR = new Mat(); + using var gMinusB = new Mat(); + Cv2.Subtract(bgr[1], bgr[2], gMinusR); + Cv2.Subtract(bgr[1], bgr[0], gMinusB); + using var thR = new Mat(); + using var thB = new Mat(); + using var thG = new Mat(); + Cv2.Threshold(gMinusR, thR, 15, 255, ThresholdTypes.Binary); + Cv2.Threshold(gMinusB, thB, 15, 255, ThresholdTypes.Binary); + Cv2.Threshold(bgr[1], thG, 80, 255, ThresholdTypes.Binary); + using var greenMask = new Mat(); + Cv2.BitwiseAnd(thR, thB, greenMask); + Cv2.BitwiseAnd(greenMask, thG, greenMask); + double greenPct = (double)Cv2.CountNonZero(greenMask) / area * 100; + return greenPct >= MaxGreenPct; + } + finally + { + foreach (var ch in bgr) ch.Dispose(); + } + } + + /// Zero out green-fire pixels from a mask in-place. + private static void SubtractGreenFire(Mat bgrImage, Mat mask) + { + Cv2.Split(bgrImage, out Mat[] bgr); + try + { + using var gMinusR = new Mat(); + using var gMinusB = new Mat(); + Cv2.Subtract(bgr[1], bgr[2], gMinusR); + Cv2.Subtract(bgr[1], bgr[0], gMinusB); + using var thR = new Mat(); + using var thB = new Mat(); + using var thG = new Mat(); + Cv2.Threshold(gMinusR, thR, 15, 255, ThresholdTypes.Binary); + Cv2.Threshold(gMinusB, thB, 15, 255, ThresholdTypes.Binary); + Cv2.Threshold(bgr[1], thG, 80, 255, ThresholdTypes.Binary); + using var greenFire = new Mat(); + Cv2.BitwiseAnd(thR, thB, greenFire); + Cv2.BitwiseAnd(greenFire, thG, greenFire); + using var notGreen = new Mat(); + Cv2.BitwiseNot(greenFire, notGreen); + Cv2.BitwiseAnd(mask, notGreen, mask); + } + finally + { + foreach (var ch in bgr) ch.Dispose(); + } + } + private static bool OverlapsAny(LabelCandidate label, List others, double iouThresh) { foreach (var o in others) { - int ix1 = Math.Max(label.X, o.X), iy1 = Math.Max(label.Y, o.Y); - int ix2 = Math.Min(label.X + label.W, o.X + o.W); - int iy2 = Math.Min(label.Y + label.H, o.Y + o.H); - int inter = Math.Max(0, ix2 - ix1) * Math.Max(0, iy2 - iy1); - int union = label.W * label.H + o.W * o.H - inter; - if (inter / (double)Math.Max(union, 1) > iouThresh) + if (ComputeIoU(label, o) > iouThresh) return true; } return false; } - /// - /// Merge labels that sit side-by-side on the same line. - /// + /// Check if label is mostly contained inside any existing detection. + private static bool ContainedByAny(LabelCandidate label, List others, double containThresh) + { + int labelArea = label.W * label.H; + if (labelArea == 0) return true; + + foreach (var o in others) + { + int xx1 = Math.Max(label.X, o.X); + int yy1 = Math.Max(label.Y, o.Y); + int xx2 = Math.Min(label.X + label.W, o.X + o.W); + int yy2 = Math.Min(label.Y + label.H, o.Y + o.H); + int inter = Math.Max(0, xx2 - xx1) * Math.Max(0, yy2 - yy1); + if ((double)inter / labelArea > containThresh) + return true; + } + return false; + } + + private static double ComputeIoU(LabelCandidate a, LabelCandidate b) + { + int xx1 = Math.Max(a.X, b.X); + int yy1 = Math.Max(a.Y, b.Y); + int xx2 = Math.Min(a.X + a.W, b.X + b.W); + int yy2 = Math.Min(a.Y + a.H, b.Y + b.H); + int inter = Math.Max(0, xx2 - xx1) * Math.Max(0, yy2 - yy1); + int union = a.W * a.H + b.W * b.H - inter; + return (double)inter / Math.Max(union, 1); + } + private static List MergeHorizontal(List labels, int gap, int yTol) { if (labels.Count < 2) return labels; var used = new bool[labels.Count]; - var indices = Enumerable.Range(0, labels.Count) + var sorted = Enumerable.Range(0, labels.Count) .OrderBy(i => labels[i].Y).ThenBy(i => labels[i].X).ToList(); var result = new List(); - for (int ii = 0; ii < indices.Count; ii++) + foreach (int i in sorted) { - int i = indices[ii]; if (used[i]) continue; used[i] = true; - var a = labels[i]; - int gx1 = a.X, gy1 = a.Y, gx2 = a.X + a.W, gy2 = a.Y + a.H; - double briArea = a.MeanBrightness * a.W * a.H; - double satArea = a.MeanSaturation * a.W * a.H; - int totalArea = a.W * a.H; + int gx1 = labels[i].X, gy1 = labels[i].Y; + int gx2 = gx1 + labels[i].W, gy2 = gy1 + labels[i].H; + double wBri = labels[i].MeanBrightness * labels[i].W * labels[i].H; + double wSat = labels[i].MeanSaturation * labels[i].W * labels[i].H; + double area = labels[i].W * labels[i].H; bool changed = true; while (changed) { changed = false; - for (int jj = 0; jj < indices.Count; jj++) + foreach (int j in sorted) { - int j = indices[jj]; if (used[j]) continue; var b = labels[j]; - double cyA = (gy1 + gy2) / 2.0; double cyB = b.Y + b.H / 2.0; if (Math.Abs(cyA - cyB) > yTol) continue; - int hGap = Math.Max(b.X - gx2, gx1 - (b.X + b.W)); if (hGap > gap) continue; - int bArea = b.W * b.H; + double bArea = b.W * b.H; gx1 = Math.Min(gx1, b.X); gy1 = Math.Min(gy1, b.Y); gx2 = Math.Max(gx2, b.X + b.W); gy2 = Math.Max(gy2, b.Y + b.H); - briArea += b.MeanBrightness * bArea; - satArea += b.MeanSaturation * bArea; - totalArea += bArea; + wBri += b.MeanBrightness * bArea; + wSat += b.MeanSaturation * bArea; + area += bArea; used[j] = true; changed = true; } } int w = gx2 - gx1, h = gy2 - gy1; - float bri = (float)(briArea / Math.Max(totalArea, 1)); - float sat = (float)(satArea / Math.Max(totalArea, 1)); - result.Add(new LabelCandidate(gx1, gy1, w, h, bri, sat)); + result.Add(new LabelCandidate(gx1, gy1, w, h, + (float)(wBri / Math.Max(area, 1)), (float)(wSat / Math.Max(area, 1)))); } - return result; } diff --git a/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs b/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs index d03d1a2..2c15277 100644 --- a/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs +++ b/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs @@ -71,6 +71,7 @@ public sealed class D2dOverlay using var ctx = new D2dRenderContext(hwnd, Width, Height); _layers.Add(new D2dEnemyBoxLayer(ctx)); + _layers.Add(new D2dLootLabelLayer(ctx)); _layers.Add(new D2dHudInfoLayer()); _layers.Add(new D2dDebugTextLayer()); @@ -182,6 +183,9 @@ public sealed class D2dOverlay { var detection = _bot.EnemyDetector.Latest; var bossDetection = _bot.BossDetector.Latest; + + var showLoot = _bot.LootDebugDetector.Enabled; + return new OverlayState( Enemies: detection.Enemies, Bosses: bossDetection.Bosses, @@ -192,6 +196,8 @@ public sealed class D2dOverlay NavPosition: _bot.Navigation.WorldPosition, IsExploring: _bot.Navigation.IsExploring, ShowHudDebug: _bot.Store.Settings.ShowHudDebug, + ShowLootDebug: showLoot, + LootLabels: _bot.LootDebugDetector.Latest, Fps: fps, Timing: timing); } diff --git a/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs b/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs index 62b579a..07c2fd2 100644 --- a/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs +++ b/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs @@ -13,6 +13,8 @@ public record OverlayState( MapPosition NavPosition, bool IsExploring, bool ShowHudDebug, + bool ShowLootDebug, + IReadOnlyList LootLabels, double Fps, RenderTiming? Timing); diff --git a/src/Poe2Trade.Ui/Overlay/Layers/D2dLootLabelLayer.cs b/src/Poe2Trade.Ui/Overlay/Layers/D2dLootLabelLayer.cs new file mode 100644 index 0000000..f6119fa --- /dev/null +++ b/src/Poe2Trade.Ui/Overlay/Layers/D2dLootLabelLayer.cs @@ -0,0 +1,61 @@ +using System.Drawing; +using Vortice.Direct2D1; +using Vortice.DirectWrite; +using Vortice.Mathematics; + +namespace Poe2Trade.Ui.Overlay.Layers; + +internal sealed class D2dLootLabelLayer : ID2dOverlayLayer, IDisposable +{ + private readonly D2dRenderContext _ctx; + private readonly Dictionary _labelCache = new(); + private ID2D1SolidColorBrush? _fillBrush; + + public D2dLootLabelLayer(D2dRenderContext ctx) + { + _ctx = ctx; + _fillBrush = ctx.RenderTarget.CreateSolidColorBrush(new Color4(0f, 1f, 0f, 0.15f)); + } + + public void Draw(D2dRenderContext ctx, OverlayState state) + { + if (!state.ShowLootDebug || state.LootLabels.Count == 0) return; + + var rt = ctx.RenderTarget; + + foreach (var label in state.LootLabels) + { + float x = label.CenterX - label.Width / 2f; + float y = label.CenterY - label.Height / 2f; + float w = label.Width; + float h = label.Height; + var rect = new RectangleF(x, y, w, h); + + rt.FillRectangle(rect, _fillBrush!); + rt.DrawRectangle(rect, ctx.Green, 3f); + + // Label: "tier (R,G,B)" + var key = $"{label.Tier} ({label.AvgR},{label.AvgG},{label.AvgB})"; + if (!_labelCache.TryGetValue(key, out var layout)) + { + layout = _ctx.CreateTextLayout(key, _ctx.LabelFormat); + _labelCache[key] = layout; + } + + var m = layout.Metrics; + var labelX = x; + var labelY = y - m.Height - 2; + + rt.FillRectangle( + new RectangleF(labelX - 1, labelY - 1, m.Width + 2, m.Height + 2), + ctx.LabelBgBrush); + + rt.DrawTextLayout(new System.Numerics.Vector2(labelX, labelY), layout, ctx.Green); + } + } + + public void Dispose() + { + foreach (var l in _labelCache.Values) l?.Dispose(); + } +} diff --git a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs index 6e350ec..85e1a8c 100644 --- a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs @@ -20,6 +20,12 @@ public partial class DebugViewModel : ObservableObject [ObservableProperty] private string _selectedGridLayout = "inventory"; [ObservableProperty] private decimal? _clickX; [ObservableProperty] private decimal? _clickY; + [ObservableProperty] private bool _showLootDebug; + + partial void OnShowLootDebugChanged(bool value) + { + _bot.LootDebugDetector.Enabled = value; + } public string[] GridLayoutNames { get; } = [ diff --git a/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs b/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs index 1798a1e..292e840 100644 --- a/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs @@ -24,6 +24,7 @@ public partial class MappingViewModel : ObservableObject, IDisposable [ObservableProperty] private string _invitationTabPath = ""; [ObservableProperty] private string _lootTabPath = ""; [ObservableProperty] private decimal? _invitationCount = 15; + [ObservableProperty] private decimal? _runCount = 15; public static MapType[] MapTypes { get; } = [MapType.TrialOfChaos, MapType.Temple, MapType.Endgame, MapType.Kulemak]; public ObservableCollection StashTabPaths { get; } = []; @@ -45,6 +46,7 @@ public partial class MappingViewModel : ObservableObject, IDisposable _invitationTabPath = bot.Config.Kulemak.InvitationTabPath; _lootTabPath = bot.Config.Kulemak.LootTabPath; _invitationCount = bot.Config.Kulemak.InvitationCount; + _runCount = bot.Config.Kulemak.RunCount; LoadStashTabPaths(); _bot.EnemyDetector.DetectionUpdated += OnDetectionUpdated; @@ -80,6 +82,11 @@ public partial class MappingViewModel : ObservableObject, IDisposable _bot.Store.UpdateSettings(s => s.Kulemak.InvitationCount = (int)(value ?? 15)); } + partial void OnRunCountChanged(decimal? value) + { + _bot.Store.UpdateSettings(s => s.Kulemak.RunCount = (int)(value ?? 15)); + } + private void LoadStashTabPaths() { StashTabPaths.Clear(); diff --git a/src/Poe2Trade.Ui/Views/MainWindow.axaml b/src/Poe2Trade.Ui/Views/MainWindow.axaml index b420fcd..3011849 100644 --- a/src/Poe2Trade.Ui/Views/MainWindow.axaml +++ b/src/Poe2Trade.Ui/Views/MainWindow.axaml @@ -255,10 +255,10 @@ SelectedItem="{Binding LootTabPath}" /> - - + @@ -334,6 +334,17 @@ + + + + + + + +