boss ready

This commit is contained in:
Boki 2026-02-21 15:44:22 -05:00
parent 89c3a0a077
commit 64a6ab694b
21 changed files with 857 additions and 249 deletions

BIN
assets/resurrect.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.4 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.3 MiB

After

Width:  |  Height:  |  Size: 7.6 MiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.8 MiB

After

Width:  |  Height:  |  Size: 6.4 MiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 297 KiB

After

Width:  |  Height:  |  Size: 323 KiB

Before After
Before After

Binary file not shown.

Before

Width:  |  Height:  |  Size: 639 KiB

After

Width:  |  Height:  |  Size: 338 KiB

Before After
Before After

View file

@ -18,13 +18,16 @@ public class BossRunExecutor : GameExecutor
private static readonly string CathedralWellTemplate = Path.Combine("assets", "black-cathedral-well.png"); private static readonly string CathedralWellTemplate = Path.Combine("assets", "black-cathedral-well.png");
private static readonly string ReturnTheRingTemplate = Path.Combine("assets", "return-the-ring.png"); private static readonly string ReturnTheRingTemplate = Path.Combine("assets", "return-the-ring.png");
private static readonly string BossHealthbarTemplate = Path.Combine("assets", "unique-boss-present.png"); private static readonly string BossHealthbarTemplate = Path.Combine("assets", "unique-boss-present.png");
private static readonly string BossHealthbarTemplate2 = Path.Combine("assets", "unique-boss-present2.png");
private static readonly string BossHealthbarTemplate3 = Path.Combine("assets", "unique-boss-present3.png");
private static readonly string NewInstanceTemplate = Path.Combine("assets", "new.png"); private static readonly string NewInstanceTemplate = Path.Combine("assets", "new.png");
private static readonly string ResurrectTemplate = Path.Combine("assets", "resurrect.png");
private BossRunState _state = BossRunState.Idle; private BossRunState _state = BossRunState.Idle;
private readonly IClientLogWatcher _logWatcher; private readonly IClientLogWatcher _logWatcher;
private readonly BossDetector _bossDetector; private readonly BossDetector _bossDetector;
private readonly HudReader _hudReader;
private readonly NavigationExecutor _nav; private readonly NavigationExecutor _nav;
private readonly CombatManager _combat;
public event Action<BossRunState>? StateChanged; public event Action<BossRunState>? StateChanged;
@ -35,7 +38,7 @@ public class BossRunExecutor : GameExecutor
{ {
_logWatcher = logWatcher; _logWatcher = logWatcher;
_bossDetector = bossDetector; _bossDetector = bossDetector;
_hudReader = hudReader; _combat = new CombatManager(game, hudReader, new FlaskManager(game, hudReader));
_nav = nav; _nav = nav;
} }
@ -56,8 +59,10 @@ public class BossRunExecutor : GameExecutor
public async Task RunBossLoop() public async Task RunBossLoop()
{ {
_stopped = false; _stopped = false;
Log.Information("Starting boss run loop ({Count} invitations)", _config.Kulemak.InvitationCount); var runCount = _config.Kulemak.RunCount;
Log.Information("Starting boss run loop ({Count} runs)", runCount);
// First run: deposit inventory and grab 1 invitation
if (!await Prepare()) if (!await Prepare())
{ {
SetState(BossRunState.Failed); SetState(BossRunState.Failed);
@ -67,11 +72,11 @@ public class BossRunExecutor : GameExecutor
} }
var completed = 0; var completed = 0;
for (var i = 0; i < _config.Kulemak.InvitationCount; i++) for (var i = 0; i < runCount; i++)
{ {
if (_stopped) break; if (_stopped) break;
Log.Information("=== Boss run {N}/{Total} ===", i + 1, _config.Kulemak.InvitationCount); Log.Information("=== Boss run {N}/{Total} ===", i + 1, runCount);
if (!await TravelToZone()) if (!await TravelToZone())
{ {
@ -113,13 +118,14 @@ public class BossRunExecutor : GameExecutor
} }
if (_stopped) break; if (_stopped) break;
await StoreLoot(); bool isLastRun = i == runCount - 1 || _stopped;
await StoreLoot(grabInvitation: !isLastRun);
completed++; completed++;
if (_stopped) break; if (_stopped) break;
} }
Log.Information("Boss run loop finished: {Completed}/{Total} runs completed", completed, _config.Kulemak.InvitationCount); Log.Information("Boss run loop finished: {Completed}/{Total} runs completed", completed, runCount);
SetState(BossRunState.Complete); SetState(BossRunState.Complete);
await Helpers.Sleep(1000); await Helpers.Sleep(1000);
SetState(BossRunState.Idle); SetState(BossRunState.Idle);
@ -186,7 +192,7 @@ public class BossRunExecutor : GameExecutor
(false, false) => "stash12", (false, false) => "stash12",
}; };
await _inventory.GrabItemsFromStash(layoutName, _config.Kulemak.InvitationCount, InvitationTemplate); await _inventory.GrabItemsFromStash(layoutName, 1, InvitationTemplate);
} }
else else
{ {
@ -310,7 +316,7 @@ public class BossRunExecutor : GameExecutor
Log.Information("Fight phase starting"); Log.Information("Fight phase starting");
// Wait for arena to settle // Wait for arena to settle
await Helpers.Sleep(5500); await Helpers.Sleep(4500);
if (_stopped) return; if (_stopped) return;
// Find and click the cathedral door // Find and click the cathedral door
@ -336,7 +342,7 @@ public class BossRunExecutor : GameExecutor
const double wellWorldX = -496; const double wellWorldX = -496;
const double wellWorldY = -378; const double wellWorldY = -378;
await WalkToWorldPosition(fightWorldX, fightWorldY); await WalkToWorldPosition(fightWorldX, fightWorldY, cancelWhen: IsBossAlive);
if (_stopped) return; if (_stopped) return;
// 3x fight-then-well loop // 3x fight-then-well loop
@ -356,14 +362,17 @@ public class BossRunExecutor : GameExecutor
Log.Information("Fight area updated to ({X:F0},{Y:F0})", fightWorldX, fightWorldY); Log.Information("Fight area updated to ({X:F0},{Y:F0})", fightWorldX, fightWorldY);
} }
// Wait for death animation before looking for well
await Helpers.Sleep(3000);
// Walk to well and click the closest match to screen center // Walk to well and click the closest match to screen center
Log.Information("Phase {Phase} done, walking to well", phase); Log.Information("Phase {Phase} done, walking to well", phase);
await WalkToWorldPosition(wellWorldX, wellWorldY); await WalkToWorldPosition(wellWorldX, wellWorldY);
await ClickClosestTemplateToCenter(CathedralWellTemplate); await ClickClosestTemplateToCenter(CathedralWellTemplate);
await Helpers.Sleep(2000); await Helpers.Sleep(200);
// Walk back to fight position for next phase // Walk back to fight position for next phase
await WalkToWorldPosition(fightWorldX, fightWorldY); await WalkToWorldPosition(fightWorldX, fightWorldY, cancelWhen: IsBossAlive);
} }
// 4th fight - no well after // 4th fight - no well after
@ -372,7 +381,10 @@ public class BossRunExecutor : GameExecutor
await AttackBossUntilGone(); await AttackBossUntilGone();
if (_stopped) return; if (_stopped) return;
// Return the ring // Walk to ring area and return it
await WalkToWorldPosition(-440, -330);
if (_stopped) return;
Log.Information("Looking for Return the Ring..."); Log.Information("Looking for Return the Ring...");
var ring = await _screen.TemplateMatch(ReturnTheRingTemplate); var ring = await _screen.TemplateMatch(ReturnTheRingTemplate);
if (ring == null) if (ring == null)
@ -419,13 +431,39 @@ public class BossRunExecutor : GameExecutor
/// <summary> /// <summary>
/// Check top-of-screen region for the unique boss healthbar frame. /// Check top-of-screen region for the unique boss healthbar frame.
/// Uses lower threshold (0.5) to tolerate partial overlay from YOLO bounding boxes. /// Healthbar spans (750,16) to (1818,112). Uses lower threshold (0.5) to tolerate YOLO overlay.
/// </summary> /// </summary>
private async Task<bool> IsBossAlive() private async Task<bool> IsBossAlive()
{ {
var topRegion = new Region(800, 0, 960, 120); var topRegion = new Region(750, 16, 1068, 96);
var matches = await _screen.TemplateMatchAll(BossHealthbarTemplate, topRegion, threshold: 0.5);
return matches.Count > 0; // Check all three healthbar templates — boss is alive if ANY matches
var m1 = await _screen.TemplateMatchAll(BossHealthbarTemplate, topRegion, threshold: 0.5);
if (m1.Count > 0) return true;
var m2 = await _screen.TemplateMatchAll(BossHealthbarTemplate2, topRegion, threshold: 0.5);
if (m2.Count > 0) return true;
var m3 = await _screen.TemplateMatchAll(BossHealthbarTemplate3, topRegion, threshold: 0.5);
if (m3.Count > 0) return true;
return false;
}
/// <summary>
/// Check for the "Resurrect at Checkpoint" button — means we died.
/// If found, click it, wait for respawn, and return true.
/// </summary>
private async Task<bool> CheckDeath()
{
var match = await _screen.TemplateMatch(ResurrectTemplate);
if (match == null) return false;
Log.Warning("Death detected! Clicking resurrect at ({X},{Y})", match.X, match.Y);
await _combat.ReleaseAll();
await _game.LeftClickAt(match.X, match.Y);
await Helpers.Sleep(3000); // wait for respawn + loading
return true;
} }
/// <summary> /// <summary>
@ -435,13 +473,13 @@ public class BossRunExecutor : GameExecutor
{ {
Log.Information("Waiting for boss healthbar to appear..."); Log.Information("Waiting for boss healthbar to appear...");
var sw = Stopwatch.StartNew(); var sw = Stopwatch.StartNew();
var atkX = 1280;
const int screenCx = 1280; var atkY = 720;
const int screenCy = 720;
while (sw.ElapsedMilliseconds < timeoutMs) while (sw.ElapsedMilliseconds < timeoutMs)
{ {
if (_stopped) return false; if (_stopped) return false;
if (await CheckDeath()) continue;
if (await IsBossAlive()) if (await IsBossAlive())
{ {
@ -449,19 +487,14 @@ public class BossRunExecutor : GameExecutor
return true; return true;
} }
// Attack at center while waiting — boss often spawns on top of us // Attack toward YOLO-detected boss if available, otherwise last known position
var targetX = screenCx + Rng.Next(-30, 31); var snapshot = _bossDetector.Latest;
var targetY = screenCy + Rng.Next(-30, 31); if (snapshot.Bosses.Count > 0)
await _game.MoveMouseFast(targetX, targetY); {
atkX = snapshot.Bosses[0].Cx;
_game.LeftMouseDown(); atkY = snapshot.Bosses[0].Cy;
await Helpers.Sleep(Rng.Next(20, 35)); }
_game.LeftMouseUp(); await _combat.Tick(atkX, atkY);
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(50, 80));
} }
Log.Warning("WaitForBossSpawn timed out after {Ms}ms", timeoutMs); Log.Warning("WaitForBossSpawn timed out after {Ms}ms", timeoutMs);
@ -490,11 +523,9 @@ public class BossRunExecutor : GameExecutor
await Helpers.Sleep(200); await Helpers.Sleep(200);
var sw = Stopwatch.StartNew(); var sw = Stopwatch.StartNew();
var consecutiveMisses = 0;
var lastCheckMs = 0L; var lastCheckMs = 0L;
var holding = false;
var manaStableCount = 0;
await _combat.Reset();
Log.Information("Boss is alive, engaging"); Log.Information("Boss is alive, engaging");
try try
@ -520,82 +551,22 @@ public class BossRunExecutor : GameExecutor
lastBossWorldPos = (bossWorldX, bossWorldY); lastBossWorldPos = (bossWorldX, bossWorldY);
} }
// Re-check healthbar every ~1.5s, need 5 consecutive misses (~7.5s) to confirm phase end // Re-check healthbar every ~0.5s, first miss = phase over
if (sw.ElapsedMilliseconds - lastCheckMs > 1500) if (sw.ElapsedMilliseconds - lastCheckMs > 500)
{ {
if (await CheckDeath()) continue;
var bossAlive = await IsBossAlive(); var bossAlive = await IsBossAlive();
lastCheckMs = sw.ElapsedMilliseconds; lastCheckMs = sw.ElapsedMilliseconds;
if (!bossAlive) if (!bossAlive)
{ {
consecutiveMisses++; Log.Information("Healthbar not found, boss phase over after {Ms}ms", sw.ElapsedMilliseconds);
Log.Information("Healthbar not found ({Misses}/5 consecutive misses, {Ms}ms elapsed)",
consecutiveMisses, sw.ElapsedMilliseconds);
if (consecutiveMisses >= 5)
{
Log.Information("Boss phase over after {Ms}ms", sw.ElapsedMilliseconds);
return lastBossWorldPos; return lastBossWorldPos;
} }
} }
else
{
if (consecutiveMisses > 0)
Log.Information("Healthbar found again, resetting miss count (was {Misses})", consecutiveMisses);
consecutiveMisses = 0;
}
}
// Check mana to decide hold vs click await _combat.Tick(atkX, atkY);
var mana = _hudReader.Current.ManaPct;
if (!holding)
{
// Ramp up: click cycle until mana is stable above 80%
if (mana >= 0.80f)
manaStableCount++;
else
manaStableCount = 0;
var targetX = atkX + Rng.Next(-30, 31);
var targetY = atkY + Rng.Next(-30, 31);
await _game.MoveMouseFast(targetX, targetY);
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.LeftMouseUp();
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(50, 80));
// After 5 consecutive checks above 80%, switch to hold
if (manaStableCount >= 5)
{
Log.Information("Mana stable at {Mana:P0}, switching to hold attack", mana);
_game.LeftMouseDown();
_game.RightMouseDown();
holding = true;
}
}
else
{
// Holding: nudge mouse toward boss and monitor mana
var targetX = atkX + Rng.Next(-30, 31);
var targetY = atkY + Rng.Next(-30, 31);
await _game.MoveMouseFast(targetX, targetY);
if (mana < 0.30f)
{
Log.Information("Mana dropped to {Mana:P0}, releasing to recover", mana);
_game.LeftMouseUp();
_game.RightMouseUp();
holding = false;
manaStableCount = 0;
}
await Helpers.Sleep(Rng.Next(80, 120));
}
} }
Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs); Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs);
@ -603,34 +574,26 @@ public class BossRunExecutor : GameExecutor
} }
finally finally
{ {
if (holding) await _combat.ReleaseAll();
{
_game.LeftMouseUp();
_game.RightMouseUp();
}
} }
} }
private async Task AttackAtPosition(int x, int y, int durationMs) private async Task AttackAtPosition(int x, int y, int durationMs)
{ {
await _combat.Reset();
var sw = Stopwatch.StartNew(); var sw = Stopwatch.StartNew();
try
{
while (sw.ElapsedMilliseconds < durationMs) while (sw.ElapsedMilliseconds < durationMs)
{ {
if (_stopped) return; if (_stopped) return;
if (await CheckDeath()) continue;
var targetX = x + Rng.Next(-20, 21); await _combat.Tick(x, y, jitter: 20);
var targetY = y + Rng.Next(-20, 21); }
await _game.MoveMouseFast(targetX, targetY); }
finally
_game.LeftMouseDown(); {
await Helpers.Sleep(Rng.Next(20, 35)); await _combat.ReleaseAll();
_game.LeftMouseUp();
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(50, 80));
} }
} }
@ -665,7 +628,8 @@ public class BossRunExecutor : GameExecutor
/// <summary> /// <summary>
/// Walk to a world position using WASD keys, checking minimap position each iteration. /// Walk to a world position using WASD keys, checking minimap position each iteration.
/// </summary> /// </summary>
private async Task WalkToWorldPosition(double worldX, double worldY, int timeoutMs = 10000, double arrivalDist = 15) private async Task WalkToWorldPosition(double worldX, double worldY, int timeoutMs = 10000,
double arrivalDist = 15, Func<Task<bool>>? cancelWhen = null)
{ {
Log.Information("Walking to world ({X:F0},{Y:F0})", worldX, worldY); Log.Information("Walking to world ({X:F0},{Y:F0})", worldX, worldY);
@ -682,6 +646,12 @@ public class BossRunExecutor : GameExecutor
{ {
if (_stopped) break; if (_stopped) break;
if (cancelWhen != null && await cancelWhen())
{
Log.Information("Walk cancelled early (cancel condition met)");
break;
}
var pos = _nav.WorldPosition; var pos = _nav.WorldPosition;
var dx = worldX - pos.X; var dx = worldX - pos.X;
var dy = worldY - pos.Y; var dy = worldY - pos.Y;
@ -795,7 +765,7 @@ public class BossRunExecutor : GameExecutor
return true; return true;
} }
private async Task StoreLoot() private async Task StoreLoot(bool grabInvitation = false)
{ {
SetState(BossRunState.StoringLoot); SetState(BossRunState.StoringLoot);
Log.Information("Storing loot"); Log.Information("Storing loot");
@ -812,12 +782,11 @@ public class BossRunExecutor : GameExecutor
} }
await Helpers.Sleep(Delays.PostStashOpen); await Helpers.Sleep(Delays.PostStashOpen);
// Click loot tab // Click loot tab and deposit all inventory items
var (lootTab, lootFolder) = ResolveTabPath(_config.Kulemak.LootTabPath); var (lootTab, lootFolder) = ResolveTabPath(_config.Kulemak.LootTabPath);
if (lootTab != null) if (lootTab != null)
await _inventory.ClickStashTab(lootTab, lootFolder); await _inventory.ClickStashTab(lootTab, lootFolder);
// Deposit all inventory items
var scanResult = await _screen.Grid.Scan("inventory"); var scanResult = await _screen.Grid.Scan("inventory");
if (scanResult.Occupied.Count > 0) if (scanResult.Occupied.Count > 0)
{ {
@ -835,6 +804,25 @@ public class BossRunExecutor : GameExecutor
await Helpers.Sleep(Delays.PostEscape); await Helpers.Sleep(Delays.PostEscape);
} }
// Grab 1 invitation for the next run while stash is still open
if (grabInvitation)
{
var (invTab, invFolder) = ResolveTabPath(_config.Kulemak.InvitationTabPath);
if (invTab != null)
{
await _inventory.ClickStashTab(invTab, invFolder);
var layoutName = (invTab.GridCols == 24, invFolder != null) switch
{
(true, true) => "stash24_folder",
(true, false) => "stash24",
(false, true) => "stash12_folder",
(false, false) => "stash12",
};
await _inventory.GrabItemsFromStash(layoutName, 1, InvitationTemplate);
Log.Information("Grabbed 1 invitation for next run");
}
}
// Close stash // Close stash
await _game.PressEscape(); await _game.PressEscape();
_inventory.ResetStashTabState(); _inventory.ResetStashTabState();

View file

@ -46,6 +46,7 @@ public class BotOrchestrator : IAsyncDisposable
public EnemyDetector EnemyDetector { get; } public EnemyDetector EnemyDetector { get; }
public BossDetector BossDetector { get; } public BossDetector BossDetector { get; }
public FrameSaver FrameSaver { get; } public FrameSaver FrameSaver { get; }
public LootDebugDetector LootDebugDetector { get; }
public BossRunExecutor BossRunExecutor { get; } public BossRunExecutor BossRunExecutor { get; }
private readonly Dictionary<string, ScrapExecutor> _scrapExecutors = new(); private readonly Dictionary<string, ScrapExecutor> _scrapExecutors = new();
@ -77,6 +78,7 @@ public class BotOrchestrator : IAsyncDisposable
EnemyDetector.Enabled = true; EnemyDetector.Enabled = true;
BossDetector = new BossDetector(); BossDetector = new BossDetector();
FrameSaver = new FrameSaver(); FrameSaver = new FrameSaver();
LootDebugDetector = new LootDebugDetector(screen);
// Register on shared pipeline // Register on shared pipeline
pipelineService.Pipeline.AddConsumer(minimapCapture); pipelineService.Pipeline.AddConsumer(minimapCapture);

View file

@ -0,0 +1,154 @@
using System.Diagnostics;
using Poe2Trade.Core;
using Poe2Trade.Game;
using Poe2Trade.Screen;
using Serilog;
namespace Poe2Trade.Bot;
/// <summary>
/// Manages the attack state machine (click → hold) with mana monitoring and flask usage.
/// Call <see cref="Tick"/> each combat loop iteration, <see cref="Reset"/> between phases,
/// and <see cref="ReleaseAll"/> when done.
/// </summary>
public class CombatManager
{
private static readonly Random Rng = new();
// Orbit: cycle W→D→S→A to dodge in a small circle
private static readonly int[] OrbitKeys =
[InputSender.VK.W, InputSender.VK.D, InputSender.VK.S, InputSender.VK.A];
private const int OrbitStepMs = 400; // time per direction
private readonly IGameController _game;
private readonly HudReader _hudReader;
private readonly FlaskManager _flasks;
private bool _holding;
private int _manaStableCount;
private readonly Stopwatch _orbitSw = Stopwatch.StartNew();
private int _orbitIndex = -1;
private long _lastOrbitMs;
public bool IsHolding => _holding;
public CombatManager(IGameController game, HudReader hudReader, FlaskManager flasks)
{
_game = game;
_hudReader = hudReader;
_flasks = flasks;
}
/// <summary>
/// One combat iteration: flask check, mana-based click/hold, mouse jitter toward target.
/// </summary>
public async Task Tick(int x, int y, int jitter = 30)
{
await _flasks.Tick();
await UpdateOrbit();
var mana = _hudReader.Current.ManaPct;
if (!_holding)
{
if (mana >= 0.80f)
_manaStableCount++;
else
_manaStableCount = 0;
var targetX = x + Rng.Next(-jitter, jitter + 1);
var targetY = y + Rng.Next(-jitter, jitter + 1);
await _game.MoveMouseFast(targetX, targetY);
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.LeftMouseUp();
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(50, 80));
if (_manaStableCount >= 5)
{
Log.Information("Mana stable at {Mana:P0}, switching to hold attack", mana);
_game.LeftMouseDown();
_game.RightMouseDown();
_holding = true;
}
}
else
{
var targetX = x + Rng.Next(-jitter, jitter + 1);
var targetY = y + Rng.Next(-jitter, jitter + 1);
await _game.MoveMouseFast(targetX, targetY);
if (mana < 0.30f)
{
Log.Information("Mana dropped to {Mana:P0}, releasing to recover", mana);
_game.LeftMouseUp();
_game.RightMouseUp();
_holding = false;
_manaStableCount = 0;
}
await Helpers.Sleep(Rng.Next(80, 120));
}
}
/// <summary>
/// Cycle WASD directions to orbit in a small circle while attacking.
/// </summary>
private async Task UpdateOrbit()
{
var now = _orbitSw.ElapsedMilliseconds;
if (now - _lastOrbitMs < OrbitStepMs) return;
_lastOrbitMs = now;
// Release previous direction
if (_orbitIndex >= 0)
await _game.KeyUp(OrbitKeys[_orbitIndex]);
// Advance to next direction
_orbitIndex = (_orbitIndex + 1) % OrbitKeys.Length;
await _game.KeyDown(OrbitKeys[_orbitIndex]);
}
private async Task ReleaseOrbit()
{
if (_orbitIndex >= 0)
{
await _game.KeyUp(OrbitKeys[_orbitIndex]);
_orbitIndex = -1;
}
}
/// <summary>
/// Reset state for a new combat phase (releases held buttons if any).
/// </summary>
public async Task Reset()
{
if (_holding)
{
_game.LeftMouseUp();
_game.RightMouseUp();
}
_holding = false;
_manaStableCount = 0;
await ReleaseOrbit();
}
/// <summary>
/// Release any held mouse buttons and movement keys. Call in finally blocks.
/// </summary>
public async Task ReleaseAll()
{
if (_holding)
{
_game.LeftMouseUp();
_game.RightMouseUp();
_holding = false;
}
await ReleaseOrbit();
}
}

View file

@ -0,0 +1,56 @@
using System.Diagnostics;
using Poe2Trade.Game;
using Poe2Trade.Screen;
using Serilog;
namespace Poe2Trade.Bot;
/// <summary>
/// Monitors life/mana and presses flask keys when they drop below thresholds.
/// Call <see cref="Tick"/> from any combat loop iteration.
/// </summary>
public class FlaskManager
{
private const float LifeThreshold = 0.50f;
private const float ManaThreshold = 0.50f;
private const long CooldownMs = 4000;
// VK codes for number keys 1 and 2
private const int VK_1 = 0x31;
private const int VK_2 = 0x32;
private readonly IGameController _game;
private readonly HudReader _hudReader;
private readonly Stopwatch _sw = Stopwatch.StartNew();
private long _lastLifeFlaskMs = -CooldownMs;
private long _lastManaFlaskMs = -CooldownMs;
public FlaskManager(IGameController game, HudReader hudReader)
{
_game = game;
_hudReader = hudReader;
}
/// <summary>
/// Check life/mana and press flask keys if needed. Call every loop iteration.
/// </summary>
public async Task Tick()
{
var hud = _hudReader.Current;
var now = _sw.ElapsedMilliseconds;
if (hud.LifePct < LifeThreshold && now - _lastLifeFlaskMs >= CooldownMs)
{
Log.Debug("Life flask: {Life:P0} < {Threshold:P0}", hud.LifePct, LifeThreshold);
await _game.PressKey(VK_1);
_lastLifeFlaskMs = now;
}
if (hud.ManaPct < ManaThreshold && now - _lastManaFlaskMs >= CooldownMs)
{
Log.Debug("Mana flask: {Mana:P0} < {Threshold:P0}", hud.ManaPct, ManaThreshold);
await _game.PressKey(VK_2);
_lastManaFlaskMs = now;
}
}
}

View file

@ -44,6 +44,7 @@ public class KulemakSettings
public string InvitationTabPath { get; set; } = ""; public string InvitationTabPath { get; set; } = "";
public string LootTabPath { get; set; } = ""; public string LootTabPath { get; set; } = "";
public int InvitationCount { get; set; } = 15; public int InvitationCount { get; set; } = 15;
public int RunCount { get; set; } = 15;
} }
public class ConfigStore public class ConfigStore

View file

@ -3,7 +3,7 @@ namespace Poe2Trade.Core;
public static class Delays public static class Delays
{ {
public const int PostFocus = 300; public const int PostFocus = 300;
public const int PostTravel = 1500; public const int PostTravel = 3000;
public const int PostStashOpen = 1000; public const int PostStashOpen = 1000;
public const int ClickInterval = 150; public const int ClickInterval = 150;
public const int PostEscape = 500; public const int PostEscape = 500;

View file

@ -0,0 +1,71 @@
using Serilog;
namespace Poe2Trade.Screen;
/// <summary>
/// Debug-only: periodically captures the screen, runs loot label detection,
/// and exposes the latest results for overlay rendering.
/// </summary>
public class LootDebugDetector : IDisposable
{
private readonly IScreenReader _screen;
private volatile List<LootLabel> _latest = [];
private Timer? _timer;
private volatile bool _enabled;
private int _running; // guard against overlapping ticks
public LootDebugDetector(IScreenReader screen)
{
_screen = screen;
}
public IReadOnlyList<LootLabel> Latest => _latest;
public bool Enabled
{
get => _enabled;
set
{
if (_enabled == value) return;
_enabled = value;
if (value)
_timer = new Timer(_ => Tick(), null, 0, 500);
else
{
_timer?.Dispose();
_timer = null;
_latest = [];
}
}
}
private void Tick()
{
if (!_enabled) return;
if (Interlocked.CompareExchange(ref _running, 1, 0) != 0) return;
try
{
using var frame = _screen.CaptureRawBitmap();
var labels = _screen.DetectLootLabels(frame, frame);
_latest = labels;
if (labels.Count > 0)
Log.Information("[LootDebug] Detected {Count} labels", labels.Count);
}
catch (Exception ex)
{
Log.Warning("[LootDebug] Detection failed: {Error}", ex.Message);
_latest = [];
}
finally
{
Interlocked.Exchange(ref _running, 0);
}
}
public void Dispose()
{
_timer?.Dispose();
_timer = null;
}
}

View file

@ -202,11 +202,20 @@ public class ScreenReader : IScreenReader
public Bitmap CaptureRawBitmap() => ScreenCapture.CaptureOrLoad(null, null); public Bitmap CaptureRawBitmap() => ScreenCapture.CaptureOrLoad(null, null);
// Nameplate search region — skip top HUD, bottom bar, and side margins
private const int NpTop = 120, NpBottom = 1080, NpMargin = 300;
public Task<OcrResponse> NameplateDiffOcr(Bitmap reference, Bitmap current) public Task<OcrResponse> NameplateDiffOcr(Bitmap reference, Bitmap current)
{ {
int w = Math.Min(reference.Width, current.Width); int w = Math.Min(reference.Width, current.Width);
int h = Math.Min(reference.Height, current.Height); int h = Math.Min(reference.Height, current.Height);
// Clamp search region to image bounds
int scanY0 = Math.Min(NpTop, h);
int scanY1 = Math.Min(NpBottom, h);
int scanX0 = Math.Min(NpMargin, w);
int scanX1 = Math.Max(scanX0, w - NpMargin);
var refData = reference.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); var refData = reference.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
byte[] refPx = new byte[refData.Stride * h]; byte[] refPx = new byte[refData.Stride * h];
@ -218,74 +227,100 @@ public class ScreenReader : IScreenReader
current.UnlockBits(curData); current.UnlockBits(curData);
// Build a binary mask of pixels that got significantly brighter (nameplates are bright text) // Build a binary mask of pixels that got significantly brighter (nameplates are bright text)
// Only scan within the play-area region to skip UI and reduce work
const int brightThresh = 30; const int brightThresh = 30;
bool[] mask = new bool[w * h]; int scanW = scanX1 - scanX0;
Parallel.For(0, h, y => int scanH = scanY1 - scanY0;
bool[] mask = new bool[scanW * scanH];
Parallel.For(0, scanH, sy =>
{ {
int y = sy + scanY0;
int rowOff = y * stride; int rowOff = y * stride;
for (int x = 0; x < w; x++) for (int sx = 0; sx < scanW; sx++)
{ {
int x = sx + scanX0;
int i = rowOff + x * 4; int i = rowOff + x * 4;
int brighter = (curPx[i] - refPx[i]) + (curPx[i + 1] - refPx[i + 1]) + (curPx[i + 2] - refPx[i + 2]); int brighter = (curPx[i] - refPx[i]) + (curPx[i + 1] - refPx[i + 1]) + (curPx[i + 2] - refPx[i + 2]);
if (brighter > brightThresh) if (brighter > brightThresh)
mask[y * w + x] = true; mask[sy * scanW + sx] = true;
} }
}); });
// Find connected clusters via row-scan: collect bounding boxes of bright regions // Find connected clusters via row-scan: collect bounding boxes of bright regions
var boxes = FindBrightClusters(mask, w, h, minWidth: 40, minHeight: 10, maxGap: 8); var boxes = FindBrightClusters(mask, scanW, scanH, minWidth: 40, minHeight: 10, maxGap: 8);
// Offset cluster boxes back to full-image coordinates
for (int i = 0; i < boxes.Count; i++)
{
var b = boxes[i];
boxes[i] = new Rectangle(b.X + scanX0, b.Y + scanY0, b.Width, b.Height);
}
Log.Information("NameplateDiff: found {Count} bright clusters", boxes.Count); Log.Information("NameplateDiff: found {Count} bright clusters", boxes.Count);
if (boxes.Count == 0) if (boxes.Count == 0)
return Task.FromResult(new OcrResponse { Text = "", Lines = [] }); return Task.FromResult(new OcrResponse { Text = "", Lines = [] });
// OCR each cluster crop, accumulate results with screen-space coordinates // Collect valid cluster crops and stitch into a single image for one OCR call
var allLines = new List<OcrLine>(); const int pad = 4;
var allText = new List<string>(); const int sep = 20; // black separator between crops to prevent cross-detection
var crops = new List<(int screenX, int screenY, int cropW, int cropH, int stitchY)>();
for (int bi = 0; bi < boxes.Count; bi++) int maxCropW = 0;
int totalH = 0;
foreach (var box in boxes)
{ {
var box = boxes[bi];
// Pad the crop slightly
int pad = 4;
int cx = Math.Max(0, box.X - pad); int cx = Math.Max(0, box.X - pad);
int cy = Math.Max(0, box.Y - pad); int cy = Math.Max(0, box.Y - pad);
int cw = Math.Min(w - cx, box.Width + pad * 2); int cw = Math.Min(w - cx, box.Width + pad * 2);
int ch = Math.Min(h - cy, box.Height + pad * 2); int ch = Math.Min(h - cy, box.Height + pad * 2);
using var crop = current.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb); crops.Add((cx, cy, cw, ch, totalH));
var clusterSw = System.Diagnostics.Stopwatch.StartNew(); maxCropW = Math.Max(maxCropW, cw);
totalH += ch + sep;
}
if (crops.Count == 0)
return Task.FromResult(new OcrResponse { Text = "", Lines = [] });
totalH -= sep; // no separator after last crop
// Stitch all crops vertically into one image
using var stitched = new Bitmap(maxCropW, totalH, PixelFormat.Format32bppArgb);
using (var g = System.Drawing.Graphics.FromImage(stitched))
{
g.Clear(System.Drawing.Color.Black);
foreach (var (sx, sy, cw, ch, sY) in crops)
g.DrawImage(current, new Rectangle(0, sY, cw, ch), new Rectangle(sx, sy, cw, ch), GraphicsUnit.Pixel);
}
// Single OCR call for all clusters
var ocrSw = System.Diagnostics.Stopwatch.StartNew();
OcrResponse ocrResult; OcrResponse ocrResult;
try try
{ {
ocrResult = _pythonBridge.OcrFromBitmap(crop); ocrResult = _pythonBridge.OcrFromBitmap(stitched);
} }
catch (TimeoutException) catch (TimeoutException)
{ {
Log.Warning("NameplateDiffOcr: cluster {I}/{Count} OCR timed out, skipping", bi + 1, boxes.Count); Log.Warning("NameplateDiffOcr: batch OCR timed out ({Count} clusters)", crops.Count);
continue; return Task.FromResult(new OcrResponse { Text = "", Lines = [] });
} }
Log.Debug("NameplateDiffOcr: cluster {I}/{Count} ({W}x{H}) OCR took {Ms}ms", Log.Information("NameplateDiffOcr: batch OCR {Count} clusters in {Ms}ms",
bi + 1, boxes.Count, cw, ch, clusterSw.ElapsedMilliseconds); crops.Count, ocrSw.ElapsedMilliseconds);
// Offset word coordinates to screen space // Map OCR results back to screen coordinates
foreach (var line in ocrResult.Lines) foreach (var line in ocrResult.Lines)
{ {
foreach (var word in line.Words) foreach (var word in line.Words)
{ {
word.X += cx; // Find which crop this word belongs to by Y position
word.Y += cy; var crop = crops.Last(c => word.Y >= c.stitchY);
} word.X += crop.screenX;
allLines.Add(line); word.Y = word.Y - crop.stitchY + crop.screenY;
allText.Add(line.Text);
} }
} }
return Task.FromResult(new OcrResponse return Task.FromResult(ocrResult);
{
Text = string.Join("\n", allText),
Lines = allLines,
});
} }
private static List<Rectangle> FindBrightClusters(bool[] mask, int w, int h, int minWidth, int minHeight, int maxGap) private static List<Rectangle> FindBrightClusters(bool[] mask, int w, int h, int minWidth, int minHeight, int maxGap)
@ -358,21 +393,29 @@ public class ScreenReader : IScreenReader
public void SetLootBaseline(Bitmap frame) { } public void SetLootBaseline(Bitmap frame) { }
// Detection parameters
// -- Loot detection constants -- // -- Loot detection constants --
private const int CannyLow = 20, CannyHigh = 80; private const int CannyLow = 20, CannyHigh = 80;
// Shape constraints // Shape constraints (passes 1 & 2)
private const int LabelMinW = 80, LabelMaxW = 500; private const int LabelMinW = 100, LabelMaxW = 500;
private const int LabelMinH = 15, LabelMaxH = 100; private const int LabelMinH = 15, LabelMaxH = 100;
private const double LabelMinAspect = 1.3, LabelMaxAspect = 10.0; private const double LabelMinAspect = 1.3, LabelMaxAspect = 10.0;
// Strict pass: well-formed rectangle contours // Pass 1: strict (well-formed bordered rectangles)
private const double MinRectangularity = 0.5; private const double MinRectangularity = 0.7;
private const float StrictMinBS = 200f; private const double StrictMinBS = 255;
private const float StrictMinEdgeDensity = 25f; // Pass 2: relaxed (play-area contours, VFX-tolerant)
// Relaxed pass: any contour bbox in play area (catches VFX-broken borders)
private const int RelaxedMinW = 100; private const int RelaxedMinW = 100;
private const float RelaxedMinBS = 250f; private const double RelaxedMinBS = 265;
private const float RelaxedMinEdgeDensity = 25f; private const double RelaxedBrightPctThreshold = 8;
private const double RelaxedBgDarkPctThreshold = 50;
private const double MaxGreenPct = 5;
// Pass 3: yellow text clusters (borderless labels)
private const int YellowHueMin = 10, YellowHueMax = 35;
private const int YellowMinSat = 120, YellowMinVal = 120;
private const double YellowTextPctThreshold = 25;
private const int TextClusterMinWidth = 100;
private const double TextClusterMinAspect = 1.5;
private const double TextClusterContainmentThreshold = 0.5;
// Play area bounds
private const double UiMarginTop = 0.08; private const double UiMarginTop = 0.08;
private const double UiMarginBottom = 0.82; private const double UiMarginBottom = 0.82;
// Post-processing // Post-processing
@ -381,9 +424,10 @@ public class ScreenReader : IScreenReader
private const double NmsIouThresh = 0.4; private const double NmsIouThresh = 0.4;
/// <summary> /// <summary>
/// Two-pass loot label detection: /// Three-pass loot label detection:
/// 1. Strict: polygon-approximated rectangle contours (high precision) /// 1. Strict: polygon-approximated rectangle contours (bordered labels)
/// 2. Relaxed: any contour bbox in play area (catches VFX-broken borders) /// 2. Relaxed: contour bbox with label-like content OR bright text on dark background
/// 3. Yellow text clusters: morphological detection of gold/yellow text without background box
/// Results merged, horizontal fragments joined, then NMS. /// Results merged, horizontal fragments joined, then NMS.
/// </summary> /// </summary>
public List<LootLabel> DetectLootLabels(Bitmap reference, Bitmap current) public List<LootLabel> DetectLootLabels(Bitmap reference, Bitmap current)
@ -402,57 +446,59 @@ public class ScreenReader : IScreenReader
using var hsv = new Mat(); using var hsv = new Mat();
Cv2.CvtColor(mat, hsv, ColorConversionCodes.BGR2HSV); Cv2.CvtColor(mat, hsv, ColorConversionCodes.BGR2HSV);
// Edge detection // Split HSV channels once for reuse
Cv2.Split(hsv, out Mat[] hsvChannels);
using var hChan = hsvChannels[0];
using var sChan = hsvChannels[1];
using var vChan = hsvChannels[2];
// ── Passes 1 & 2: Edge-based detection ──
using var edges = new Mat(); using var edges = new Mat();
Cv2.Canny(gray, edges, CannyLow, CannyHigh); Cv2.Canny(gray, edges, CannyLow, CannyHigh);
using var dilateKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3)); using var dilateKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3));
using var dilated = new Mat(); Cv2.Dilate(edges, edges, dilateKernel, iterations: 1);
Cv2.Dilate(edges, dilated, dilateKernel, iterations: 1);
Cv2.FindContours(dilated, out var contours, out _, RetrievalModes.Tree, ContourApproximationModes.ApproxSimple); Cv2.FindContours(edges, out var contours, out _, RetrievalModes.Tree, ContourApproximationModes.ApproxSimple);
var strict = new List<LabelCandidate>(); var strict = new List<LabelCandidate>();
var relaxed = new List<LabelCandidate>(); var relaxed = new List<LabelCandidate>();
foreach (var contour in contours) foreach (var contour in contours)
{ {
var box = Cv2.BoundingRect(contour); var bbox = Cv2.BoundingRect(contour);
int x = bbox.X, y = bbox.Y, w = bbox.Width, h = bbox.Height;
// Common shape gate // Common shape gate
if (box.Width <= LabelMinW || box.Width >= LabelMaxW) continue; if (w < LabelMinW || w > LabelMaxW || h < LabelMinH || h > LabelMaxH) continue;
if (box.Height <= LabelMinH || box.Height >= LabelMaxH) continue; double aspect = (double)w / Math.Max(h, 1);
double aspect = (double)box.Width / Math.Max(box.Height, 1); if (aspect < LabelMinAspect || aspect > LabelMaxAspect) continue;
if (aspect <= LabelMinAspect || aspect >= LabelMaxAspect) continue;
// Content metrics // Content metrics (mean brightness + saturation)
using var roiHsv = new Mat(hsv, box); using var roiV = new Mat(vChan, bbox);
var meanHsv = Cv2.Mean(roiHsv); using var roiS = new Mat(sChan, bbox);
float meanVal = (float)meanHsv[2]; double meanVal = Cv2.Mean(roiV).Val0;
float meanSat = (float)meanHsv[1]; double meanSat = Cv2.Mean(roiS).Val0;
float bs = meanVal + meanSat; double bs = meanVal + meanSat;
using var roiGray = new Mat(gray, box);
using var roiEdges = new Mat();
Cv2.Canny(roiGray, roiEdges, 50, 150);
float ed = (float)Cv2.Mean(roiEdges)[0];
// Strict pass: well-formed polygon (4-8 vertices)
var peri = Cv2.ArcLength(contour, true);
var approx = Cv2.ApproxPolyDP(contour, 0.02 * peri, true);
// Pass 1: strict well-formed polygon (4-8 vertices)
var approx = Cv2.ApproxPolyDP(contour, Cv2.ArcLength(contour, true) * 0.02, true);
if (approx.Length >= 4 && approx.Length <= 8) if (approx.Length >= 4 && approx.Length <= 8)
{ {
double contourArea = Cv2.ContourArea(approx); double contourArea = Cv2.ContourArea(approx);
double rect = contourArea / Math.Max(box.Width * box.Height, 1); double rectangularity = contourArea / Math.Max(w * h, 1);
if (rect >= MinRectangularity && bs >= StrictMinBS && ed >= StrictMinEdgeDensity) if (rectangularity >= MinRectangularity && bs >= StrictMinBS)
strict.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat)); strict.Add(new LabelCandidate(x, y, w, h, (float)meanVal, (float)meanSat));
} }
// Relaxed pass: any contour bbox in play area // Pass 2: relaxed play area, bs OR bright-on-dark
bool inPlay = box.Y > playTop && box.Y + box.Height < playBot; bool inPlay = y > playTop && (y + h) < playBot;
if (inPlay && box.Width >= RelaxedMinW && bs >= RelaxedMinBS && ed >= RelaxedMinEdgeDensity) if (!inPlay || w < RelaxedMinW) continue;
relaxed.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat));
bool passesBs = bs >= RelaxedMinBS;
bool passesTextOnDark = !passesBs && CheckBrightTextOnDark(mat, vChan, sChan, bbox);
if (passesBs || passesTextOnDark)
relaxed.Add(new LabelCandidate(x, y, w, h, (float)meanVal, (float)meanSat));
} }
// Merge strict + relaxed (strict wins on overlap) // Merge strict + relaxed (strict wins on overlap)
@ -466,6 +512,14 @@ public class ScreenReader : IScreenReader
// Join horizontal fragments // Join horizontal fragments
merged = MergeHorizontal(merged, MergeGap, MergeYTolerance); merged = MergeHorizontal(merged, MergeGap, MergeYTolerance);
// ── Pass 3: Yellow text cluster detection (borderless labels) ──
var textClusters = DetectYellowTextClusters(mat, hChan, sChan, vChan, playTop, playBot);
foreach (var tc in textClusters)
{
if (!ContainedByAny(tc, merged, TextClusterContainmentThreshold))
merged.Add(tc);
}
// Build LootLabels with color classification // Build LootLabels with color classification
var scored = new List<(LootLabel Label, float Score)>(); var scored = new List<(LootLabel Label, float Score)>();
foreach (var c in merged) foreach (var c in merged)
@ -487,7 +541,6 @@ public class ScreenReader : IScreenReader
{ {
current.Save("debug_loot_capture.png", System.Drawing.Imaging.ImageFormat.Png); current.Save("debug_loot_capture.png", System.Drawing.Imaging.ImageFormat.Png);
Cv2.ImWrite("debug_loot_edges.png", edges); Cv2.ImWrite("debug_loot_edges.png", edges);
Cv2.ImWrite("debug_loot_dilated.png", dilated);
using var debugMat = mat.Clone(); using var debugMat = mat.Clone();
foreach (var label in labels) foreach (var label in labels)
Cv2.Rectangle(debugMat, Cv2.Rectangle(debugMat,
@ -500,8 +553,8 @@ public class ScreenReader : IScreenReader
Log.Warning(ex, "Failed to save debug images"); Log.Warning(ex, "Failed to save debug images");
} }
Log.Information("DetectLootLabels: strict={Strict} relaxed={Relaxed} merged={Merged} final={Final}", Log.Information("DetectLootLabels: strict={Strict} relaxed={Relaxed} yellow={Yellow} final={Final}",
strict.Count, relaxed.Count, merged.Count, labels.Count); strict.Count, relaxed.Count, textClusters.Count, labels.Count);
foreach (var label in labels) foreach (var label in labels)
Log.Information(" Label ({X},{Y}) {W}x{H} color=({R},{G},{B}) tier={Tier}", Log.Information(" Label ({X},{Y}) {W}x{H} color=({R},{G},{B}) tier={Tier}",
label.CenterX - label.Width / 2, label.CenterY - label.Height / 2, label.CenterX - label.Width / 2, label.CenterY - label.Height / 2,
@ -526,81 +579,271 @@ public class ScreenReader : IScreenReader
return ((byte)mean[2], (byte)mean[1], (byte)mean[0]); return ((byte)mean[2], (byte)mean[1], (byte)mean[0]);
} }
/// <summary>
/// Pass 2 helper: verify bright+saturated text on dark background with green fire rejection.
/// </summary>
private bool CheckBrightTextOnDark(Mat bgrImage, Mat vChan, Mat sChan, Rect bbox)
{
int area = bbox.Width * bbox.Height;
if (area == 0) return false;
using var roiV = new Mat(vChan, bbox);
using var roiS = new Mat(sChan, bbox);
// Bright + saturated pixels (the text)
using var brightMask = new Mat();
using var satMask = new Mat();
Cv2.Threshold(roiV, brightMask, 150, 255, ThresholdTypes.Binary);
Cv2.Threshold(roiS, satMask, 100, 255, ThresholdTypes.Binary);
using var brightSat = new Mat();
Cv2.BitwiseAnd(brightMask, satMask, brightSat);
double brightPct = (double)Cv2.CountNonZero(brightSat) / area * 100;
if (brightPct < RelaxedBrightPctThreshold)
return false;
// Background darkness: non-text pixels should be dark
using var textMask = new Mat();
using var bgMask = new Mat();
Cv2.Threshold(roiV, textMask, 120, 255, ThresholdTypes.Binary);
using var textSatMask = new Mat();
Cv2.Threshold(roiS, textSatMask, 100, 255, ThresholdTypes.Binary);
Cv2.BitwiseAnd(textMask, textSatMask, textMask);
Cv2.BitwiseNot(textMask, bgMask);
int bgCount = Cv2.CountNonZero(bgMask);
if (bgCount == 0) return false;
using var bgV = new Mat();
roiV.CopyTo(bgV, bgMask);
using var darkBg = new Mat();
Cv2.Threshold(bgV, darkBg, 40, 255, ThresholdTypes.BinaryInv);
Cv2.BitwiseAnd(darkBg, bgMask, darkBg);
double bgDarkPct = (double)Cv2.CountNonZero(darkBg) / bgCount * 100;
if (bgDarkPct < RelaxedBgDarkPctThreshold)
return false;
// Green fire rejection
return !IsGreenDominant(bgrImage, bbox, area);
}
/// <summary>
/// Pass 3: detect gold/yellow text clusters without background boxes (normal rarity items).
/// Uses HSV thresholding, green fire subtraction, and morphological grouping.
/// </summary>
private List<LabelCandidate> DetectYellowTextClusters(
Mat bgrImage, Mat hChan, Mat sChan, Mat vChan,
int playTop, int playBot)
{
var results = new List<LabelCandidate>();
// Build yellow text mask: H:10-35, S>120, V>120
using var hMin = new Mat();
using var hMax = new Mat();
using var sThresh = new Mat();
using var vThresh = new Mat();
Cv2.Threshold(hChan, hMin, YellowHueMin - 1, 255, ThresholdTypes.Binary);
Cv2.Threshold(hChan, hMax, YellowHueMax, 255, ThresholdTypes.BinaryInv);
Cv2.Threshold(sChan, sThresh, YellowMinSat - 1, 255, ThresholdTypes.Binary);
Cv2.Threshold(vChan, vThresh, YellowMinVal - 1, 255, ThresholdTypes.Binary);
using var yellowMask = new Mat();
Cv2.BitwiseAnd(hMin, hMax, yellowMask);
Cv2.BitwiseAnd(yellowMask, sThresh, yellowMask);
Cv2.BitwiseAnd(yellowMask, vThresh, yellowMask);
// Subtract green fire pixels
SubtractGreenFire(bgrImage, yellowMask);
// Morphological grouping
using var kH = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(25, 1));
using var dilated = new Mat();
Cv2.Dilate(yellowMask, dilated, kH, iterations: 1);
using var kV = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(1, 8));
using var closed = new Mat();
Cv2.MorphologyEx(dilated, closed, MorphTypes.Close, kV);
using var kO = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(40, 5));
using var cleaned = new Mat();
Cv2.MorphologyEx(closed, cleaned, MorphTypes.Open, kO);
// Find and filter text clusters
Cv2.FindContours(cleaned, out var textContours, out _, RetrievalModes.External, ContourApproximationModes.ApproxSimple);
foreach (var contour in textContours)
{
var bbox = Cv2.BoundingRect(contour);
int x = bbox.X, y = bbox.Y, w = bbox.Width, h = bbox.Height;
if (w < TextClusterMinWidth || h < LabelMinH || h > 120) continue;
double aspect = (double)w / Math.Max(h, 1);
if (aspect < TextClusterMinAspect) continue;
bool inPlay = y > playTop && (y + h) < playBot;
if (!inPlay) continue;
// Verify yellow text density in the bounding box
using var roiYellow = new Mat(yellowMask, bbox);
double yellowPct = (double)Cv2.CountNonZero(roiYellow) / (w * h) * 100;
if (yellowPct >= YellowTextPctThreshold)
results.Add(new LabelCandidate(x, y, w, h, (float)(yellowPct * 10), 0));
}
return results;
}
/// <summary>Check if region is green-fire dominant (G > R+15, G > B+15, G > 80).</summary>
private static bool IsGreenDominant(Mat bgrImage, Rect bbox, int area)
{
using var roiBgr = new Mat(bgrImage, bbox);
Cv2.Split(roiBgr, out Mat[] bgr);
try
{
using var gMinusR = new Mat();
using var gMinusB = new Mat();
Cv2.Subtract(bgr[1], bgr[2], gMinusR);
Cv2.Subtract(bgr[1], bgr[0], gMinusB);
using var thR = new Mat();
using var thB = new Mat();
using var thG = new Mat();
Cv2.Threshold(gMinusR, thR, 15, 255, ThresholdTypes.Binary);
Cv2.Threshold(gMinusB, thB, 15, 255, ThresholdTypes.Binary);
Cv2.Threshold(bgr[1], thG, 80, 255, ThresholdTypes.Binary);
using var greenMask = new Mat();
Cv2.BitwiseAnd(thR, thB, greenMask);
Cv2.BitwiseAnd(greenMask, thG, greenMask);
double greenPct = (double)Cv2.CountNonZero(greenMask) / area * 100;
return greenPct >= MaxGreenPct;
}
finally
{
foreach (var ch in bgr) ch.Dispose();
}
}
/// <summary>Zero out green-fire pixels from a mask in-place.</summary>
private static void SubtractGreenFire(Mat bgrImage, Mat mask)
{
Cv2.Split(bgrImage, out Mat[] bgr);
try
{
using var gMinusR = new Mat();
using var gMinusB = new Mat();
Cv2.Subtract(bgr[1], bgr[2], gMinusR);
Cv2.Subtract(bgr[1], bgr[0], gMinusB);
using var thR = new Mat();
using var thB = new Mat();
using var thG = new Mat();
Cv2.Threshold(gMinusR, thR, 15, 255, ThresholdTypes.Binary);
Cv2.Threshold(gMinusB, thB, 15, 255, ThresholdTypes.Binary);
Cv2.Threshold(bgr[1], thG, 80, 255, ThresholdTypes.Binary);
using var greenFire = new Mat();
Cv2.BitwiseAnd(thR, thB, greenFire);
Cv2.BitwiseAnd(greenFire, thG, greenFire);
using var notGreen = new Mat();
Cv2.BitwiseNot(greenFire, notGreen);
Cv2.BitwiseAnd(mask, notGreen, mask);
}
finally
{
foreach (var ch in bgr) ch.Dispose();
}
}
private static bool OverlapsAny(LabelCandidate label, List<LabelCandidate> others, double iouThresh) private static bool OverlapsAny(LabelCandidate label, List<LabelCandidate> others, double iouThresh)
{ {
foreach (var o in others) foreach (var o in others)
{ {
int ix1 = Math.Max(label.X, o.X), iy1 = Math.Max(label.Y, o.Y); if (ComputeIoU(label, o) > iouThresh)
int ix2 = Math.Min(label.X + label.W, o.X + o.W);
int iy2 = Math.Min(label.Y + label.H, o.Y + o.H);
int inter = Math.Max(0, ix2 - ix1) * Math.Max(0, iy2 - iy1);
int union = label.W * label.H + o.W * o.H - inter;
if (inter / (double)Math.Max(union, 1) > iouThresh)
return true; return true;
} }
return false; return false;
} }
/// <summary> /// <summary>Check if label is mostly contained inside any existing detection.</summary>
/// Merge labels that sit side-by-side on the same line. private static bool ContainedByAny(LabelCandidate label, List<LabelCandidate> others, double containThresh)
/// </summary> {
int labelArea = label.W * label.H;
if (labelArea == 0) return true;
foreach (var o in others)
{
int xx1 = Math.Max(label.X, o.X);
int yy1 = Math.Max(label.Y, o.Y);
int xx2 = Math.Min(label.X + label.W, o.X + o.W);
int yy2 = Math.Min(label.Y + label.H, o.Y + o.H);
int inter = Math.Max(0, xx2 - xx1) * Math.Max(0, yy2 - yy1);
if ((double)inter / labelArea > containThresh)
return true;
}
return false;
}
private static double ComputeIoU(LabelCandidate a, LabelCandidate b)
{
int xx1 = Math.Max(a.X, b.X);
int yy1 = Math.Max(a.Y, b.Y);
int xx2 = Math.Min(a.X + a.W, b.X + b.W);
int yy2 = Math.Min(a.Y + a.H, b.Y + b.H);
int inter = Math.Max(0, xx2 - xx1) * Math.Max(0, yy2 - yy1);
int union = a.W * a.H + b.W * b.H - inter;
return (double)inter / Math.Max(union, 1);
}
private static List<LabelCandidate> MergeHorizontal(List<LabelCandidate> labels, int gap, int yTol) private static List<LabelCandidate> MergeHorizontal(List<LabelCandidate> labels, int gap, int yTol)
{ {
if (labels.Count < 2) return labels; if (labels.Count < 2) return labels;
var used = new bool[labels.Count]; var used = new bool[labels.Count];
var indices = Enumerable.Range(0, labels.Count) var sorted = Enumerable.Range(0, labels.Count)
.OrderBy(i => labels[i].Y).ThenBy(i => labels[i].X).ToList(); .OrderBy(i => labels[i].Y).ThenBy(i => labels[i].X).ToList();
var result = new List<LabelCandidate>(); var result = new List<LabelCandidate>();
for (int ii = 0; ii < indices.Count; ii++) foreach (int i in sorted)
{ {
int i = indices[ii];
if (used[i]) continue; if (used[i]) continue;
used[i] = true; used[i] = true;
var a = labels[i]; int gx1 = labels[i].X, gy1 = labels[i].Y;
int gx1 = a.X, gy1 = a.Y, gx2 = a.X + a.W, gy2 = a.Y + a.H; int gx2 = gx1 + labels[i].W, gy2 = gy1 + labels[i].H;
double briArea = a.MeanBrightness * a.W * a.H; double wBri = labels[i].MeanBrightness * labels[i].W * labels[i].H;
double satArea = a.MeanSaturation * a.W * a.H; double wSat = labels[i].MeanSaturation * labels[i].W * labels[i].H;
int totalArea = a.W * a.H; double area = labels[i].W * labels[i].H;
bool changed = true; bool changed = true;
while (changed) while (changed)
{ {
changed = false; changed = false;
for (int jj = 0; jj < indices.Count; jj++) foreach (int j in sorted)
{ {
int j = indices[jj];
if (used[j]) continue; if (used[j]) continue;
var b = labels[j]; var b = labels[j];
double cyA = (gy1 + gy2) / 2.0; double cyA = (gy1 + gy2) / 2.0;
double cyB = b.Y + b.H / 2.0; double cyB = b.Y + b.H / 2.0;
if (Math.Abs(cyA - cyB) > yTol) continue; if (Math.Abs(cyA - cyB) > yTol) continue;
int hGap = Math.Max(b.X - gx2, gx1 - (b.X + b.W)); int hGap = Math.Max(b.X - gx2, gx1 - (b.X + b.W));
if (hGap > gap) continue; if (hGap > gap) continue;
int bArea = b.W * b.H; double bArea = b.W * b.H;
gx1 = Math.Min(gx1, b.X); gx1 = Math.Min(gx1, b.X);
gy1 = Math.Min(gy1, b.Y); gy1 = Math.Min(gy1, b.Y);
gx2 = Math.Max(gx2, b.X + b.W); gx2 = Math.Max(gx2, b.X + b.W);
gy2 = Math.Max(gy2, b.Y + b.H); gy2 = Math.Max(gy2, b.Y + b.H);
briArea += b.MeanBrightness * bArea; wBri += b.MeanBrightness * bArea;
satArea += b.MeanSaturation * bArea; wSat += b.MeanSaturation * bArea;
totalArea += bArea; area += bArea;
used[j] = true; used[j] = true;
changed = true; changed = true;
} }
} }
int w = gx2 - gx1, h = gy2 - gy1; int w = gx2 - gx1, h = gy2 - gy1;
float bri = (float)(briArea / Math.Max(totalArea, 1)); result.Add(new LabelCandidate(gx1, gy1, w, h,
float sat = (float)(satArea / Math.Max(totalArea, 1)); (float)(wBri / Math.Max(area, 1)), (float)(wSat / Math.Max(area, 1))));
result.Add(new LabelCandidate(gx1, gy1, w, h, bri, sat));
} }
return result; return result;
} }

View file

@ -71,6 +71,7 @@ public sealed class D2dOverlay
using var ctx = new D2dRenderContext(hwnd, Width, Height); using var ctx = new D2dRenderContext(hwnd, Width, Height);
_layers.Add(new D2dEnemyBoxLayer(ctx)); _layers.Add(new D2dEnemyBoxLayer(ctx));
_layers.Add(new D2dLootLabelLayer(ctx));
_layers.Add(new D2dHudInfoLayer()); _layers.Add(new D2dHudInfoLayer());
_layers.Add(new D2dDebugTextLayer()); _layers.Add(new D2dDebugTextLayer());
@ -182,6 +183,9 @@ public sealed class D2dOverlay
{ {
var detection = _bot.EnemyDetector.Latest; var detection = _bot.EnemyDetector.Latest;
var bossDetection = _bot.BossDetector.Latest; var bossDetection = _bot.BossDetector.Latest;
var showLoot = _bot.LootDebugDetector.Enabled;
return new OverlayState( return new OverlayState(
Enemies: detection.Enemies, Enemies: detection.Enemies,
Bosses: bossDetection.Bosses, Bosses: bossDetection.Bosses,
@ -192,6 +196,8 @@ public sealed class D2dOverlay
NavPosition: _bot.Navigation.WorldPosition, NavPosition: _bot.Navigation.WorldPosition,
IsExploring: _bot.Navigation.IsExploring, IsExploring: _bot.Navigation.IsExploring,
ShowHudDebug: _bot.Store.Settings.ShowHudDebug, ShowHudDebug: _bot.Store.Settings.ShowHudDebug,
ShowLootDebug: showLoot,
LootLabels: _bot.LootDebugDetector.Latest,
Fps: fps, Fps: fps,
Timing: timing); Timing: timing);
} }

View file

@ -13,6 +13,8 @@ public record OverlayState(
MapPosition NavPosition, MapPosition NavPosition,
bool IsExploring, bool IsExploring,
bool ShowHudDebug, bool ShowHudDebug,
bool ShowLootDebug,
IReadOnlyList<LootLabel> LootLabels,
double Fps, double Fps,
RenderTiming? Timing); RenderTiming? Timing);

View file

@ -0,0 +1,61 @@
using System.Drawing;
using Vortice.Direct2D1;
using Vortice.DirectWrite;
using Vortice.Mathematics;
namespace Poe2Trade.Ui.Overlay.Layers;
internal sealed class D2dLootLabelLayer : ID2dOverlayLayer, IDisposable
{
private readonly D2dRenderContext _ctx;
private readonly Dictionary<string, IDWriteTextLayout> _labelCache = new();
private ID2D1SolidColorBrush? _fillBrush;
public D2dLootLabelLayer(D2dRenderContext ctx)
{
_ctx = ctx;
_fillBrush = ctx.RenderTarget.CreateSolidColorBrush(new Color4(0f, 1f, 0f, 0.15f));
}
public void Draw(D2dRenderContext ctx, OverlayState state)
{
if (!state.ShowLootDebug || state.LootLabels.Count == 0) return;
var rt = ctx.RenderTarget;
foreach (var label in state.LootLabels)
{
float x = label.CenterX - label.Width / 2f;
float y = label.CenterY - label.Height / 2f;
float w = label.Width;
float h = label.Height;
var rect = new RectangleF(x, y, w, h);
rt.FillRectangle(rect, _fillBrush!);
rt.DrawRectangle(rect, ctx.Green, 3f);
// Label: "tier (R,G,B)"
var key = $"{label.Tier} ({label.AvgR},{label.AvgG},{label.AvgB})";
if (!_labelCache.TryGetValue(key, out var layout))
{
layout = _ctx.CreateTextLayout(key, _ctx.LabelFormat);
_labelCache[key] = layout;
}
var m = layout.Metrics;
var labelX = x;
var labelY = y - m.Height - 2;
rt.FillRectangle(
new RectangleF(labelX - 1, labelY - 1, m.Width + 2, m.Height + 2),
ctx.LabelBgBrush);
rt.DrawTextLayout(new System.Numerics.Vector2(labelX, labelY), layout, ctx.Green);
}
}
public void Dispose()
{
foreach (var l in _labelCache.Values) l?.Dispose();
}
}

View file

@ -20,6 +20,12 @@ public partial class DebugViewModel : ObservableObject
[ObservableProperty] private string _selectedGridLayout = "inventory"; [ObservableProperty] private string _selectedGridLayout = "inventory";
[ObservableProperty] private decimal? _clickX; [ObservableProperty] private decimal? _clickX;
[ObservableProperty] private decimal? _clickY; [ObservableProperty] private decimal? _clickY;
[ObservableProperty] private bool _showLootDebug;
partial void OnShowLootDebugChanged(bool value)
{
_bot.LootDebugDetector.Enabled = value;
}
public string[] GridLayoutNames { get; } = public string[] GridLayoutNames { get; } =
[ [

View file

@ -24,6 +24,7 @@ public partial class MappingViewModel : ObservableObject, IDisposable
[ObservableProperty] private string _invitationTabPath = ""; [ObservableProperty] private string _invitationTabPath = "";
[ObservableProperty] private string _lootTabPath = ""; [ObservableProperty] private string _lootTabPath = "";
[ObservableProperty] private decimal? _invitationCount = 15; [ObservableProperty] private decimal? _invitationCount = 15;
[ObservableProperty] private decimal? _runCount = 15;
public static MapType[] MapTypes { get; } = [MapType.TrialOfChaos, MapType.Temple, MapType.Endgame, MapType.Kulemak]; public static MapType[] MapTypes { get; } = [MapType.TrialOfChaos, MapType.Temple, MapType.Endgame, MapType.Kulemak];
public ObservableCollection<string> StashTabPaths { get; } = []; public ObservableCollection<string> StashTabPaths { get; } = [];
@ -45,6 +46,7 @@ public partial class MappingViewModel : ObservableObject, IDisposable
_invitationTabPath = bot.Config.Kulemak.InvitationTabPath; _invitationTabPath = bot.Config.Kulemak.InvitationTabPath;
_lootTabPath = bot.Config.Kulemak.LootTabPath; _lootTabPath = bot.Config.Kulemak.LootTabPath;
_invitationCount = bot.Config.Kulemak.InvitationCount; _invitationCount = bot.Config.Kulemak.InvitationCount;
_runCount = bot.Config.Kulemak.RunCount;
LoadStashTabPaths(); LoadStashTabPaths();
_bot.EnemyDetector.DetectionUpdated += OnDetectionUpdated; _bot.EnemyDetector.DetectionUpdated += OnDetectionUpdated;
@ -80,6 +82,11 @@ public partial class MappingViewModel : ObservableObject, IDisposable
_bot.Store.UpdateSettings(s => s.Kulemak.InvitationCount = (int)(value ?? 15)); _bot.Store.UpdateSettings(s => s.Kulemak.InvitationCount = (int)(value ?? 15));
} }
partial void OnRunCountChanged(decimal? value)
{
_bot.Store.UpdateSettings(s => s.Kulemak.RunCount = (int)(value ?? 15));
}
private void LoadStashTabPaths() private void LoadStashTabPaths()
{ {
StashTabPaths.Clear(); StashTabPaths.Clear();

View file

@ -255,10 +255,10 @@
SelectedItem="{Binding LootTabPath}" /> SelectedItem="{Binding LootTabPath}" />
</DockPanel> </DockPanel>
<DockPanel> <DockPanel>
<TextBlock Text="Invitations per batch" FontSize="11" Foreground="#8b949e" <TextBlock Text="Runs" FontSize="11" Foreground="#8b949e"
Width="140" VerticalAlignment="Center" /> Width="140" VerticalAlignment="Center" />
<NumericUpDown Value="{Binding InvitationCount}" <NumericUpDown Value="{Binding RunCount}"
Minimum="1" Maximum="60" Increment="1" Width="100" /> Minimum="1" Maximum="999" Increment="1" Width="100" />
</DockPanel> </DockPanel>
</StackPanel> </StackPanel>
</Border> </Border>
@ -334,6 +334,17 @@
</StackPanel> </StackPanel>
</Border> </Border>
<!-- Debug overlays -->
<Border Background="#161b22" BorderBrush="#30363d" BorderThickness="1"
CornerRadius="8" Padding="8">
<StackPanel Spacing="6">
<TextBlock Text="DEBUG OVERLAYS" FontSize="11" FontWeight="SemiBold"
Foreground="#8b949e" />
<ToggleSwitch IsChecked="{Binding ShowLootDebug}"
Content="Loot Labels" Foreground="#e6edf3" />
</StackPanel>
</Border>
<!-- Row 2: Find text --> <!-- Row 2: Find text -->
<Border Background="#161b22" BorderBrush="#30363d" BorderThickness="1" <Border Background="#161b22" BorderBrush="#30363d" BorderThickness="1"
CornerRadius="8" Padding="8"> CornerRadius="8" Padding="8">