boki/poe2-bot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rwork on kulemak bot and cleanup

Browse source
This commit is contained in:
Boki 2026-02-21 09:18:10 -05:00
parent c75b2b27f0
commit 053a016c8b
15 changed files with 727 additions and 160 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
assets/black-cathedral-door.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 4.1 KiB

BIN
assets/return-the-ring.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 21 KiB

394
src/Poe2Trade.Bot/BossRunExecutor.cs
View file

@ -3,38 +3,37 @@ using Poe2Trade.Core;
using Poe2Trade.Game;
using Poe2Trade.GameLog;
using Poe2Trade.Inventory;
using Poe2Trade.Navigation;
using Poe2Trade.Screen;
using Serilog;
namespace Poe2Trade.Bot;
public class BossRunExecutor
public class BossRunExecutor : GameExecutor
{
private static readonly string WellOfSoulsTemplate = Path.Combine("assets", "well-of-souls.png");
private static readonly string BlackCathedralTemplate = Path.Combine("assets", "black-cathedral.png");
private static readonly string InvitationTemplate = Path.Combine("assets", "invitation.png");
private static readonly string CathedralDoorTemplate = Path.Combine("assets", "black-cathedral-door.png");
private static readonly string ReturnTheRingTemplate = Path.Combine("assets", "return-the-ring.png");
private BossRunState _state = BossRunState.Idle;
private bool _stopped;
private readonly IGameController _game;
private readonly IScreenReader _screen;
private readonly IInventoryManager _inventory;
private readonly IClientLogWatcher _logWatcher;
private readonly SavedSettings _config;
private readonly BossDetector _bossDetector;
private readonly HudReader _hudReader;
private readonly NavigationExecutor _nav;
public event Action<BossRunState>? StateChanged;
public BossRunExecutor(IGameController game, IScreenReader screen,
IInventoryManager inventory, IClientLogWatcher logWatcher, SavedSettings config,
BossDetector bossDetector)
BossDetector bossDetector, HudReader hudReader, NavigationExecutor nav)
: base(game, screen, inventory, config)
{
_game = game;
_screen = screen;
_inventory = inventory;
_logWatcher = logWatcher;
_config = config;
_bossDetector = bossDetector;
_hudReader = hudReader;
_nav = nav;
}
public BossRunState State => _state;
@ -45,9 +44,9 @@ public class BossRunExecutor
StateChanged?.Invoke(s);
}
public void Stop()
public override void Stop()
{
_stopped = true;
base.Stop();
Log.Information("Boss run executor stop requested");
}
@ -100,6 +99,7 @@ public class BossRunExecutor
await Fight();
if (_stopped) break;
SetState(BossRunState.Looting);
await Loot();
if (_stopped) break;
@ -299,17 +299,251 @@ public class BossRunExecutor
private async Task Fight()
{
SetState(BossRunState.Fighting);
Log.Information("[PLACEHOLDER] Fight phase - waiting for manual combat");
// Placeholder: user handles combat manually for now
await Helpers.Sleep(1000);
Log.Information("Fight phase starting");
// Wait for arena to settle
await Helpers.Sleep(6000);
if (_stopped) return;
// Find and click the cathedral door
Log.Information("Looking for cathedral door...");
var door = await _screen.TemplateMatch(CathedralDoorTemplate);
if (door == null)
{
Log.Error("Could not find cathedral door template");
return;
}
Log.Information("Found cathedral door at ({X},{Y}), clicking", door.X, door.Y);
await _game.LeftClickAt(door.X, door.Y);
// Wait for cathedral interior to load
await Helpers.Sleep(12000);
if (_stopped) return;
// Walk to fight area (world coords)
const double fightWorldX = -454;
const double fightWorldY = -332;
const double wellWorldX = -496;
const double wellWorldY = -378;
await WalkToWorldPosition(fightWorldX, fightWorldY);
if (_stopped) return;
// 3x fight-then-well loop
for (var phase = 1; phase <= 3; phase++)
{
if (_stopped) return;
Log.Information("=== Boss phase {Phase}/4 ===", phase);
await AttackBossUntilGone();
if (_stopped) return;
// Walk to well and click it
Log.Information("Phase {Phase} done, walking to well", phase);
await WalkToWorldPosition(wellWorldX, wellWorldY);
// Click at screen center (well should be near character)
await _game.LeftClickAt(1280, 720);
await Helpers.Sleep(2000);
// Walk back to fight position for next phase
await WalkToWorldPosition(fightWorldX, fightWorldY);
}
// 4th fight - no well after
if (_stopped) return;
Log.Information("=== Boss phase 4/4 ===");
await AttackBossUntilGone();
if (_stopped) return;
// Return the ring
Log.Information("Looking for Return the Ring...");
var ring = await _screen.TemplateMatch(ReturnTheRingTemplate);
if (ring == null)
{
Log.Warning("Could not find Return the Ring template, retrying after 2s...");
await Helpers.Sleep(2000);
ring = await _screen.TemplateMatch(ReturnTheRingTemplate);
}
if (ring != null)
{
Log.Information("Found Return the Ring at ({X},{Y}), clicking", ring.X, ring.Y);
await _game.LeftClickAt(ring.X, ring.Y);
await Helpers.Sleep(2000);
}
else
{
Log.Error("Could not find Return the Ring template");
}
if (_stopped) return;
// Walk up and press Q
Log.Information("Walking up and pressing Q");
await _game.KeyDown(InputSender.VK.W);
await Helpers.Sleep(1500);
await _game.KeyUp(InputSender.VK.W);
await Helpers.Sleep(300);
await _game.PressKey(InputSender.VK.Q);
await Helpers.Sleep(500);
// Spam L+R at position for 7s
Log.Information("Attacking at ring fight position (Q phase)");
await AttackAtPosition(1280, 720, 7000);
if (_stopped) return;
// Press E, spam L+R at same position for 7s
Log.Information("Pressing E and continuing attack");
await _game.PressKey(InputSender.VK.E);
await Helpers.Sleep(500);
await AttackAtPosition(1280, 720, 7000);
Log.Information("Fight complete");
}
private async Task Loot()
private async Task AttackBossUntilGone(int timeoutMs = 120_000)
{
SetState(BossRunState.Looting);
Log.Information("[PLACEHOLDER] Loot phase - waiting for manual looting");
// Placeholder: user handles looting manually for now
await Helpers.Sleep(1000);
// Move mouse to screen center initially
await _game.MoveMouseFast(1280, 720);
await Helpers.Sleep(200);
var sw = Stopwatch.StartNew();
var consecutiveMisses = 0;
while (sw.ElapsedMilliseconds < timeoutMs)
{
if (_stopped) return;
var snapshot = _bossDetector.Latest;
if (snapshot.Bosses.Count > 0)
{
consecutiveMisses = 0;
var boss = snapshot.Bosses[0];
// Check mana before attacking
var hud = _hudReader.Current;
if (hud.ManaPct < 0.80f)
{
await Helpers.Sleep(200);
continue;
}
// Move to boss and attack
var targetX = boss.Cx + Rng.Next(-10, 11);
var targetY = boss.Cy + Rng.Next(-10, 11);
await _game.MoveMouseFast(targetX, targetY);
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
_game.LeftMouseUp();
await Helpers.Sleep(Rng.Next(20, 40));
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(100, 150));
}
else
{
consecutiveMisses++;
if (consecutiveMisses >= 15)
{
Log.Information("Boss gone after {Ms}ms ({Misses} consecutive misses)",
sw.ElapsedMilliseconds, consecutiveMisses);
return;
}
await Helpers.Sleep(200);
}
}
Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs);
}
private async Task AttackAtPosition(int x, int y, int durationMs)
{
var sw = Stopwatch.StartNew();
while (sw.ElapsedMilliseconds < durationMs)
{
if (_stopped) return;
var targetX = x + Rng.Next(-20, 21);
var targetY = y + Rng.Next(-20, 21);
await _game.MoveMouseFast(targetX, targetY);
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
_game.LeftMouseUp();
await Helpers.Sleep(Rng.Next(20, 40));
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(100, 150));
}
}
/// <summary>
/// Walk to a world position using WASD keys, checking minimap position each iteration.
/// </summary>
private async Task WalkToWorldPosition(double worldX, double worldY, int timeoutMs = 10000, double arrivalDist = 15)
{
Log.Information("Walking to world ({X:F0},{Y:F0})", worldX, worldY);
var sw = Stopwatch.StartNew();
var heldKeys = new HashSet<int>();
try
{
while (sw.ElapsedMilliseconds < timeoutMs)
{
if (_stopped) break;
var pos = _nav.WorldPosition;
var dx = worldX - pos.X;
var dy = worldY - pos.Y;
var dist = Math.Sqrt(dx * dx + dy * dy);
if (dist <= arrivalDist)
{
Log.Information("Arrived at ({X:F0},{Y:F0}), dist={Dist:F0}", pos.X, pos.Y, dist);
break;
}
// Normalize direction
var len = Math.Sqrt(dx * dx + dy * dy);
var dirX = dx / len;
var dirY = dy / len;
// Map direction to WASD keys
var wanted = new HashSet<int>();
if (dirY < -0.3) wanted.Add(InputSender.VK.W); // up
if (dirY > 0.3) wanted.Add(InputSender.VK.S); // down
if (dirX < -0.3) wanted.Add(InputSender.VK.A); // left
if (dirX > 0.3) wanted.Add(InputSender.VK.D); // right
// Release keys no longer needed
foreach (var key in heldKeys.Except(wanted).ToList())
{
await _game.KeyUp(key);
heldKeys.Remove(key);
}
// Press new keys
foreach (var key in wanted.Except(heldKeys).ToList())
{
await _game.KeyDown(key);
heldKeys.Add(key);
}
await Helpers.Sleep(100);
}
if (sw.ElapsedMilliseconds >= timeoutMs)
Log.Warning("WalkToWorldPosition timed out after {Ms}ms", timeoutMs);
}
finally
{
// Release all held keys
foreach (var key in heldKeys)
await _game.KeyUp(key);
}
}
private async Task<bool> ReturnHome()
@ -408,122 +642,4 @@ public class BossRunExecutor
Log.Information("Loot stored");
}
private async Task<TemplateMatchResult?> WalkAndMatch(string templatePath, int vk1, int vk2,
int timeoutMs = 15000, int closeRadius = 350)
{
const int screenCx = 2560 / 2;
const int screenCy = 1440 / 2;
await _game.KeyDown(vk1);
await _game.KeyDown(vk2);
try
{
var sw = Stopwatch.StartNew();
bool spotted = false;
while (sw.ElapsedMilliseconds < timeoutMs)
{
if (_stopped) return null;
var match = await _screen.TemplateMatch(templatePath);
if (match == null)
{
await Helpers.Sleep(500);
continue;
}
var dx = match.X - screenCx;
var dy = match.Y - screenCy;
var dist = Math.Sqrt(dx * dx + dy * dy);
if (!spotted)
{
Log.Information("Template spotted at ({X},{Y}), dist={Dist:F0}px from center, approaching...",
match.X, match.Y, dist);
spotted = true;
}
if (dist <= closeRadius)
{
Log.Information("Close enough at ({X},{Y}), dist={Dist:F0}px, stopping", match.X, match.Y, dist);
// Stop, settle, re-match for accurate position
await _game.KeyUp(vk2);
await _game.KeyUp(vk1);
await Helpers.Sleep(300);
var fresh = await _screen.TemplateMatch(templatePath);
if (fresh != null)
{
Log.Information("Final position at ({X},{Y})", fresh.X, fresh.Y);
return fresh;
}
Log.Warning("Re-match failed, using last known position");
return match;
}
await Helpers.Sleep(200);
}
Log.Error("WalkAndMatch timed out after {Ms}ms (spotted={Spotted})", timeoutMs, spotted);
return null;
}
finally
{
await _game.KeyUp(vk2);
await _game.KeyUp(vk1);
}
}
private (StashTabInfo? Tab, StashTabInfo? Folder) ResolveTabPath(string tabPath)
{
if (string.IsNullOrEmpty(tabPath) || _config.StashCalibration == null)
return (null, null);
var parts = tabPath.Split('/');
if (parts.Length == 1)
{
// Simple tab name
var tab = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0]);
return (tab, null);
}
if (parts.Length == 2)
{
// Folder/SubTab
var folder = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0] && t.IsFolder);
if (folder == null) return (null, null);
var subTab = folder.SubTabs.FirstOrDefault(t => t.Name == parts[1]);
return (subTab, folder);
}
return (null, null);
}
private async Task RecoverToHideout()
{
try
{
Log.Information("Recovering: escaping and going to hideout");
await _game.FocusGame();
await _game.PressEscape();
await Helpers.Sleep(Delays.PostEscape);
await _game.PressEscape();
await Helpers.Sleep(Delays.PostEscape);
var arrived = await _inventory.WaitForAreaTransition(
_config.TravelTimeoutMs, () => _game.GoToHideout());
if (arrived)
{
_inventory.SetLocation(true);
Log.Information("Recovery: arrived at hideout");
}
else
{
Log.Warning("Recovery: timed out going to hideout");
}
}
catch (Exception ex)
{
Log.Error(ex, "Recovery failed");
}
}
}

12
src/Poe2Trade.Bot/BotOrchestrator.cs
View file

@ -74,7 +74,9 @@ public class BotOrchestrator : IAsyncDisposable
GameState = new GameStateDetector();
HudReader = new HudReader();
EnemyDetector = new EnemyDetector();
EnemyDetector.Enabled = true;
BossDetector = new BossDetector();
BossDetector.Enabled = true;
FrameSaver = new FrameSaver();
// Register on shared pipeline
@ -89,7 +91,7 @@ public class BotOrchestrator : IAsyncDisposable
Navigation = new NavigationExecutor(game, pipelineService.Pipeline, minimapCapture,
enemyDetector: EnemyDetector);
BossRunExecutor = new BossRunExecutor(game, screen, inventory, logWatcher, store.Settings, BossDetector);
BossRunExecutor = new BossRunExecutor(game, screen, inventory, logWatcher, store.Settings, BossDetector, HudReader, Navigation);
logWatcher.AreaEntered += area =>
{
@ -111,13 +113,7 @@ public class BotOrchestrator : IAsyncDisposable
if (BossZones.TryGetValue(area, out var boss))
{
BossDetector.SetBoss(boss);
BossDetector.Enabled = true;
Log.Information("Boss zone detected: {Area} → enabling {Boss} detector", area, boss);
}
else if (BossDetector.Enabled)
{
BossDetector.Enabled = false;
Log.Information("Left boss zone → disabling boss detector");
Log.Information("Boss zone detected: {Area} → switching to {Boss} model", area, boss);
}
}

222
src/Poe2Trade.Bot/GameExecutor.cs Normal file
View file

@ -0,0 +1,222 @@
using System.Diagnostics;
using Poe2Trade.Core;
using Poe2Trade.Game;
using Poe2Trade.Inventory;
using Poe2Trade.Screen;
using Serilog;
namespace Poe2Trade.Bot;
/// <summary>
/// Base class for game executors that interact with the game world.
/// Provides shared utilities: loot pickup, recovery, walking, stash tab resolution.
/// </summary>
public abstract class GameExecutor
{
protected static readonly Random Rng = new();
protected readonly IGameController _game;
protected readonly IScreenReader _screen;
protected readonly IInventoryManager _inventory;
protected readonly SavedSettings _config;
protected volatile bool _stopped;
protected GameExecutor(IGameController game, IScreenReader screen,
IInventoryManager inventory, SavedSettings config)
{
_game = game;
_screen = screen;
_inventory = inventory;
_config = config;
}
public virtual void Stop()
{
_stopped = true;
}
// ------ Loot pickup ------
// Tiers to skip (noise, low-value, or hidden by filter)
private static readonly HashSet<string> SkipTiers = ["unknown", "gold"];
public async Task Loot()
{
Log.Information("Starting loot pickup");
const int maxRounds = 5;
for (var round = 0; round < maxRounds; round++)
{
if (_stopped) return;
// Move mouse out of the way so it doesn't cover labels
_game.MoveMouseInstant(0, 1440);
await Helpers.Sleep(100);
// Hold Alt to ensure all labels are visible, then capture
await _game.KeyDown(InputSender.VK.MENU);
await Helpers.Sleep(250);
var capture = _screen.CaptureRawBitmap();
// Detect magenta-bordered labels directly (no diff needed)
var labels = _screen.DetectLootLabels(capture, capture);
capture.Dispose();
// Filter out noise and unwanted tiers
var pickups = labels.Where(l => !SkipTiers.Contains(l.Tier)).ToList();
if (pickups.Count == 0)
{
await _game.KeyUp(InputSender.VK.MENU);
Log.Information("No loot labels in round {Round} (total detected: {Total}, filtered: {Filtered})",
round + 1, labels.Count, labels.Count - pickups.Count);
break;
}
Log.Information("Round {Round}: {Count} loot labels ({Skipped} skipped)",
round + 1, pickups.Count, labels.Count - pickups.Count);
foreach (var skip in labels.Where(l => SkipTiers.Contains(l.Tier)))
Log.Debug("Skipped: tier={Tier} color=({R},{G},{B}) at ({X},{Y})",
skip.Tier, skip.AvgR, skip.AvgG, skip.AvgB, skip.CenterX, skip.CenterY);
// Click each label center (Alt still held so labels visible)
foreach (var label in pickups)
{
if (_stopped) break;
Log.Information("Picking up: tier={Tier} color=({R},{G},{B}) at ({X},{Y})",
label.Tier, label.AvgR, label.AvgG, label.AvgB, label.CenterX, label.CenterY);
await _game.LeftClickAt(label.CenterX, label.CenterY);
await Helpers.Sleep(300);
}
await _game.KeyUp(InputSender.VK.MENU);
await Helpers.Sleep(500);
}
Log.Information("Loot pickup complete");
}
// ------ Recovery ------
protected async Task RecoverToHideout()
{
try
{
Log.Information("Recovering: escaping and going to hideout");
await _game.FocusGame();
await _game.PressEscape();
await Helpers.Sleep(Delays.PostEscape);
await _game.PressEscape();
await Helpers.Sleep(Delays.PostEscape);
var arrived = await _inventory.WaitForAreaTransition(
_config.TravelTimeoutMs, () => _game.GoToHideout());
if (arrived)
{
_inventory.SetLocation(true);
Log.Information("Recovery: arrived at hideout");
}
else
{
Log.Warning("Recovery: timed out going to hideout");
}
}
catch (Exception ex)
{
Log.Error(ex, "Recovery failed");
}
}
// ------ Walk + template match ------
protected async Task<TemplateMatchResult?> WalkAndMatch(string templatePath, int vk1, int vk2,
int timeoutMs = 15000, int closeRadius = 350)
{
const int screenCx = 2560 / 2;
const int screenCy = 1440 / 2;
await _game.KeyDown(vk1);
await _game.KeyDown(vk2);
try
{
var sw = Stopwatch.StartNew();
bool spotted = false;
while (sw.ElapsedMilliseconds < timeoutMs)
{
if (_stopped) return null;
var match = await _screen.TemplateMatch(templatePath);
if (match == null)
{
await Helpers.Sleep(500);
continue;
}
var dx = match.X - screenCx;
var dy = match.Y - screenCy;
var dist = Math.Sqrt(dx * dx + dy * dy);
if (!spotted)
{
Log.Information("Template spotted at ({X},{Y}), dist={Dist:F0}px from center, approaching...",
match.X, match.Y, dist);
spotted = true;
}
if (dist <= closeRadius)
{
Log.Information("Close enough at ({X},{Y}), dist={Dist:F0}px, stopping", match.X, match.Y, dist);
// Stop, settle, re-match for accurate position
await _game.KeyUp(vk2);
await _game.KeyUp(vk1);
await Helpers.Sleep(300);
var fresh = await _screen.TemplateMatch(templatePath);
if (fresh != null)
{
Log.Information("Final position at ({X},{Y})", fresh.X, fresh.Y);
return fresh;
}
Log.Warning("Re-match failed, using last known position");
return match;
}
await Helpers.Sleep(200);
}
Log.Error("WalkAndMatch timed out after {Ms}ms (spotted={Spotted})", timeoutMs, spotted);
return null;
}
finally
{
await _game.KeyUp(vk2);
await _game.KeyUp(vk1);
}
}
// ------ Stash tab resolution ------
protected (StashTabInfo? Tab, StashTabInfo? Folder) ResolveTabPath(string tabPath)
{
if (string.IsNullOrEmpty(tabPath) || _config.StashCalibration == null)
return (null, null);
var parts = tabPath.Split('/');
if (parts.Length == 1)
{
var tab = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0]);
return (tab, null);
}
if (parts.Length == 2)
{
var folder = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0] && t.IsFolder);
if (folder == null) return (null, null);
var subTab = folder.SubTabs.FirstOrDefault(t => t.Name == parts[1]);
return (subTab, folder);
}
return (null, null);
}
}

2
src/Poe2Trade.Game/InputSender.cs
View file

@ -34,6 +34,8 @@ public class InputSender
public const int W = 0x57;
public const int S = 0x53;
public const int D = 0x44;
public const int E = 0x45;
public const int Q = 0x51;
public const int Z = 0x5A;
}

1
src/Poe2Trade.Navigation/NavigationExecutor.cs
View file

@ -491,6 +491,7 @@ public class NavigationExecutor : IDisposable
public bool IsExploring => _state != NavigationState.Idle && _state != NavigationState.Completed && _state != NavigationState.Failed;
public MapPosition Position => _worldMap.Position;
public MapPosition WorldPosition => _worldMap.WorldPosition;
public byte[] GetMapSnapshot() => _worldMap.GetMapSnapshot();
public byte[] GetViewportSnapshot(int viewSize = 400)
{

37
src/Poe2Trade.Navigation/WorldMap.cs
View file

@ -21,7 +21,19 @@ public class WorldMap : IDisposable
private readonly List<(Point Pos, long LastSeenMs)> _checkpointsOn = [];
private const int CheckpointDedupRadius = 20;
// World origin: cumulative offset from canvas (0,0) to world (0,0).
// World coords = canvas coords - _worldOrigin. Stable across canvas growth.
private double _worldOriginX;
private double _worldOriginY;
public MapPosition Position => _position;
/// <summary>
/// Player position in stable world coordinates (invariant to canvas growth).
/// World (0,0) = where the player spawned.
/// </summary>
public MapPosition WorldPosition => new(_position.X - _worldOriginX, _position.Y - _worldOriginY);
public bool LastMatchSucceeded { get; private set; }
public int CanvasSize => _canvasSize;
internal List<Point>? LastBfsPath => _pathFinder.LastResult?.Path;
@ -36,6 +48,9 @@ public class WorldMap : IDisposable
_canvas = new Mat(_canvasSize, _canvasSize, MatType.CV_8UC1, Scalar.Black);
_confidence = new Mat(_canvasSize, _canvasSize, MatType.CV_16SC1, Scalar.Black);
_position = new MapPosition(_canvasSize / 2.0, _canvasSize / 2.0);
// World origin = initial player position, so WorldPosition starts at (0,0)
_worldOriginX = _position.X;
_worldOriginY = _position.Y;
}
private void EnsureCapacity()
@ -63,7 +78,17 @@ public class WorldMap : IDisposable
_confidence = newConf;
_canvasSize = newSize;
_position = new MapPosition(_position.X + offset, _position.Y + offset);
Log.Information("Canvas grown: {Old}→{New}, offset={Offset}", oldSize, newSize, offset);
_worldOriginX += offset;
_worldOriginY += offset;
// Shift checkpoint canvas coordinates
for (var i = 0; i < _checkpointsOff.Count; i++)
_checkpointsOff[i] = (new Point(_checkpointsOff[i].Pos.X + offset, _checkpointsOff[i].Pos.Y + offset), _checkpointsOff[i].LastSeenMs);
for (var i = 0; i < _checkpointsOn.Count; i++)
_checkpointsOn[i] = (new Point(_checkpointsOn[i].Pos.X + offset, _checkpointsOn[i].Pos.Y + offset), _checkpointsOn[i].LastSeenMs);
Log.Information("Canvas grown: {Old}→{New}, offset={Offset}, worldOrigin=({Ox:F0},{Oy:F0})",
oldSize, newSize, offset, _worldOriginX, _worldOriginY);
}
/// <summary>
@ -619,6 +644,14 @@ public class WorldMap : IDisposable
return best;
}
/// <summary>Convert world coordinates to canvas coordinates.</summary>
public MapPosition WorldToCanvas(double worldX, double worldY) =>
new(worldX + _worldOriginX, worldY + _worldOriginY);
/// <summary>Convert canvas coordinates to world coordinates.</summary>
public MapPosition CanvasToWorld(double canvasX, double canvasY) =>
new(canvasX - _worldOriginX, canvasY - _worldOriginY);
public void Reset()
{
_canvas.Dispose();
@ -629,6 +662,8 @@ public class WorldMap : IDisposable
_prevWallMask?.Dispose();
_prevWallMask = null;
_position = new MapPosition(_canvasSize / 2.0, _canvasSize / 2.0);
_worldOriginX = _position.X;
_worldOriginY = _position.Y;
_frameCount = 0;
_consecutiveMatchFails = 0;
LastMatchSucceeded = false;

2
src/Poe2Trade.Screen/IScreenReader.cs
View file

@ -18,6 +18,8 @@ public interface IScreenReader : IDisposable
Task<DiffOcrResponse> DiffOcr(string? savePath = null, Region? region = null);
Task<TemplateMatchResult?> TemplateMatch(string templatePath, Region? region = null);
Task<OcrResponse> NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);
void SetLootBaseline(System.Drawing.Bitmap frame);
List<LootLabel> DetectLootLabels(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);
System.Drawing.Bitmap CaptureRawBitmap();
Task SaveScreenshot(string path);
Task SaveRegion(Region region, string path);

110
src/Poe2Trade.Screen/LootLabel.cs Normal file
View file

@ -0,0 +1,110 @@
namespace Poe2Trade.Screen;
/// <summary>
/// A detected loot label on screen with its position and classified tier.
/// </summary>
public record LootLabel(int CenterX, int CenterY, int Width, int Height, string Tier, byte AvgR, byte AvgG, byte AvgB);
/// <summary>
/// Classifies loot label background colors to NeverSink filter tiers
/// by matching against known filter color palette.
/// </summary>
public static class LootColorClassifier
{
private record ColorEntry(byte R, byte G, byte B, string Tier);
// Background colors from NeverSink's Uber Strict filter
private static readonly ColorEntry[] KnownBgColors =
[
// S-tier (apex): white bg
new(255, 255, 255, "S"),
// A-tier: red bg, white text
new(245, 105, 90, "A"),
// C-tier: orange bg
new(245, 139, 87, "C"),
// D-tier: yellow bg
new(240, 180, 100, "D"),
// E-tier: text-only, yellowish
new(240, 207, 132, "E"),
// Unique high: brown bg
new(188, 96, 37, "unique"),
// Unique T3: dark red bg
new(53, 13, 13, "unique-low"),
// Exotic bases: dark green bg
new(0, 75, 30, "exotic"),
// Identified mods: dark purple bg
new(47, 0, 74, "exotic-mod"),
// Rare jewellery: olive bg
new(75, 75, 0, "rare-jewellery"),
// Fragments: bright purple bg
new(220, 0, 255, "fragment"),
// Fragments lower: light purple bg
new(180, 75, 225, "fragment"),
// Fragment splinter: dark purple bg
new(50, 0, 75, "fragment"),
// Maps special: lavender bg
new(235, 220, 245, "map"),
// Maps regular high: light grey bg
new(235, 235, 235, "map"),
// Maps regular: grey bg
new(200, 200, 200, "map"),
// Crafting magic: dark blue-purple bg
new(30, 0, 70, "crafting"),
// Gems: cyan text (20,240,240) - no bg
new(20, 240, 240, "gem"),
// Gems: dark blue bg
new(6, 0, 60, "gem"),
// Flasks/charms: dark green bg
new(10, 60, 40, "flask"),
// Currency artifact: dark brown bg
new(76, 51, 12, "artifact"),
// Socketables (runes): orange-tan bg
new(220, 175, 132, "socketable"),
// Gold drops: gold/yellow text
new(180, 160, 80, "gold"),
new(200, 180, 100, "gold"),
// Pink/magenta catch-all (e.g. boss-specific drops like invitations)
new(255, 0, 255, "special"),
new(220, 50, 220, "special"),
];
private const double MaxDistance = 50.0;
/// <summary>
/// Classify an average RGB color to the closest NeverSink filter tier.
/// Returns "unknown" if no known color is within MaxDistance.
/// </summary>
public static string Classify(byte avgR, byte avgG, byte avgB)
{
double bestDist = double.MaxValue;
string bestTier = "unknown";
foreach (var entry in KnownBgColors)
{
double dr = avgR - entry.R;
double dg = avgG - entry.G;
double db = avgB - entry.B;
double dist = Math.Sqrt(dr * dr + dg * dg + db * db);
if (dist < bestDist)
{
bestDist = dist;
bestTier = entry.Tier;
}
}
return bestDist <= MaxDistance ? bestTier : "unknown";
}
}

76
src/Poe2Trade.Screen/ScreenReader.cs
View file

@ -1,10 +1,12 @@
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Poe2Trade.Core;
using OpenCvSharp;
using OpenCvSharp.Extensions;
using Poe2Trade.Core;
using Serilog;
using Region = Poe2Trade.Core.Region;
using Size = OpenCvSharp.Size;
namespace Poe2Trade.Screen;
@ -320,6 +322,78 @@ public class ScreenReader : IScreenReader
return boxes;
}
// -- Loot label detection (magenta background) --
//
// All loot labels: white border, magenta (255,0,255) background, black text.
// Magenta never appears in the game world → detect directly, no diff needed.
public void SetLootBaseline(Bitmap frame) { }
public List<LootLabel> DetectLootLabels(Bitmap reference, Bitmap current)
{
using var mat = BitmapConverter.ToMat(current);
if (mat.Channels() == 4)
Cv2.CvtColor(mat, mat, ColorConversionCodes.BGRA2BGR);
// Mask magenta background pixels (BGR: B≈255, G≈0, R≈255)
using var mask = new Mat();
Cv2.InRange(mat, new Scalar(200, 0, 200), new Scalar(255, 60, 255), mask);
// Morph close fills text gaps within a label
// Height=2 bridges line gaps within multi-line labels but not between separate labels
using var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(12, 2));
using var closed = new Mat();
Cv2.MorphologyEx(mask, closed, MorphTypes.Close, kernel);
// Save debug images
try
{
Cv2.ImWrite("debug_loot_mask.png", mask);
Cv2.ImWrite("debug_loot_closed.png", closed);
current.Save("debug_loot_capture.png", System.Drawing.Imaging.ImageFormat.Png);
Log.Information("Saved debug images: debug_loot_mask.png, debug_loot_closed.png, debug_loot_capture.png");
}
catch (Exception ex)
{
Log.Warning(ex, "Failed to save debug images");
}
Cv2.FindContours(closed, out var contours, out _,
RetrievalModes.External, ContourApproximationModes.ApproxSimple);
Log.Information("DetectLootLabels: {N} magenta contours", contours.Length);
const int minW = 40, maxW = 600;
const int minH = 8, maxH = 100;
const double minAspect = 1.5;
int yMax = mat.Height - 210;
var labels = new List<LootLabel>();
foreach (var contour in contours)
{
var box = Cv2.BoundingRect(contour);
double aspect = box.Height > 0 ? (double)box.Width / box.Height : 0;
if (box.Width < minW || box.Width > maxW ||
box.Height < minH || box.Height > maxH ||
aspect < minAspect ||
box.Y < 65 || box.Y + box.Height > yMax)
{
Log.Information("Rejected contour: ({X},{Y}) {W}x{H} aspect={Aspect:F1} yMax={YMax}",
box.X, box.Y, box.Width, box.Height, aspect, yMax);
continue;
}
int cx = box.X + box.Width / 2;
int cy = box.Y + box.Height / 2;
Log.Information("Label at ({X},{Y}) {W}x{H}", box.X, box.Y, box.Width, box.Height);
labels.Add(new LootLabel(cx, cy, box.Width, box.Height, "loot", 255, 0, 255));
}
return labels;
}
public void Dispose() => _pythonBridge.Dispose();
// -- OCR text matching --

2
src/Poe2Trade.Ui/Overlay/D2dOverlay.cs
View file

@ -189,7 +189,7 @@ public sealed class D2dOverlay
InferenceMs: detection.InferenceMs,
Hud: _bot.HudReader.Current,
NavState: _bot.Navigation.State,
NavPosition: _bot.Navigation.Position,
NavPosition: _bot.Navigation.WorldPosition,
IsExploring: _bot.Navigation.IsExploring,
ShowHudDebug: _bot.Store.Settings.ShowHudDebug,
Fps: fps,

18
src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs
View file

@ -323,4 +323,22 @@ public partial class DebugViewModel : ObservableObject
catch (Exception ex) { DebugResult = $"Failed: {ex.Message}"; }
}
[RelayCommand]
private async Task LootTest()
{
try
{
DebugResult = "Loot test: focusing game...";
await _bot.Game.FocusGame();
await Task.Delay(300);
await _bot.BossRunExecutor.Loot();
DebugResult = "Loot test: complete";
}
catch (Exception ex)
{
DebugResult = $"Loot test failed: {ex.Message}";
Log.Error(ex, "Loot test failed");
}
}
}

7
src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs
View file

@ -16,7 +16,6 @@ public partial class MappingViewModel : ObservableObject, IDisposable
[ObservableProperty] private MapType _selectedMapType;
[ObservableProperty] private bool _isFrameSaverEnabled;
[ObservableProperty] private int _framesSaved;
[ObservableProperty] private bool _isDetectionEnabled;
[ObservableProperty] private int _enemiesDetected;
[ObservableProperty] private float _inferenceMs;
[ObservableProperty] private bool _hasModel;
@ -106,12 +105,6 @@ public partial class MappingViewModel : ObservableObject, IDisposable
_bot.FrameSaver.Enabled = value;
}
partial void OnIsDetectionEnabledChanged(bool value)
{
_bot.EnemyDetector.Enabled = value;
_bot.BossDetector.Enabled = value;
}
private void OnDetectionUpdated(DetectionSnapshot snapshot)
{
Dispatcher.UIThread.Post(() =>

4
src/Poe2Trade.Ui/Views/MainWindow.axaml
View file

@ -287,9 +287,6 @@
<StackPanel Spacing="8">
<TextBlock Text="ENEMY DETECTION" FontSize="11" FontWeight="SemiBold"
Foreground="#8b949e" />
<ToggleSwitch IsChecked="{Binding IsDetectionEnabled}"
IsEnabled="{Binding HasModel}"
OnContent="Detection On" OffContent="Detection Off" />
<StackPanel Orientation="Horizontal" Spacing="16"
IsVisible="{Binding HasModel}">
<TextBlock Text="{Binding EnemiesDetected, StringFormat='{}{0} enemies'}"
@ -329,6 +326,7 @@
<Button Content="STASH" Command="{Binding ClickStashCommand}" />
<Button Content="SALVAGE" Command="{Binding ClickSalvageCommand}" />
<Button Content="Attack Test" Command="{Binding AttackTestCommand}" />
<Button Content="Loot" Command="{Binding LootTestCommand}" />
<Button Content="Detection?" Command="{Binding DetectionStatusCommand}" />
<Button Content="{Binding BurstCaptureLabel}"
Command="{Binding ToggleBurstCaptureCommand}" />