work on first boss

This commit is contained in:
Boki 2026-02-21 12:30:41 -05:00
parent 053a016c8b
commit 89c3a0a077
16 changed files with 702 additions and 150 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

BIN
assets/new.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

BIN
debug_loot_capture.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.3 MiB

BIN
debug_loot_detected.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.8 MiB

BIN
debug_loot_dilated.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 297 KiB

BIN
debug_loot_edges.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 639 KiB

View file

@ -15,7 +15,10 @@ public class BossRunExecutor : GameExecutor
private static readonly string BlackCathedralTemplate = Path.Combine("assets", "black-cathedral.png");
private static readonly string InvitationTemplate = Path.Combine("assets", "invitation.png");
private static readonly string CathedralDoorTemplate = Path.Combine("assets", "black-cathedral-door.png");
private static readonly string CathedralWellTemplate = Path.Combine("assets", "black-cathedral-well.png");
private static readonly string ReturnTheRingTemplate = Path.Combine("assets", "return-the-ring.png");
private static readonly string BossHealthbarTemplate = Path.Combine("assets", "unique-boss-present.png");
private static readonly string NewInstanceTemplate = Path.Combine("assets", "new.png");
private BossRunState _state = BossRunState.Idle;
private readonly IClientLogWatcher _logWatcher;
@ -53,7 +56,6 @@ public class BossRunExecutor : GameExecutor
public async Task RunBossLoop()
{
_stopped = false;
_bossDetector.SetBoss("kulemak");
Log.Information("Starting boss run loop ({Count} invitations)", _config.Kulemak.InvitationCount);
if (!await Prepare())
@ -193,6 +195,7 @@ public class BossRunExecutor : GameExecutor
// Close stash
await _game.PressEscape();
_inventory.ResetStashTabState();
await Helpers.Sleep(Delays.PostEscape);
Log.Information("Preparation complete");
@ -214,7 +217,7 @@ public class BossRunExecutor : GameExecutor
Log.Error("Could not find Waypoint nameplate");
return false;
}
await Helpers.Sleep(1000);
await Helpers.Sleep(500);
// Template match well-of-souls.png and click
var match = await _screen.TemplateMatch(WellOfSoulsTemplate);
@ -256,32 +259,24 @@ public class BossRunExecutor : GameExecutor
// Hover first so the game registers the target, then use invitation
await _game.MoveMouseTo(x, y);
await Helpers.Sleep(500);
await Helpers.Sleep(200);
await _game.CtrlLeftClickAt(x, y);
await Helpers.Sleep(1000);
await Helpers.Sleep(500);
// Find "NEW" text — pick the leftmost instance
var ocr = await _screen.Ocr();
var newWords = ocr.Lines
.SelectMany(l => l.Words)
.Where(w => w.Text.Equals("NEW", StringComparison.OrdinalIgnoreCase)
|| w.Text.Equals("New", StringComparison.Ordinal))
.OrderBy(w => w.X)
.ToList();
if (newWords.Count == 0)
// Find "NEW" button via template match — pick the leftmost
var matches = await _screen.TemplateMatchAll(NewInstanceTemplate);
if (matches.Count == 0)
{
Log.Error("Could not find 'NEW' text for instance selection");
Log.Error("Could not find 'NEW' template for instance selection");
return false;
}
var target = newWords[0];
var clickX = target.X + target.Width / 2;
var clickY = target.Y + target.Height / 2;
Log.Information("Found {Count} 'NEW' matches, clicking leftmost at ({X},{Y})", newWords.Count, clickX, clickY);
await _game.MoveMouseTo(clickX, clickY);
await Helpers.Sleep(500);
await _game.LeftClickAt(clickX, clickY);
var target = matches.OrderBy(m => m.X).First();
Log.Information("Found {Count} 'NEW' matches, clicking leftmost at ({X},{Y}) conf={Conf:F3}",
matches.Count, target.X, target.Y, target.Confidence);
await _game.MoveMouseTo(target.X, target.Y);
await Helpers.Sleep(150);
await _game.LeftClickAt(target.X, target.Y);
// Wait for area transition into boss arena
var arrived = await _inventory.WaitForAreaTransition(_config.TravelTimeoutMs);
@ -296,13 +291,26 @@ public class BossRunExecutor : GameExecutor
return true;
}
private void StartBossDetection()
{
_bossDetector.SetBoss("kulemak");
_bossDetector.Enabled = true;
Log.Information("Boss detection started");
}
private void StopBossDetection()
{
_bossDetector.Enabled = false;
Log.Information("Boss detection stopped");
}
private async Task Fight()
{
SetState(BossRunState.Fighting);
Log.Information("Fight phase starting");
// Wait for arena to settle
await Helpers.Sleep(6000);
await Helpers.Sleep(5500);
if (_stopped) return;
// Find and click the cathedral door
@ -317,12 +325,14 @@ public class BossRunExecutor : GameExecutor
await _game.LeftClickAt(door.X, door.Y);
// Wait for cathedral interior to load
await Helpers.Sleep(12000);
await Helpers.Sleep(14000);
if (_stopped) return;
StartBossDetection();
// Walk to fight area (world coords)
const double fightWorldX = -454;
const double fightWorldY = -332;
var fightWorldX = -454.0;
var fightWorldY = -332.0;
const double wellWorldX = -496;
const double wellWorldY = -378;
@ -335,14 +345,21 @@ public class BossRunExecutor : GameExecutor
if (_stopped) return;
Log.Information("=== Boss phase {Phase}/4 ===", phase);
await AttackBossUntilGone();
var lastBossPos = await AttackBossUntilGone();
if (_stopped) return;
// Walk to well and click it
// Update fight area to where the boss was last seen
if (lastBossPos != null)
{
fightWorldX = lastBossPos.Value.X;
fightWorldY = lastBossPos.Value.Y;
Log.Information("Fight area updated to ({X:F0},{Y:F0})", fightWorldX, fightWorldY);
}
// Walk to well and click the closest match to screen center
Log.Information("Phase {Phase} done, walking to well", phase);
await WalkToWorldPosition(wellWorldX, wellWorldY);
// Click at screen center (well should be near character)
await _game.LeftClickAt(1280, 720);
await ClickClosestTemplateToCenter(CathedralWellTemplate);
await Helpers.Sleep(2000);
// Walk back to fight position for next phase
@ -396,70 +413,208 @@ public class BossRunExecutor : GameExecutor
await Helpers.Sleep(500);
await AttackAtPosition(1280, 720, 7000);
StopBossDetection();
Log.Information("Fight complete");
}
private async Task AttackBossUntilGone(int timeoutMs = 120_000)
/// <summary>
/// Check top-of-screen region for the unique boss healthbar frame.
/// Uses lower threshold (0.5) to tolerate partial overlay from YOLO bounding boxes.
/// </summary>
private async Task<bool> IsBossAlive()
{
// Move mouse to screen center initially
await _game.MoveMouseFast(1280, 720);
var topRegion = new Region(800, 0, 960, 120);
var matches = await _screen.TemplateMatchAll(BossHealthbarTemplate, topRegion, threshold: 0.5);
return matches.Count > 0;
}
/// <summary>
/// Wait for the boss healthbar to appear (boss spawns/becomes active).
/// </summary>
private async Task<bool> WaitForBossSpawn(int timeoutMs = 30_000)
{
Log.Information("Waiting for boss healthbar to appear...");
var sw = Stopwatch.StartNew();
const int screenCx = 1280;
const int screenCy = 720;
while (sw.ElapsedMilliseconds < timeoutMs)
{
if (_stopped) return false;
if (await IsBossAlive())
{
Log.Information("Boss healthbar detected after {Ms}ms", sw.ElapsedMilliseconds);
return true;
}
// Attack at center while waiting — boss often spawns on top of us
var targetX = screenCx + Rng.Next(-30, 31);
var targetY = screenCy + Rng.Next(-30, 31);
await _game.MoveMouseFast(targetX, targetY);
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.LeftMouseUp();
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(50, 80));
}
Log.Warning("WaitForBossSpawn timed out after {Ms}ms", timeoutMs);
return false;
}
/// <summary>
/// Wait for boss to spawn, then attack until healthbar disappears.
/// Returns the last world position where YOLO spotted the boss, or null.
/// </summary>
private async Task<(double X, double Y)?> AttackBossUntilGone(int timeoutMs = 120_000)
{
// Wait for boss to actually appear before attacking
if (!await WaitForBossSpawn())
return null;
const int screenCx = 1280;
const int screenCy = 720;
// Attack target — defaults to screen center, updated by YOLO
var atkX = screenCx;
var atkY = screenCy;
(double X, double Y)? lastBossWorldPos = null;
await _game.MoveMouseFast(atkX, atkY);
await Helpers.Sleep(200);
var sw = Stopwatch.StartNew();
var consecutiveMisses = 0;
var lastCheckMs = 0L;
var holding = false;
var manaStableCount = 0;
while (sw.ElapsedMilliseconds < timeoutMs)
Log.Information("Boss is alive, engaging");
try
{
if (_stopped) return;
var snapshot = _bossDetector.Latest;
if (snapshot.Bosses.Count > 0)
while (sw.ElapsedMilliseconds < timeoutMs)
{
consecutiveMisses = 0;
var boss = snapshot.Bosses[0];
if (_stopped) return lastBossWorldPos;
// Check mana before attacking
var hud = _hudReader.Current;
if (hud.ManaPct < 0.80f)
// Update attack target from YOLO when available
var snapshot = _bossDetector.Latest;
if (snapshot.Bosses.Count > 0)
{
await Helpers.Sleep(200);
continue;
var boss = snapshot.Bosses[0];
atkX = boss.Cx;
atkY = boss.Cy;
// Convert boss screen offset to world position
// Scale: sqrt(72²+65²)/835 ≈ 0.116 minimap px per screen px
const double screenToWorld = 97.0 / 835.0;
var wp = _nav.WorldPosition;
var bossWorldX = wp.X + (boss.Cx - screenCx) * screenToWorld;
var bossWorldY = wp.Y + (boss.Cy - screenCy) * screenToWorld;
lastBossWorldPos = (bossWorldX, bossWorldY);
}
// Move to boss and attack
var targetX = boss.Cx + Rng.Next(-10, 11);
var targetY = boss.Cy + Rng.Next(-10, 11);
await _game.MoveMouseFast(targetX, targetY);
// Re-check healthbar every ~1.5s, need 5 consecutive misses (~7.5s) to confirm phase end
if (sw.ElapsedMilliseconds - lastCheckMs > 1500)
{
var bossAlive = await IsBossAlive();
lastCheckMs = sw.ElapsedMilliseconds;
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
_game.LeftMouseUp();
await Helpers.Sleep(Rng.Next(20, 40));
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
_game.RightMouseUp();
if (!bossAlive)
{
consecutiveMisses++;
Log.Information("Healthbar not found ({Misses}/5 consecutive misses, {Ms}ms elapsed)",
consecutiveMisses, sw.ElapsedMilliseconds);
if (consecutiveMisses >= 5)
{
Log.Information("Boss phase over after {Ms}ms", sw.ElapsedMilliseconds);
return lastBossWorldPos;
}
}
else
{
if (consecutiveMisses > 0)
Log.Information("Healthbar found again, resetting miss count (was {Misses})", consecutiveMisses);
consecutiveMisses = 0;
}
}
await Helpers.Sleep(Rng.Next(100, 150));
// Check mana to decide hold vs click
var mana = _hudReader.Current.ManaPct;
if (!holding)
{
// Ramp up: click cycle until mana is stable above 80%
if (mana >= 0.80f)
manaStableCount++;
else
manaStableCount = 0;
var targetX = atkX + Rng.Next(-30, 31);
var targetY = atkY + Rng.Next(-30, 31);
await _game.MoveMouseFast(targetX, targetY);
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.LeftMouseUp();
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(20, 35));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(50, 80));
// After 5 consecutive checks above 80%, switch to hold
if (manaStableCount >= 5)
{
Log.Information("Mana stable at {Mana:P0}, switching to hold attack", mana);
_game.LeftMouseDown();
_game.RightMouseDown();
holding = true;
}
}
else
{
// Holding: nudge mouse toward boss and monitor mana
var targetX = atkX + Rng.Next(-30, 31);
var targetY = atkY + Rng.Next(-30, 31);
await _game.MoveMouseFast(targetX, targetY);
if (mana < 0.30f)
{
Log.Information("Mana dropped to {Mana:P0}, releasing to recover", mana);
_game.LeftMouseUp();
_game.RightMouseUp();
holding = false;
manaStableCount = 0;
}
await Helpers.Sleep(Rng.Next(80, 120));
}
}
else
Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs);
return lastBossWorldPos;
}
finally
{
if (holding)
{
consecutiveMisses++;
if (consecutiveMisses >= 15)
{
Log.Information("Boss gone after {Ms}ms ({Misses} consecutive misses)",
sw.ElapsedMilliseconds, consecutiveMisses);
return;
}
await Helpers.Sleep(200);
_game.LeftMouseUp();
_game.RightMouseUp();
}
}
Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs);
}
private async Task AttackAtPosition(int x, int y, int durationMs)
{
var sw = Stopwatch.StartNew();
while (sw.ElapsedMilliseconds < durationMs)
{
if (_stopped) return;
@ -469,17 +624,44 @@ public class BossRunExecutor : GameExecutor
await _game.MoveMouseFast(targetX, targetY);
_game.LeftMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
await Helpers.Sleep(Rng.Next(20, 35));
_game.LeftMouseUp();
await Helpers.Sleep(Rng.Next(20, 40));
_game.RightMouseDown();
await Helpers.Sleep(Rng.Next(30, 50));
await Helpers.Sleep(Rng.Next(20, 35));
_game.RightMouseUp();
await Helpers.Sleep(Rng.Next(100, 150));
await Helpers.Sleep(Rng.Next(50, 80));
}
}
/// <summary>
/// Find all template matches and click the one closest to screen center.
/// </summary>
private async Task ClickClosestTemplateToCenter(string templatePath)
{
const int screenCx = 2560 / 2;
const int screenCy = 1440 / 2;
var matches = await _screen.TemplateMatchAll(templatePath);
if (matches.Count == 0)
{
Log.Warning("No matches found for {Template}, clicking screen center", Path.GetFileName(templatePath));
await _game.LeftClickAt(screenCx, screenCy);
return;
}
var closest = matches.OrderBy(m =>
{
var dx = m.X - screenCx;
var dy = m.Y - screenCy;
return dx * dx + dy * dy;
}).First();
Log.Information("Clicking closest match at ({X},{Y}) conf={Conf:F3} (of {Count} matches)",
closest.X, closest.Y, closest.Confidence, matches.Count);
await _game.LeftClickAt(closest.X, closest.Y);
}
/// <summary>
/// Walk to a world position using WASD keys, checking minimap position each iteration.
/// </summary>
@ -487,8 +669,12 @@ public class BossRunExecutor : GameExecutor
{
Log.Information("Walking to world ({X:F0},{Y:F0})", worldX, worldY);
const int screenCx = 1280;
const int screenCy = 720;
var sw = Stopwatch.StartNew();
var heldKeys = new HashSet<int>();
var lastBlinkMs = -2300L; // allow immediate first blink
try
{
@ -512,6 +698,19 @@ public class BossRunExecutor : GameExecutor
var dirX = dx / len;
var dirY = dy / len;
// Blink toward destination every ~2.3s ± 0.3s
var blinkCooldown = 2300 + Rng.Next(-300, 301);
if (sw.ElapsedMilliseconds - lastBlinkMs >= blinkCooldown)
{
// Move mouse in the travel direction so blink goes the right way
var blinkX = screenCx + (int)(dirX * 400);
var blinkY = screenCy + (int)(dirY * 400);
await _game.MoveMouseFast(blinkX, blinkY);
await Helpers.Sleep(30);
await _game.PressKey(InputSender.VK.SPACE);
lastBlinkMs = sw.ElapsedMilliseconds;
}
// Map direction to WASD keys
var wanted = new HashSet<int>();
if (dirY < -0.3) wanted.Add(InputSender.VK.W); // up
@ -556,13 +755,13 @@ public class BossRunExecutor : GameExecutor
// Walk away from loot (hold S briefly)
await _game.KeyDown(InputSender.VK.S);
await Helpers.Sleep(1000);
await Helpers.Sleep(500);
await _game.KeyUp(InputSender.VK.S);
await Helpers.Sleep(300);
await Helpers.Sleep(200);
// Press + to open portal
await _game.PressPlus();
await Helpers.Sleep(1500);
await Helpers.Sleep(800);
// Find "The Ardura Caravan" and click it
var caravanPos = await _inventory.FindAndClickNameplate("The Ardura Caravan", maxRetries: 5, retryDelayMs: 1500);
@ -638,6 +837,7 @@ public class BossRunExecutor : GameExecutor
// Close stash
await _game.PressEscape();
_inventory.ResetStashTabState();
await Helpers.Sleep(Delays.PostEscape);
Log.Information("Loot stored");

View file

@ -76,7 +76,6 @@ public class BotOrchestrator : IAsyncDisposable
EnemyDetector = new EnemyDetector();
EnemyDetector.Enabled = true;
BossDetector = new BossDetector();
BossDetector.Enabled = true;
FrameSaver = new FrameSaver();
// Register on shared pipeline

View file

@ -38,13 +38,15 @@ public abstract class GameExecutor
// ------ Loot pickup ------
// Tiers to skip (noise, low-value, or hidden by filter)
private static readonly HashSet<string> SkipTiers = ["unknown", "gold"];
private static readonly HashSet<string> SkipTiers = ["gold"];
public async Task Loot()
{
Log.Information("Starting loot pickup");
const int maxRounds = 5;
var totalPicked = 0;
for (var round = 0; round < maxRounds; round++)
{
if (_stopped) return;
@ -53,49 +55,58 @@ public abstract class GameExecutor
_game.MoveMouseInstant(0, 1440);
await Helpers.Sleep(100);
// Hold Alt to ensure all labels are visible, then capture
// Hold Alt, capture, detect
await _game.KeyDown(InputSender.VK.MENU);
await Helpers.Sleep(250);
var capture = _screen.CaptureRawBitmap();
// Detect magenta-bordered labels directly (no diff needed)
using var capture = _screen.CaptureRawBitmap();
var labels = _screen.DetectLootLabels(capture, capture);
capture.Dispose();
// Filter out noise and unwanted tiers
var pickups = labels.Where(l => !SkipTiers.Contains(l.Tier)).ToList();
if (pickups.Count == 0)
{
await _game.KeyUp(InputSender.VK.MENU);
Log.Information("No loot labels in round {Round} (total detected: {Total}, filtered: {Filtered})",
round + 1, labels.Count, labels.Count - pickups.Count);
Log.Information("No loot labels in round {Round} (picked {Total} total)", round + 1, totalPicked);
break;
}
Log.Information("Round {Round}: {Count} loot labels ({Skipped} skipped)",
round + 1, pickups.Count, labels.Count - pickups.Count);
foreach (var skip in labels.Where(l => SkipTiers.Contains(l.Tier)))
Log.Debug("Skipped: tier={Tier} color=({R},{G},{B}) at ({X},{Y})",
skip.Tier, skip.AvgR, skip.AvgG, skip.AvgB, skip.CenterX, skip.CenterY);
// Click each label center (Alt still held so labels visible)
// Click all detected labels (positions are stable)
foreach (var label in pickups)
{
if (_stopped) break;
Log.Information("Picking up: tier={Tier} color=({R},{G},{B}) at ({X},{Y})",
label.Tier, label.AvgR, label.AvgG, label.AvgB, label.CenterX, label.CenterY);
await _game.LeftClickAt(label.CenterX, label.CenterY);
await Helpers.Sleep(300);
totalPicked++;
await Helpers.Sleep(200);
}
// Quick check: capture small region around each clicked label to see if
// new labels appeared underneath. If none changed, we're done.
await Helpers.Sleep(300);
_game.MoveMouseInstant(0, 1440);
await Helpers.Sleep(100);
using var recheck = _screen.CaptureRawBitmap();
var newLabels = _screen.DetectLootLabels(recheck, recheck);
var newPickups = newLabels.Where(l => !SkipTiers.Contains(l.Tier)).ToList();
await _game.KeyUp(InputSender.VK.MENU);
await Helpers.Sleep(500);
if (newPickups.Count == 0)
{
Log.Information("Quick recheck: no new labels, done (picked {Total} total)", totalPicked);
break;
}
Log.Information("Quick recheck: {Count} new labels appeared, continuing", newPickups.Count);
await Helpers.Sleep(300);
}
Log.Information("Loot pickup complete");
Log.Information("Loot pickup complete ({Count} items)", totalPicked);
}
// ------ Recovery ------

View file

@ -20,5 +20,6 @@ public interface IInventoryManager
Task<bool> SalvageItems(List<PlacedItem> items);
(bool[,] Grid, List<PlacedItem> Items, int Free) GetInventoryState();
Task ClickStashTab(StashTabInfo tab, StashTabInfo? parentFolder = null);
void ResetStashTabState();
Task GrabItemsFromStash(string layoutName, int maxItems, string? templatePath = null);
}

View file

@ -15,6 +15,8 @@ public class InventoryManager : IInventoryManager
private bool _atOwnHideout = true;
private string _sellerAccount = "";
private string? _currentFolder;
private string? _currentTab;
private readonly IGameController _game;
private readonly IScreenReader _screen;
private readonly IClientLogWatcher _logWatcher;
@ -344,7 +346,17 @@ public class InventoryManager : IInventoryManager
public async Task ClickStashTab(StashTabInfo tab, StashTabInfo? parentFolder = null)
{
if (parentFolder != null)
var folderName = parentFolder?.Name;
var tabName = tab.Name;
// Already on the right folder and tab — skip
if (_currentFolder == folderName && _currentTab == tabName)
{
Log.Debug("Already on tab '{Tab}' (folder '{Folder}'), skipping", tabName, folderName ?? "none");
return;
}
if (parentFolder != null && _currentFolder != folderName)
{
Log.Information("Clicking folder '{Folder}' at ({X},{Y})", parentFolder.Name, parentFolder.ClickX, parentFolder.ClickY);
await _game.LeftClickAt(parentFolder.ClickX, parentFolder.ClickY);
@ -354,6 +366,15 @@ public class InventoryManager : IInventoryManager
Log.Information("Clicking tab '{Tab}' at ({X},{Y})", tab.Name, tab.ClickX, tab.ClickY);
await _game.LeftClickAt(tab.ClickX, tab.ClickY);
await Helpers.Sleep(Delays.PostStashOpen);
_currentFolder = folderName;
_currentTab = tabName;
}
public void ResetStashTabState()
{
_currentFolder = null;
_currentTab = null;
}
public async Task GrabItemsFromStash(string layoutName, int maxItems, string? templatePath = null)

View file

@ -17,6 +17,7 @@ public interface IScreenReader : IDisposable
Task Snapshot();
Task<DiffOcrResponse> DiffOcr(string? savePath = null, Region? region = null);
Task<TemplateMatchResult?> TemplateMatch(string templatePath, Region? region = null);
Task<List<TemplateMatchResult>> TemplateMatchAll(string templatePath, Region? region = null, double threshold = 0.7);
Task<OcrResponse> NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);
void SetLootBaseline(System.Drawing.Bitmap frame);
List<LootLabel> DetectLootLabels(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);

View file

@ -74,7 +74,17 @@ class PythonOcrBridge : IDisposable
{
_proc!.StandardInput.WriteLine(json);
_proc.StandardInput.Flush();
responseLine = _proc.StandardOutput.ReadLine()
// Read with timeout to prevent indefinite hang
var readTask = Task.Run(() => _proc.StandardOutput.ReadLine());
if (!readTask.Wait(TimeSpan.FromSeconds(15)))
{
Log.Warning("Python OCR daemon timed out after 15s, restarting");
try { _proc.Kill(); } catch { /* best effort */ }
_proc = null;
throw new TimeoutException("Python OCR daemon timed out");
}
responseLine = readTask.Result
?? throw new Exception("Python daemon returned null");
}

View file

@ -52,21 +52,28 @@ public class ScreenReader : IScreenReader
public Task<OcrResponse> Ocr(Region? region = null, string? preprocess = null)
{
var sw = System.Diagnostics.Stopwatch.StartNew();
using var bitmap = ScreenCapture.CaptureOrLoad(null, region);
OcrResponse result;
if (preprocess == "tophat")
{
using var processed = ImagePreprocessor.PreprocessForOcr(bitmap);
return Task.FromResult(_pythonBridge.OcrFromBitmap(processed));
result = _pythonBridge.OcrFromBitmap(processed);
}
if (preprocess == "clahe")
else if (preprocess == "clahe")
{
using var processed = ImagePreprocessor.PreprocessClahe(bitmap);
return Task.FromResult(_pythonBridge.OcrFromBitmap(processed));
result = _pythonBridge.OcrFromBitmap(processed);
}
else
{
result = _pythonBridge.OcrFromBitmap(bitmap);
}
return Task.FromResult(_pythonBridge.OcrFromBitmap(bitmap));
var allText = string.Join(" | ", result.Lines.Select(l => l.Text));
Log.Information("OCR completed in {Ms}ms ({Lines} lines): {Text}", sw.ElapsedMilliseconds, result.Lines.Count, allText);
return Task.FromResult(result);
}
public async Task<(int X, int Y)?> FindTextOnScreen(string searchText, bool fuzzy = false)
@ -170,6 +177,13 @@ public class ScreenReader : IScreenReader
return Task.FromResult(result);
}
public Task<List<TemplateMatchResult>> TemplateMatchAll(string templatePath, Region? region = null, double threshold = 0.7)
{
var results = _templateMatch.MatchAll(templatePath, region, threshold);
Log.Information("TemplateMatchAll: {Count} matches for {Template}", results.Count, Path.GetFileName(templatePath));
return Task.FromResult(results);
}
// -- Save --
public Task SaveScreenshot(string path)
@ -229,8 +243,9 @@ public class ScreenReader : IScreenReader
var allLines = new List<OcrLine>();
var allText = new List<string>();
foreach (var box in boxes)
for (int bi = 0; bi < boxes.Count; bi++)
{
var box = boxes[bi];
// Pad the crop slightly
int pad = 4;
int cx = Math.Max(0, box.X - pad);
@ -239,7 +254,19 @@ public class ScreenReader : IScreenReader
int ch = Math.Min(h - cy, box.Height + pad * 2);
using var crop = current.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb);
var ocrResult = _pythonBridge.OcrFromBitmap(crop);
var clusterSw = System.Diagnostics.Stopwatch.StartNew();
OcrResponse ocrResult;
try
{
ocrResult = _pythonBridge.OcrFromBitmap(crop);
}
catch (TimeoutException)
{
Log.Warning("NameplateDiffOcr: cluster {I}/{Count} OCR timed out, skipping", bi + 1, boxes.Count);
continue;
}
Log.Debug("NameplateDiffOcr: cluster {I}/{Count} ({W}x{H}) OCR took {Ms}ms",
bi + 1, boxes.Count, cw, ch, clusterSw.ElapsedMilliseconds);
// Offset word coordinates to screen space
foreach (var line in ocrResult.Lines)
@ -322,78 +349,301 @@ public class ScreenReader : IScreenReader
return boxes;
}
// -- Loot label detection (magenta background) --
// -- Loot label detection (Canny edge + contour) --
//
// All loot labels: white border, magenta (255,0,255) background, black text.
// Magenta never appears in the game world → detect directly, no diff needed.
// Finds rectangular contours from Canny edges, filters by shape
// (aspect ratio, size, rectangularity) and content (label interior
// must have visible color/brightness, unlike the dark game world).
// Single frame — no diff needed, no custom filter colors required.
public void SetLootBaseline(Bitmap frame) { }
// Detection parameters
// -- Loot detection constants --
private const int CannyLow = 20, CannyHigh = 80;
// Shape constraints
private const int LabelMinW = 80, LabelMaxW = 500;
private const int LabelMinH = 15, LabelMaxH = 100;
private const double LabelMinAspect = 1.3, LabelMaxAspect = 10.0;
// Strict pass: well-formed rectangle contours
private const double MinRectangularity = 0.5;
private const float StrictMinBS = 200f;
private const float StrictMinEdgeDensity = 25f;
// Relaxed pass: any contour bbox in play area (catches VFX-broken borders)
private const int RelaxedMinW = 100;
private const float RelaxedMinBS = 250f;
private const float RelaxedMinEdgeDensity = 25f;
private const double UiMarginTop = 0.08;
private const double UiMarginBottom = 0.82;
// Post-processing
private const int MergeGap = 30;
private const int MergeYTolerance = 15;
private const double NmsIouThresh = 0.4;
/// <summary>
/// Two-pass loot label detection:
/// 1. Strict: polygon-approximated rectangle contours (high precision)
/// 2. Relaxed: any contour bbox in play area (catches VFX-broken borders)
/// Results merged, horizontal fragments joined, then NMS.
/// </summary>
public List<LootLabel> DetectLootLabels(Bitmap reference, Bitmap current)
{
using var mat = BitmapConverter.ToMat(current);
if (mat.Channels() == 4)
Cv2.CvtColor(mat, mat, ColorConversionCodes.BGRA2BGR);
// Mask magenta background pixels (BGR: B≈255, G≈0, R≈255)
using var mask = new Mat();
Cv2.InRange(mat, new Scalar(200, 0, 200), new Scalar(255, 60, 255), mask);
int imgH = mat.Height, imgW = mat.Width;
int playTop = (int)(imgH * UiMarginTop);
int playBot = (int)(imgH * UiMarginBottom);
// Morph close fills text gaps within a label
// Height=2 bridges line gaps within multi-line labels but not between separate labels
using var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(12, 2));
using var closed = new Mat();
Cv2.MorphologyEx(mask, closed, MorphTypes.Close, kernel);
using var gray = new Mat();
Cv2.CvtColor(mat, gray, ColorConversionCodes.BGR2GRAY);
// Save debug images
using var hsv = new Mat();
Cv2.CvtColor(mat, hsv, ColorConversionCodes.BGR2HSV);
// Edge detection
using var edges = new Mat();
Cv2.Canny(gray, edges, CannyLow, CannyHigh);
using var dilateKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3));
using var dilated = new Mat();
Cv2.Dilate(edges, dilated, dilateKernel, iterations: 1);
Cv2.FindContours(dilated, out var contours, out _, RetrievalModes.Tree, ContourApproximationModes.ApproxSimple);
var strict = new List<LabelCandidate>();
var relaxed = new List<LabelCandidate>();
foreach (var contour in contours)
{
var box = Cv2.BoundingRect(contour);
// Common shape gate
if (box.Width <= LabelMinW || box.Width >= LabelMaxW) continue;
if (box.Height <= LabelMinH || box.Height >= LabelMaxH) continue;
double aspect = (double)box.Width / Math.Max(box.Height, 1);
if (aspect <= LabelMinAspect || aspect >= LabelMaxAspect) continue;
// Content metrics
using var roiHsv = new Mat(hsv, box);
var meanHsv = Cv2.Mean(roiHsv);
float meanVal = (float)meanHsv[2];
float meanSat = (float)meanHsv[1];
float bs = meanVal + meanSat;
using var roiGray = new Mat(gray, box);
using var roiEdges = new Mat();
Cv2.Canny(roiGray, roiEdges, 50, 150);
float ed = (float)Cv2.Mean(roiEdges)[0];
// Strict pass: well-formed polygon (4-8 vertices)
var peri = Cv2.ArcLength(contour, true);
var approx = Cv2.ApproxPolyDP(contour, 0.02 * peri, true);
if (approx.Length >= 4 && approx.Length <= 8)
{
double contourArea = Cv2.ContourArea(approx);
double rect = contourArea / Math.Max(box.Width * box.Height, 1);
if (rect >= MinRectangularity && bs >= StrictMinBS && ed >= StrictMinEdgeDensity)
strict.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat));
}
// Relaxed pass: any contour bbox in play area
bool inPlay = box.Y > playTop && box.Y + box.Height < playBot;
if (inPlay && box.Width >= RelaxedMinW && bs >= RelaxedMinBS && ed >= RelaxedMinEdgeDensity)
relaxed.Add(new LabelCandidate(box.X, box.Y, box.Width, box.Height, meanVal, meanSat));
}
// Merge strict + relaxed (strict wins on overlap)
var merged = new List<LabelCandidate>(strict);
foreach (var rlb in relaxed)
{
if (!OverlapsAny(rlb, strict, 0.3))
merged.Add(rlb);
}
// Join horizontal fragments
merged = MergeHorizontal(merged, MergeGap, MergeYTolerance);
// Build LootLabels with color classification
var scored = new List<(LootLabel Label, float Score)>();
foreach (var c in merged)
{
var (avgR, avgG, avgB) = SampleLabelColor(mat, c.X, c.Y, c.W, c.H);
var tier = LootColorClassifier.Classify(avgR, avgG, avgB);
int cx = c.X + c.W / 2;
int cy = c.Y + c.H / 2;
scored.Add((new LootLabel(cx, cy, c.W, c.H, tier, avgR, avgG, avgB),
c.MeanBrightness + c.MeanSaturation));
}
// Final NMS
var labels = NmsLootLabels(scored, NmsIouThresh);
labels.Sort((a, b) => a.CenterY.CompareTo(b.CenterY));
// Debug images
try
{
Cv2.ImWrite("debug_loot_mask.png", mask);
Cv2.ImWrite("debug_loot_closed.png", closed);
current.Save("debug_loot_capture.png", System.Drawing.Imaging.ImageFormat.Png);
Log.Information("Saved debug images: debug_loot_mask.png, debug_loot_closed.png, debug_loot_capture.png");
Cv2.ImWrite("debug_loot_edges.png", edges);
Cv2.ImWrite("debug_loot_dilated.png", dilated);
using var debugMat = mat.Clone();
foreach (var label in labels)
Cv2.Rectangle(debugMat,
new Rect(label.CenterX - label.Width / 2, label.CenterY - label.Height / 2, label.Width, label.Height),
new Scalar(0, 255, 0), 2);
Cv2.ImWrite("debug_loot_detected.png", debugMat);
}
catch (Exception ex)
{
Log.Warning(ex, "Failed to save debug images");
}
Cv2.FindContours(closed, out var contours, out _,
RetrievalModes.External, ContourApproximationModes.ApproxSimple);
Log.Information("DetectLootLabels: {N} magenta contours", contours.Length);
const int minW = 40, maxW = 600;
const int minH = 8, maxH = 100;
const double minAspect = 1.5;
int yMax = mat.Height - 210;
var labels = new List<LootLabel>();
foreach (var contour in contours)
{
var box = Cv2.BoundingRect(contour);
double aspect = box.Height > 0 ? (double)box.Width / box.Height : 0;
if (box.Width < minW || box.Width > maxW ||
box.Height < minH || box.Height > maxH ||
aspect < minAspect ||
box.Y < 65 || box.Y + box.Height > yMax)
{
Log.Information("Rejected contour: ({X},{Y}) {W}x{H} aspect={Aspect:F1} yMax={YMax}",
box.X, box.Y, box.Width, box.Height, aspect, yMax);
continue;
}
int cx = box.X + box.Width / 2;
int cy = box.Y + box.Height / 2;
Log.Information("Label at ({X},{Y}) {W}x{H}", box.X, box.Y, box.Width, box.Height);
labels.Add(new LootLabel(cx, cy, box.Width, box.Height, "loot", 255, 0, 255));
}
Log.Information("DetectLootLabels: strict={Strict} relaxed={Relaxed} merged={Merged} final={Final}",
strict.Count, relaxed.Count, merged.Count, labels.Count);
foreach (var label in labels)
Log.Information(" Label ({X},{Y}) {W}x{H} color=({R},{G},{B}) tier={Tier}",
label.CenterX - label.Width / 2, label.CenterY - label.Height / 2,
label.Width, label.Height, label.AvgR, label.AvgG, label.AvgB, label.Tier);
return labels;
}
// -- Loot detection helpers --
private record struct LabelCandidate(int X, int Y, int W, int H, float MeanBrightness, float MeanSaturation);
private static (byte R, byte G, byte B) SampleLabelColor(Mat mat, int x, int y, int w, int h)
{
var roiX = x + w / 4;
var roiY = y + h / 4;
var roiW = Math.Min(mat.Cols - roiX, w / 2);
var roiH = Math.Min(mat.Rows - roiY, h / 2);
if (roiW <= 0 || roiH <= 0) return (0, 0, 0);
using var roi = new Mat(mat, new Rect(roiX, roiY, roiW, roiH));
var mean = Cv2.Mean(roi);
return ((byte)mean[2], (byte)mean[1], (byte)mean[0]);
}
private static bool OverlapsAny(LabelCandidate label, List<LabelCandidate> others, double iouThresh)
{
foreach (var o in others)
{
int ix1 = Math.Max(label.X, o.X), iy1 = Math.Max(label.Y, o.Y);
int ix2 = Math.Min(label.X + label.W, o.X + o.W);
int iy2 = Math.Min(label.Y + label.H, o.Y + o.H);
int inter = Math.Max(0, ix2 - ix1) * Math.Max(0, iy2 - iy1);
int union = label.W * label.H + o.W * o.H - inter;
if (inter / (double)Math.Max(union, 1) > iouThresh)
return true;
}
return false;
}
/// <summary>
/// Merge labels that sit side-by-side on the same line.
/// </summary>
private static List<LabelCandidate> MergeHorizontal(List<LabelCandidate> labels, int gap, int yTol)
{
if (labels.Count < 2) return labels;
var used = new bool[labels.Count];
var indices = Enumerable.Range(0, labels.Count)
.OrderBy(i => labels[i].Y).ThenBy(i => labels[i].X).ToList();
var result = new List<LabelCandidate>();
for (int ii = 0; ii < indices.Count; ii++)
{
int i = indices[ii];
if (used[i]) continue;
used[i] = true;
var a = labels[i];
int gx1 = a.X, gy1 = a.Y, gx2 = a.X + a.W, gy2 = a.Y + a.H;
double briArea = a.MeanBrightness * a.W * a.H;
double satArea = a.MeanSaturation * a.W * a.H;
int totalArea = a.W * a.H;
bool changed = true;
while (changed)
{
changed = false;
for (int jj = 0; jj < indices.Count; jj++)
{
int j = indices[jj];
if (used[j]) continue;
var b = labels[j];
double cyA = (gy1 + gy2) / 2.0;
double cyB = b.Y + b.H / 2.0;
if (Math.Abs(cyA - cyB) > yTol) continue;
int hGap = Math.Max(b.X - gx2, gx1 - (b.X + b.W));
if (hGap > gap) continue;
int bArea = b.W * b.H;
gx1 = Math.Min(gx1, b.X);
gy1 = Math.Min(gy1, b.Y);
gx2 = Math.Max(gx2, b.X + b.W);
gy2 = Math.Max(gy2, b.Y + b.H);
briArea += b.MeanBrightness * bArea;
satArea += b.MeanSaturation * bArea;
totalArea += bArea;
used[j] = true;
changed = true;
}
}
int w = gx2 - gx1, h = gy2 - gy1;
float bri = (float)(briArea / Math.Max(totalArea, 1));
float sat = (float)(satArea / Math.Max(totalArea, 1));
result.Add(new LabelCandidate(gx1, gy1, w, h, bri, sat));
}
return result;
}
private static List<LootLabel> NmsLootLabels(List<(LootLabel Label, float Score)> candidates, double iouThresh)
{
if (candidates.Count == 0) return [];
candidates.Sort((a, b) => b.Score.CompareTo(a.Score));
var keep = new List<LootLabel>();
var suppressed = new bool[candidates.Count];
for (int i = 0; i < candidates.Count; i++)
{
if (suppressed[i]) continue;
keep.Add(candidates[i].Label);
var a = candidates[i].Label;
int ax1 = a.CenterX - a.Width / 2, ay1 = a.CenterY - a.Height / 2;
int ax2 = ax1 + a.Width, ay2 = ay1 + a.Height;
int areaA = a.Width * a.Height;
for (int j = i + 1; j < candidates.Count; j++)
{
if (suppressed[j]) continue;
var b = candidates[j].Label;
int bx1 = b.CenterX - b.Width / 2, by1 = b.CenterY - b.Height / 2;
int bx2 = bx1 + b.Width, by2 = by1 + b.Height;
int areaB = b.Width * b.Height;
int ix1 = Math.Max(ax1, bx1), iy1 = Math.Max(ay1, by1);
int ix2 = Math.Min(ax2, bx2), iy2 = Math.Min(ay2, by2);
int inter = Math.Max(0, ix2 - ix1) * Math.Max(0, iy2 - iy1);
double iou = inter / (double)(areaA + areaB - inter + 1);
if (iou >= iouThresh)
suppressed[j] = true;
}
}
return keep;
}
public void Dispose() => _pythonBridge.Dispose();
// -- OCR text matching --

View file

@ -50,6 +50,65 @@ class TemplateMatchHandler
return best;
}
/// <summary>
/// Find all matches above threshold, suppressing overlapping detections.
/// </summary>
public List<TemplateMatchResult> MatchAll(string templatePath, Region? region = null,
double threshold = 0.7)
{
if (!System.IO.File.Exists(templatePath))
throw new FileNotFoundException($"Template file not found: {templatePath}");
using var screenshot = ScreenCapture.CaptureOrLoad(null, region);
using var screenMat = BitmapConverter.ToMat(screenshot);
using var template = Cv2.ImRead(templatePath, ImreadModes.Color);
if (template.Empty())
throw new InvalidOperationException($"Failed to load template image: {templatePath}");
using var screenBgr = new Mat();
if (screenMat.Channels() == 4)
Cv2.CvtColor(screenMat, screenBgr, ColorConversionCodes.BGRA2BGR);
else
screenMat.CopyTo(screenBgr);
if (template.Rows > screenBgr.Rows || template.Cols > screenBgr.Cols)
return [];
using var result = new Mat();
Cv2.MatchTemplate(screenBgr, template, result, TemplateMatchModes.CCoeffNormed);
var offsetX = region?.X ?? 0;
var offsetY = region?.Y ?? 0;
var matches = new List<TemplateMatchResult>();
// Find all peaks above threshold using non-maximum suppression
while (true)
{
Cv2.MinMaxLoc(result, out _, out double maxVal, out _, out OpenCvSharp.Point maxLoc);
if (maxVal < threshold) break;
matches.Add(new TemplateMatchResult
{
X = offsetX + maxLoc.X + template.Cols / 2,
Y = offsetY + maxLoc.Y + template.Rows / 2,
Width = template.Cols,
Height = template.Rows,
Confidence = maxVal,
});
// Suppress this region so we find the next match
var suppressX = Math.Max(0, maxLoc.X - template.Cols / 2);
var suppressY = Math.Max(0, maxLoc.Y - template.Rows / 2);
var suppressW = Math.Min(result.Cols - suppressX, template.Cols);
var suppressH = Math.Min(result.Rows - suppressY, template.Rows);
using var roi = new Mat(result, new Rect(suppressX, suppressY, suppressW, suppressH));
roi.SetTo(new Scalar(0));
}
return matches;
}
private static TemplateMatchResult? MatchAtScale(Mat screen, Mat template,
Region? region, double scale, double threshold)
{