517 lines
20 KiB
C#
517 lines
20 KiB
C#
namespace OcrDaemon;
|
|
|
|
using System.Drawing;
|
|
using System.Drawing.Imaging;
|
|
using System.Runtime.InteropServices;
|
|
using System.Text.Json;
|
|
using Tesseract;
|
|
using SdImageFormat = System.Drawing.Imaging.ImageFormat;
|
|
|
|
class OcrHandler(TesseractEngine engine)
|
|
{
|
|
private Bitmap? _referenceFrame;
|
|
|
|
public object HandleOcr(Request req)
|
|
{
|
|
using var bitmap = ScreenCapture.CaptureOrLoad(req.File, req.Region);
|
|
using var pix = ImageUtils.BitmapToPix(bitmap);
|
|
using var page = engine.Process(pix);
|
|
|
|
var text = page.GetText();
|
|
var lines = ImageUtils.ExtractLinesFromPage(page, offsetX: 0, offsetY: 0);
|
|
return new OcrResponse { Text = text, Lines = lines };
|
|
}
|
|
|
|
public object HandleScreenshot(Request req)
|
|
{
|
|
if (string.IsNullOrEmpty(req.Path))
|
|
return new ErrorResponse("screenshot command requires 'path'");
|
|
|
|
// If a reference frame exists, save that (same image used for diff-ocr).
|
|
// Otherwise capture a new frame.
|
|
var bitmap = _referenceFrame ?? ScreenCapture.CaptureOrLoad(req.File, req.Region);
|
|
var format = ImageUtils.GetImageFormat(req.Path);
|
|
var dir = Path.GetDirectoryName(req.Path);
|
|
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
|
|
Directory.CreateDirectory(dir);
|
|
bitmap.Save(req.Path, format);
|
|
if (bitmap != _referenceFrame) bitmap.Dispose();
|
|
return new OkResponse();
|
|
}
|
|
|
|
public object HandleCapture(Request req)
|
|
{
|
|
using var bitmap = ScreenCapture.CaptureOrLoad(req.File, req.Region);
|
|
using var ms = new MemoryStream();
|
|
bitmap.Save(ms, SdImageFormat.Png);
|
|
var base64 = Convert.ToBase64String(ms.ToArray());
|
|
return new CaptureResponse { Image = base64 };
|
|
}
|
|
|
|
public object HandleSnapshot(Request req)
|
|
{
|
|
_referenceFrame?.Dispose();
|
|
_referenceFrame = ScreenCapture.CaptureOrLoad(req.File, req.Region);
|
|
return new OkResponse();
|
|
}
|
|
|
|
public object HandleDiffOcr(Request req) => HandleDiffOcr(req, new DiffOcrParams
|
|
{
|
|
DiffThresh = req.Threshold > 0 ? req.Threshold : 30,
|
|
});
|
|
|
|
public object HandleDiffOcr(Request req, DiffOcrParams p)
|
|
{
|
|
if (_referenceFrame == null)
|
|
return new ErrorResponse("No reference snapshot stored. Send 'snapshot' first.");
|
|
|
|
using var current = ScreenCapture.CaptureOrLoad(req.File, null);
|
|
|
|
int w = Math.Min(_referenceFrame.Width, current.Width);
|
|
int h = Math.Min(_referenceFrame.Height, current.Height);
|
|
|
|
// Get raw pixels for both frames
|
|
var refData = _referenceFrame.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
|
|
byte[] refPx = new byte[refData.Stride * h];
|
|
Marshal.Copy(refData.Scan0, refPx, 0, refPx.Length);
|
|
_referenceFrame.UnlockBits(refData);
|
|
int stride = refData.Stride;
|
|
|
|
var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
|
|
byte[] curPx = new byte[curData.Stride * h];
|
|
Marshal.Copy(curData.Scan0, curPx, 0, curPx.Length);
|
|
current.UnlockBits(curData);
|
|
|
|
// Detect pixels that got DARKER (tooltip = dark overlay).
|
|
// This filters out item highlight glow (brighter) and cursor changes.
|
|
int diffThresh = p.DiffThresh;
|
|
bool[] changed = new bool[w * h];
|
|
int totalChanged = 0;
|
|
|
|
for (int y = 0; y < h; y++)
|
|
{
|
|
for (int x = 0; x < w; x++)
|
|
{
|
|
int i = y * stride + x * 4;
|
|
int darkerB = refPx[i] - curPx[i];
|
|
int darkerG = refPx[i + 1] - curPx[i + 1];
|
|
int darkerR = refPx[i + 2] - curPx[i + 2];
|
|
if (darkerB + darkerG + darkerR > diffThresh)
|
|
{
|
|
changed[y * w + x] = true;
|
|
totalChanged++;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool debug = req.Debug;
|
|
|
|
if (totalChanged == 0)
|
|
{
|
|
if (debug) Console.Error.WriteLine(" diff-ocr: no changes detected");
|
|
return new OcrResponse { Text = "", Lines = [] };
|
|
}
|
|
|
|
// Two-pass density detection:
|
|
// Pass 1: Find row range using full-width row counts
|
|
// Pass 2: Find column range using only pixels within detected row range
|
|
// This makes the column threshold relative to tooltip height, not screen height.
|
|
int maxGap = p.MaxGap;
|
|
|
|
// Pass 1: count changed pixels per row, find longest active run
|
|
int[] rowCounts = new int[h];
|
|
for (int y = 0; y < h; y++)
|
|
for (int x = 0; x < w; x++)
|
|
if (changed[y * w + x])
|
|
rowCounts[y]++;
|
|
|
|
int rowThresh = w / p.RowThreshDiv;
|
|
int bestRowStart = 0, bestRowEnd = 0, bestRowLen = 0;
|
|
int curRowStart = -1, lastActiveRow = -1;
|
|
for (int y = 0; y < h; y++)
|
|
{
|
|
if (rowCounts[y] >= rowThresh)
|
|
{
|
|
if (curRowStart < 0) curRowStart = y;
|
|
lastActiveRow = y;
|
|
}
|
|
else if (curRowStart >= 0 && y - lastActiveRow > maxGap)
|
|
{
|
|
int len = lastActiveRow - curRowStart + 1;
|
|
if (len > bestRowLen) { bestRowStart = curRowStart; bestRowEnd = lastActiveRow; bestRowLen = len; }
|
|
curRowStart = -1;
|
|
}
|
|
}
|
|
if (curRowStart >= 0)
|
|
{
|
|
int len = lastActiveRow - curRowStart + 1;
|
|
if (len > bestRowLen) { bestRowStart = curRowStart; bestRowEnd = lastActiveRow; bestRowLen = len; }
|
|
}
|
|
|
|
// Pass 2: count changed pixels per column, but only within the detected row range
|
|
int[] colCounts = new int[w];
|
|
for (int y = bestRowStart; y <= bestRowEnd; y++)
|
|
for (int x = 0; x < w; x++)
|
|
if (changed[y * w + x])
|
|
colCounts[x]++;
|
|
|
|
int tooltipHeight = bestRowEnd - bestRowStart + 1;
|
|
int colThresh = tooltipHeight / p.ColThreshDiv;
|
|
|
|
int bestColStart = 0, bestColEnd = 0, bestColLen = 0;
|
|
int curColStart = -1, lastActiveCol = -1;
|
|
for (int x = 0; x < w; x++)
|
|
{
|
|
if (colCounts[x] >= colThresh)
|
|
{
|
|
if (curColStart < 0) curColStart = x;
|
|
lastActiveCol = x;
|
|
}
|
|
else if (curColStart >= 0 && x - lastActiveCol > maxGap)
|
|
{
|
|
int len = lastActiveCol - curColStart + 1;
|
|
if (len > bestColLen) { bestColStart = curColStart; bestColEnd = lastActiveCol; bestColLen = len; }
|
|
curColStart = -1;
|
|
}
|
|
}
|
|
if (curColStart >= 0)
|
|
{
|
|
int len = lastActiveCol - curColStart + 1;
|
|
if (len > bestColLen) { bestColStart = curColStart; bestColEnd = lastActiveCol; bestColLen = len; }
|
|
}
|
|
|
|
// Log density detection results
|
|
Console.Error.WriteLine($" diff-ocr: changed={totalChanged} rows={bestRowStart}-{bestRowEnd}({bestRowLen}) cols={bestColStart}-{bestColEnd}({bestColLen}) rowThresh={rowThresh} colThresh={colThresh}");
|
|
|
|
if (bestRowLen < 50 || bestColLen < 50)
|
|
{
|
|
Console.Error.WriteLine($" diff-ocr: no tooltip-sized region found (rows={bestRowLen}, cols={bestColLen})");
|
|
return new OcrResponse { Text = "", Lines = [] };
|
|
}
|
|
|
|
int pad = 0;
|
|
int minX = Math.Max(bestColStart - pad, 0);
|
|
int minY = Math.Max(bestRowStart - pad, 0);
|
|
int maxX = Math.Min(bestColEnd + pad, w - 1);
|
|
int maxY = Math.Min(bestRowEnd + pad, h - 1);
|
|
|
|
// Dynamic right-edge trim: if the rightmost columns are much sparser than
|
|
// the tooltip body, trim them. This handles the ~5% of cases where ambient
|
|
// noise extends the detected region slightly on the right.
|
|
int colSpan = maxX - minX + 1;
|
|
if (colSpan > 100)
|
|
{
|
|
// Compute median column density in the middle 50% of the range
|
|
int q1 = minX + colSpan / 4;
|
|
int q3 = minX + colSpan * 3 / 4;
|
|
long midSum = 0;
|
|
int midCount = 0;
|
|
for (int x = q1; x <= q3; x++) { midSum += colCounts[x]; midCount++; }
|
|
double avgMidDensity = (double)midSum / midCount;
|
|
double cutoff = avgMidDensity * p.TrimCutoff;
|
|
|
|
// Trim from right while below cutoff
|
|
while (maxX > minX + 100 && colCounts[maxX] < cutoff)
|
|
maxX--;
|
|
}
|
|
int rw = maxX - minX + 1;
|
|
int rh = maxY - minY + 1;
|
|
|
|
if (debug) Console.Error.WriteLine($" diff-ocr: tooltip region ({minX},{minY}) {rw}x{rh}");
|
|
|
|
// Simple crop of the tooltip region from the current frame (no per-pixel masking).
|
|
// The top-hat preprocessing will handle suppressing background text.
|
|
using var cropped = current.Clone(new Rectangle(minX, minY, rw, rh), PixelFormat.Format32bppArgb);
|
|
|
|
// Save before/after preprocessing images if path is provided
|
|
if (!string.IsNullOrEmpty(req.Path))
|
|
{
|
|
var dir = Path.GetDirectoryName(req.Path);
|
|
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
|
|
Directory.CreateDirectory(dir);
|
|
cropped.Save(req.Path, ImageUtils.GetImageFormat(req.Path));
|
|
if (debug) Console.Error.WriteLine($" diff-ocr: saved raw to {req.Path}");
|
|
}
|
|
|
|
// Pre-process for OCR: top-hat + binarize + upscale
|
|
using var processed = ImagePreprocessor.PreprocessForOcr(cropped, p.KernelSize, p.Upscale);
|
|
|
|
// Save fullscreen and preprocessed versions alongside raw
|
|
if (!string.IsNullOrEmpty(req.Path))
|
|
{
|
|
var ext = Path.GetExtension(req.Path);
|
|
var fullPath = Path.ChangeExtension(req.Path, ".full" + ext);
|
|
current.Save(fullPath, ImageUtils.GetImageFormat(fullPath));
|
|
if (debug) Console.Error.WriteLine($" diff-ocr: saved fullscreen to {fullPath}");
|
|
|
|
var prePath = Path.ChangeExtension(req.Path, ".pre" + ext);
|
|
processed.Save(prePath, ImageUtils.GetImageFormat(prePath));
|
|
if (debug) Console.Error.WriteLine($" diff-ocr: saved preprocessed to {prePath}");
|
|
}
|
|
using var pix = ImageUtils.BitmapToPix(processed);
|
|
using var page = engine.Process(pix);
|
|
|
|
var text = page.GetText();
|
|
var lines = ImageUtils.ExtractLinesFromPage(page, offsetX: minX, offsetY: minY);
|
|
|
|
return new DiffOcrResponse
|
|
{
|
|
Text = text,
|
|
Lines = lines,
|
|
Region = new RegionRect { X = minX, Y = minY, Width = rw, Height = rh },
|
|
};
|
|
}
|
|
|
|
public object HandleTest(Request req) => RunTestCases(new DiffOcrParams(), verbose: true);
|
|
|
|
public object HandleTune(Request req)
|
|
{
|
|
// Coordinate descent: optimize one parameter at a time, repeat until stable.
|
|
var best = new DiffOcrParams();
|
|
double bestScore = ScoreParams(best);
|
|
Console.Error.WriteLine($"\n=== Tuning start === baseline score={bestScore:F3} {best}\n");
|
|
|
|
// Define search ranges for each parameter
|
|
var sweeps = new (string Name, int[] Values, Action<DiffOcrParams, int> Set)[]
|
|
{
|
|
("diffThresh", [10, 15, 20, 25, 30, 40, 50, 60], (p, v) => p.DiffThresh = v),
|
|
("rowThreshDiv", [10, 15, 20, 25, 30, 40, 50, 60], (p, v) => p.RowThreshDiv = v),
|
|
("colThreshDiv", [5, 8, 10, 12, 15, 20, 25, 30], (p, v) => p.ColThreshDiv = v),
|
|
("maxGap", [5, 8, 10, 12, 15, 20, 25, 30], (p, v) => p.MaxGap = v),
|
|
("kernelSize", [11, 15, 19, 21, 25, 31, 35, 41], (p, v) => p.KernelSize = v),
|
|
("upscale", [1, 2, 3], (p, v) => p.Upscale = v),
|
|
};
|
|
|
|
// trimCutoff needs double values — handle separately
|
|
double[] trimValues = [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5];
|
|
|
|
int totalEvals = 0;
|
|
const int maxRounds = 3;
|
|
|
|
for (int round = 0; round < maxRounds; round++)
|
|
{
|
|
bool improved = false;
|
|
Console.Error.WriteLine($"--- Round {round + 1} ---");
|
|
|
|
// Sweep integer params
|
|
foreach (var (name, values, set) in sweeps)
|
|
{
|
|
Console.Error.Write($" {name}: ");
|
|
int bestVal = 0;
|
|
double bestValScore = -1;
|
|
|
|
foreach (int v in values)
|
|
{
|
|
var trial = best.Clone();
|
|
set(trial, v);
|
|
double score = ScoreParams(trial);
|
|
totalEvals++;
|
|
Console.Error.Write($"{v}={score:F3} ");
|
|
|
|
if (score > bestValScore) { bestValScore = score; bestVal = v; }
|
|
}
|
|
Console.Error.WriteLine();
|
|
|
|
if (bestValScore > bestScore)
|
|
{
|
|
set(best, bestVal);
|
|
bestScore = bestValScore;
|
|
improved = true;
|
|
Console.Error.WriteLine($" → {name}={bestVal} score={bestScore:F3}");
|
|
}
|
|
}
|
|
|
|
// Sweep trimCutoff
|
|
{
|
|
Console.Error.Write($" trimCutoff: ");
|
|
double bestTrim = best.TrimCutoff;
|
|
double bestTrimScore = bestScore;
|
|
|
|
foreach (double v in trimValues)
|
|
{
|
|
var trial = best.Clone();
|
|
trial.TrimCutoff = v;
|
|
double score = ScoreParams(trial);
|
|
totalEvals++;
|
|
Console.Error.Write($"{v:F2}={score:F3} ");
|
|
|
|
if (score > bestTrimScore) { bestTrimScore = score; bestTrim = v; }
|
|
}
|
|
Console.Error.WriteLine();
|
|
|
|
if (bestTrimScore > bestScore)
|
|
{
|
|
best.TrimCutoff = bestTrim;
|
|
bestScore = bestTrimScore;
|
|
improved = true;
|
|
Console.Error.WriteLine($" → trimCutoff={bestTrim:F2} score={bestScore:F3}");
|
|
}
|
|
}
|
|
|
|
Console.Error.WriteLine($" End of round {round + 1}: score={bestScore:F3} {best}");
|
|
if (!improved) break;
|
|
}
|
|
|
|
Console.Error.WriteLine($"\n=== Tuning done === evals={totalEvals} bestScore={bestScore:F3}\n {best}\n");
|
|
|
|
// Run verbose test with best params for final report
|
|
var finalResult = RunTestCases(best, verbose: true);
|
|
|
|
return new TuneResponse
|
|
{
|
|
BestScore = bestScore,
|
|
BestParams = best,
|
|
Iterations = totalEvals,
|
|
};
|
|
}
|
|
|
|
/// <summary>Score a param set: average match ratio across all test cases (0-1).</summary>
|
|
private double ScoreParams(DiffOcrParams p)
|
|
{
|
|
var result = RunTestCases(p, verbose: false);
|
|
if (result is TestResponse tr && tr.Total > 0)
|
|
return tr.Results.Average(r => r.Score);
|
|
return 0;
|
|
}
|
|
|
|
private object RunTestCases(DiffOcrParams p, bool verbose)
|
|
{
|
|
var tessdataDir = Path.Combine(AppContext.BaseDirectory, "tessdata");
|
|
var casesPath = Path.Combine(tessdataDir, "cases.json");
|
|
if (!File.Exists(casesPath))
|
|
return new ErrorResponse($"cases.json not found at {casesPath}");
|
|
|
|
var json = File.ReadAllText(casesPath);
|
|
var cases = JsonSerializer.Deserialize<List<TestCase>>(json);
|
|
if (cases == null || cases.Count == 0)
|
|
return new ErrorResponse("No test cases found in cases.json");
|
|
|
|
var results = new List<TestCaseResult>();
|
|
int passCount = 0;
|
|
|
|
foreach (var tc in cases)
|
|
{
|
|
if (verbose) Console.Error.WriteLine($"\n=== Test: {tc.Id} ===");
|
|
|
|
var fullPath = Path.Combine(tessdataDir, tc.FullImage);
|
|
var imagePath = Path.Combine(tessdataDir, tc.Image);
|
|
|
|
if (!File.Exists(fullPath))
|
|
{
|
|
if (verbose) Console.Error.WriteLine($" SKIP: full image not found: {fullPath}");
|
|
results.Add(new TestCaseResult { Id = tc.Id, Passed = false, Score = 0, Missed = tc.Expected });
|
|
continue;
|
|
}
|
|
if (!File.Exists(imagePath))
|
|
{
|
|
if (verbose) Console.Error.WriteLine($" SKIP: tooltip image not found: {imagePath}");
|
|
results.Add(new TestCaseResult { Id = tc.Id, Passed = false, Score = 0, Missed = tc.Expected });
|
|
continue;
|
|
}
|
|
|
|
// Run the same pipeline: snapshot (reference) then diff-ocr (with tooltip)
|
|
HandleSnapshot(new Request { File = fullPath });
|
|
var diffResult = HandleDiffOcr(new Request { File = imagePath, Debug = verbose }, p);
|
|
|
|
// Extract actual lines from the response
|
|
List<string> actualLines;
|
|
if (diffResult is DiffOcrResponse diffResp)
|
|
actualLines = diffResp.Lines.Select(l => l.Text.Trim()).Where(l => l.Length > 0).ToList();
|
|
else if (diffResult is OcrResponse ocrResp)
|
|
actualLines = ocrResp.Lines.Select(l => l.Text.Trim()).Where(l => l.Length > 0).ToList();
|
|
else
|
|
{
|
|
if (verbose) Console.Error.WriteLine($" ERROR: unexpected response type");
|
|
results.Add(new TestCaseResult { Id = tc.Id, Passed = false, Score = 0, Missed = tc.Expected });
|
|
continue;
|
|
}
|
|
|
|
// Fuzzy match expected vs actual
|
|
var matched = new List<string>();
|
|
var missed = new List<string>();
|
|
var usedActual = new HashSet<int>();
|
|
|
|
foreach (var expected in tc.Expected)
|
|
{
|
|
int bestIdx = -1;
|
|
double bestSim = 0;
|
|
for (int i = 0; i < actualLines.Count; i++)
|
|
{
|
|
if (usedActual.Contains(i)) continue;
|
|
double sim = LevenshteinSimilarity(expected, actualLines[i]);
|
|
if (sim > bestSim) { bestSim = sim; bestIdx = i; }
|
|
}
|
|
|
|
if (bestIdx >= 0 && bestSim >= 0.75)
|
|
{
|
|
matched.Add(expected);
|
|
usedActual.Add(bestIdx);
|
|
if (verbose && bestSim < 1.0)
|
|
Console.Error.WriteLine($" ~ {expected} → {actualLines[bestIdx]} (sim={bestSim:F2})");
|
|
}
|
|
else
|
|
{
|
|
missed.Add(expected);
|
|
if (verbose)
|
|
Console.Error.WriteLine($" MISS: {expected}" + (bestIdx >= 0 ? $" (best: {actualLines[bestIdx]}, sim={bestSim:F2})" : ""));
|
|
}
|
|
}
|
|
|
|
var extra = actualLines.Where((_, i) => !usedActual.Contains(i)).ToList();
|
|
if (verbose)
|
|
foreach (var e in extra)
|
|
Console.Error.WriteLine($" EXTRA: {e}");
|
|
|
|
double score = tc.Expected.Count > 0 ? (double)matched.Count / tc.Expected.Count : 1.0;
|
|
bool passed = missed.Count == 0;
|
|
if (passed) passCount++;
|
|
|
|
if (verbose)
|
|
Console.Error.WriteLine($" Result: {(passed ? "PASS" : "FAIL")} matched={matched.Count}/{tc.Expected.Count} extra={extra.Count} score={score:F2}");
|
|
|
|
results.Add(new TestCaseResult
|
|
{
|
|
Id = tc.Id,
|
|
Passed = passed,
|
|
Score = score,
|
|
Matched = matched,
|
|
Missed = missed,
|
|
Extra = extra,
|
|
});
|
|
}
|
|
|
|
if (verbose)
|
|
Console.Error.WriteLine($"\n=== Summary: {passCount}/{cases.Count} passed ===\n");
|
|
|
|
return new TestResponse
|
|
{
|
|
Passed = passCount,
|
|
Failed = cases.Count - passCount,
|
|
Total = cases.Count,
|
|
Results = results,
|
|
};
|
|
}
|
|
|
|
private static double LevenshteinSimilarity(string a, string b)
|
|
{
|
|
a = a.ToLowerInvariant();
|
|
b = b.ToLowerInvariant();
|
|
if (a == b) return 1.0;
|
|
|
|
int la = a.Length, lb = b.Length;
|
|
if (la == 0 || lb == 0) return 0.0;
|
|
|
|
var d = new int[la + 1, lb + 1];
|
|
for (int i = 0; i <= la; i++) d[i, 0] = i;
|
|
for (int j = 0; j <= lb; j++) d[0, j] = j;
|
|
|
|
for (int i = 1; i <= la; i++)
|
|
for (int j = 1; j <= lb; j++)
|
|
{
|
|
int cost = a[i - 1] == b[j - 1] ? 0 : 1;
|
|
d[i, j] = Math.Min(Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1), d[i - 1, j - 1] + cost);
|
|
}
|
|
|
|
return 1.0 - (double)d[la, lb] / Math.Max(la, lb);
|
|
}
|
|
}
|