jpnadas/finrod
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ocr): adaptive binarisation + per-word confidence filter

Browse Source 
Two layers to stop tesseract hallucinating text on phone photos:

1. Bradley adaptive threshold (O(N) via integral images) turns the
   downscaled grayscale image into a binary B/W mask. Cuts out
   gradients, shadows, and surface texture that tesseract
   misreads as ink.
2. After recognise(), filter data.words by confidence >= 60 and
   re-assemble lines from kept words. Status line surfaces how
   many were dropped so the cost is visible.

Output of preprocess stage is PNG (not JPEG) so the sharp B/W
edges don't get re-blurred.
This commit is contained in:
João Pedro Battistella Nadas
2026-06-09 18:54:27 +02:00
parent dcac01e51b
commit 4d4e351860
2 changed files with 92 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@@ -10,6 +10,8 @@ Versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Added
- "Scan image" button next to the input pane: opens the phone's rear camera (or a file picker on desktop), runs OCR locally in the browser via Tesseract.js (loaded from esm.sh on first use), and replaces the editor contents with the recognised text. Language follows the "From" dropdown via an ISO 639-2 mapping; "Auto-detect" loads a common European set (`eng+por+deu+nld`). Progress status shows beneath the input ("Loading OCR engine…" → "OCR 40%" → "OCR done in 3.2s"). Zero image-size cost — same CDN pattern as CodeMirror.
- Images are downscaled to 1600 px on the long edge before OCR. A 12 MP phone photo (~4000 px) drops to ~1.3 MP, making Tesseract 4-8× faster with no real accuracy loss for printed text. Smaller-than-1600 px inputs are passed through unchanged.
- Adaptive thresholding (Bradley's method with integral images) binarises the downscaled image before OCR — drops hallucinated text from shadows, gradients, and surface texture on phone photos. Output is pure black-on-white, much closer to the scanned-page input Tesseract was trained on.
- Per-word confidence filter (≥ 60) drops the low-confidence noise Tesseract still emits. Re-assembles lines from kept words; status line reports how many words were dropped.
### Fixed
- "Hot: …" status no longer claims "unloads in X" when the server's `keep_alive` is `-1` (Ollama returns an absurd year-4001 `expires_at`). Threshold: any expiry >7 days drops the suffix and just shows the model name.

108
static/index.html
View File

@@ -329,30 +329,97 @@
scanStatus.classList.toggle("error", error);
}
// Downscale to at most OCR_MAX_DIM on the long edge. A typical phone
// photo is 4000+ px on the long edge — Tesseract is 4-8× faster at
// 1600 px with no real accuracy loss for printed text.
// Downscale to at most OCR_MAX_DIM on the long edge AND binarise with
// Bradley's adaptive threshold. Phone photos have gradients, shadows
// and texture that Tesseract trained on scanned pages doesn't expect,
// so it hallucinates text. Adaptive thresholding turns the input into
// a binary "ink vs paper" mask which is what the recogniser wants.
const OCR_MAX_DIM = 1600;
async function downscaleForOCR(file) {
const OCR_CONFIDENCE_MIN = 60; // word-level confidence (0-100)
async function preprocessForOCR(file) {
const bitmap = await createImageBitmap(file);
const long = Math.max(bitmap.width, bitmap.height);
if (long <= OCR_MAX_DIM) {
bitmap.close?.();
return { blob: file, w: bitmap.width, h: bitmap.height, scaled: false };
}
const scale = OCR_MAX_DIM / long;
const scale = Math.min(1, OCR_MAX_DIM / long);
const w = Math.round(bitmap.width * scale);
const h = Math.round(bitmap.height * scale);
const canvas = document.createElement("canvas");
canvas.width = w;
canvas.height = h;
const ctx = canvas.getContext("2d");
ctx.drawImage(bitmap, 0, 0, w, h);
bitmap.close?.();
const blob = await new Promise(resolve =>
canvas.toBlob(resolve, "image/jpeg", 0.85)
);
return { blob, w, h, scaled: true, origLong: long };
binarizeAdaptive(ctx, w, h);
// PNG, not JPEG — JPEG would re-blur the freshly-sharp B/W edges.
const blob = await new Promise(resolve => canvas.toBlob(resolve, "image/png"));
return { blob, w, h, scaled: scale < 1, origLong: long };
}
// Bradley/Roth adaptive threshold: compare each pixel against the
// mean of a local window. Computed in O(N) total via an integral
// image. Output is a 3-channel B/W image (R=G=B=0 or 255).
function binarizeAdaptive(ctx, w, h) {
const imgData = ctx.getImageData(0, 0, w, h);
const data = imgData.data;
// 1. Grayscale via BT.601 luminance
const gray = new Uint8Array(w * h);
for (let i = 0, j = 0; i < data.length; i += 4, j++) {
gray[j] = (data[i] * 0.299 + data[i + 1] * 0.587 + data[i + 2] * 0.114) | 0;
}
// 2. Integral image (summed-area table)
const integral = new Float64Array(w * h);
for (let y = 0; y < h; y++) {
let rowSum = 0;
const yw = y * w;
for (let x = 0; x < w; x++) {
rowSum += gray[yw + x];
integral[yw + x] = rowSum + (y > 0 ? integral[yw - w + x] : 0);
}
}
// 3. Threshold: pixel becomes black if it's > T% below the local mean
const winSize = Math.max(15, Math.floor(w / 32)) | 1; // odd window
const halfWin = (winSize - 1) >> 1;
const T = 0.15;
for (let y = 0; y < h; y++) {
const yw = y * w;
const y1 = Math.max(0, y - halfWin);
const y2 = Math.min(h - 1, y + halfWin);
for (let x = 0; x < w; x++) {
const x1 = Math.max(0, x - halfWin);
const x2 = Math.min(w - 1, x + halfWin);
const count = (x2 - x1 + 1) * (y2 - y1 + 1);
const A = (x1 > 0 && y1 > 0) ? integral[(y1 - 1) * w + (x1 - 1)] : 0;
const B = (y1 > 0) ? integral[(y1 - 1) * w + x2] : 0;
const C = (x1 > 0) ? integral[y2 * w + (x1 - 1)] : 0;
const D = integral[y2 * w + x2];
const mean = (D - B - C + A) / count;
const black = gray[yw + x] < mean * (1 - T) ? 0 : 255;
const idx = (yw + x) * 4;
data[idx] = data[idx + 1] = data[idx + 2] = black;
}
}
ctx.putImageData(imgData, 0, 0);
}
// Drop words below confidence threshold; re-assemble preserving line breaks.
function filterByConfidence(result, threshold = OCR_CONFIDENCE_MIN) {
const lines = result.lines || [];
if (!lines.length) return result.text || "";
const keptLines = lines.map(line => {
const words = (line.words || [])
.filter(w => (w.confidence ?? 100) >= threshold)
.map(w => w.text);
return words.length ? words.join(" ") : "";
});
// Collapse consecutive blank lines so dropped runs don't leave gaps.
return keptLines.join("\n").replace(/\n{3,}/g, "\n\n").trim();
}
async function runOCR(file) {
@@ -364,9 +431,11 @@
const t0 = performance.now();
try {
const { blob, w, h, scaled, origLong } = await downscaleForOCR(file);
const { blob, w, h, scaled, origLong } = await preprocessForOCR(file);
if (scaled) {
setScanStatus(`Downscaled ${origLong}px → ${Math.max(w, h)}px`);
setScanStatus(`Preprocessed ${origLong}px → ${Math.max(w, h)}px (binarised)`);
} else {
setScanStatus(`Preprocessed ${Math.max(w, h)}px (binarised)`);
}
// Lazy import — first scan triggers the ~5 MB JS + language pack
@@ -381,15 +450,18 @@
}
},
});
const { data: { text } } = await worker.recognize(blob);
const { data } = await worker.recognize(blob);
await worker.terminate();
const text = filterByConfidence(data);
const droppedWords = (data.words || []).filter(w => (w.confidence ?? 100) < OCR_CONFIDENCE_MIN).length;
editor.dispatch({
changes: { from: 0, to: editor.state.doc.length, insert: text.trim() },
changes: { from: 0, to: editor.state.doc.length, insert: text },
selection: { anchor: 0 },
});
const dt = ((performance.now() - t0) / 1000).toFixed(1);
setScanStatus(`OCR done in ${dt}s (${lang})`);
const dropped = droppedWords ? `, dropped ${droppedWords} low-confidence` : "";
setScanStatus(`OCR done in ${dt}s (${lang}${dropped})`);
} catch (e) {
console.error(e);
setScanStatus(`OCR failed: ${e.message || e}`, { error: true });