jpnadas/finrod
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ocr): live camera viewfinder + invert-colours toggle

Browse Source 
Four wins:

1. Replace <input type=file capture> with a getUserMedia
   viewfinder modal on mobile. Tapping the shutter goes straight
   to the crop modal — skips the OS "retake/use" confirm step.
   Falls back to file picker if camera permission is denied or
   the API is unavailable. "Pick from file" icon
   (photo_library) inside the camera modal opens the gallery
   on demand.

2. Drop the Reset button — with no default crop box, "reset" is
   just "no box again," which is the starting state anyway.

3. The check (Use this) button is now larger and more prominent
   — wider padding, 24px icon, accent fill.

4. Invert-colours toggle (invert_colors icon) in the crop modal.
   White-on-dark text (signs, menus, screenshots) breaks
   Tesseract because the binariser was made for ink on paper.
   Toggling the button flips the grayscale buffer before
   adaptive thresholding so the binary output ends up
   black-on-white. State plumbed cleanly: openCropper now
   resolves with {blob, invert}; preprocessForOCR takes an
   {invert} option; binarizeAdaptive applies the flip.

Also: drop `capture=environment` from the file <input>. Without
it the file path correctly opens the gallery/file dialog instead
of re-opening the OS camera when the user picks "Pick from file".
This commit is contained in:
João Pedro Battistella Nadas
2026-06-09 19:27:34 +02:00
parent cc249fb444
commit a63b11e467
2 changed files with 141 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
static/index.html
View File

@@ -269,13 +269,36 @@
.modal-btn .material-symbols-outlined { font-size: 20px; }
.modal-btn.icon { padding: 0.35rem 0.55rem; min-width: 2.4rem; }
.modal-btn:hover { border-color: var(--accent); color: var(--fg); }
.modal-btn.active {
background: var(--accent);
color: #fff;
border-color: var(--accent);
}
.modal-btn.primary {
background: var(--accent);
color: #fff;
border-color: var(--accent);
}
.modal-btn.primary.icon {
padding: 0.55rem 1.1rem;
min-width: 3.2rem;
}
.modal-btn.primary.icon .material-symbols-outlined { font-size: 24px; }
.modal-btn.primary:hover { color: #fff; }
/* Camera viewfinder modal */
.cam-body {
display: flex;
align-items: center;
justify-content: center;
padding: 0 !important;
}
.cam-body video {
max-width: 100%;
max-height: 70vh;
display: block;
}
/* CodeMirror editor host — matches .output styling so input & output align */
.cm-editor {
min-height: 180px;
@@ -330,7 +353,7 @@
<div>
<label for="input">Input</label>
<div id="input"></div>
<input type="file" id="scan-input" accept="image/*" capture="environment" hidden>
<input type="file" id="scan-input" accept="image/*" hidden>
<div id="scan-status" class="scan-status"></div>
</div>
<div>
@@ -443,7 +466,7 @@
<div class="group">
<button class="modal-btn icon" data-act="rotL" title="Rotate left"><span class="material-symbols-outlined">rotate_left</span></button>
<button class="modal-btn icon" data-act="rotR" title="Rotate right"><span class="material-symbols-outlined">rotate_right</span></button>
<button class="modal-btn icon" data-act="reset" title="Reset"><span class="material-symbols-outlined">restart_alt</span></button>
<button class="modal-btn icon" data-act="invert" title="Invert colours (for light text on dark background)"><span class="material-symbols-outlined">invert_colors</span></button>
</div>
<div class="group">
<button class="modal-btn" data-act="cancel">Cancel</button>
@@ -465,13 +488,15 @@
checkOrientation: true,
});
let invert = false;
const cleanup = () => {
cropper.destroy();
URL.revokeObjectURL(url);
backdrop.remove();
document.removeEventListener("keydown", onKey);
};
const finish = (blob) => { cleanup(); resolve(blob); };
const finish = (result) => { cleanup(); resolve(result); };
const onKey = (e) => {
if (e.key === "Escape") finish(null);
@@ -479,10 +504,13 @@
};
document.addEventListener("keydown", onKey);
function emit(act) {
function emit(act, btn) {
if (act === "rotL") cropper.rotate(-90);
else if (act === "rotR") cropper.rotate(90);
else if (act === "reset") cropper.reset();
else if (act === "invert") {
invert = !invert;
btn?.classList.toggle("active", invert);
}
else if (act === "cancel") finish(null);
else if (act === "use") {
// With autoCrop:false, if the user never drew a region, force a
@@ -492,14 +520,14 @@
maxWidth: OCR_MAX_DIM * 2,
maxHeight: OCR_MAX_DIM * 2,
});
if (canvas) canvas.toBlob(blob => finish(blob), "image/png");
if (canvas) canvas.toBlob(blob => finish({ blob, invert }), "image/png");
else finish(null);
}
}
modal.addEventListener("click", (e) => {
const act = e.target.closest("[data-act]")?.dataset.act;
if (act) emit(act);
const btn = e.target.closest("[data-act]");
if (btn) emit(btn.dataset.act, btn);
});
// Click on backdrop (outside the modal) = cancel
backdrop.addEventListener("click", (e) => {
@@ -516,7 +544,7 @@
const OCR_MAX_DIM = 1600;
const OCR_CONFIDENCE_MIN = 60; // word-level confidence (0-100)
async function preprocessForOCR(file) {
async function preprocessForOCR(file, { invert = false } = {}) {
const bitmap = await createImageBitmap(file);
const long = Math.max(bitmap.width, bitmap.height);
const scale = Math.min(1, OCR_MAX_DIM / long);
@@ -530,7 +558,7 @@
ctx.drawImage(bitmap, 0, 0, w, h);
bitmap.close?.();
binarizeAdaptive(ctx, w, h);
binarizeAdaptive(ctx, w, h, { invert });
// PNG, not JPEG — JPEG would re-blur the freshly-sharp B/W edges.
const blob = await new Promise(resolve => canvas.toBlob(resolve, "image/png"));
@@ -540,7 +568,9 @@
// Bradley/Roth adaptive threshold: compare each pixel against the
// mean of a local window. Computed in O(N) total via an integral
// image. Output is a 3-channel B/W image (R=G=B=0 or 255).
function binarizeAdaptive(ctx, w, h) {
// invert=true flips the grayscale before thresholding so white-on-dark
// text comes out as the expected black-on-white for Tesseract.
function binarizeAdaptive(ctx, w, h, { invert = false } = {}) {
const imgData = ctx.getImageData(0, 0, w, h);
const data = imgData.data;
@@ -549,6 +579,9 @@
for (let i = 0, j = 0; i < data.length; i += 4, j++) {
gray[j] = (data[i] * 0.299 + data[i + 1] * 0.587 + data[i + 2] * 0.114) | 0;
}
if (invert) {
for (let i = 0; i < gray.length; i++) gray[i] = 255 - gray[i];
}
// 2. Integral image (summed-area table)
const integral = new Float64Array(w * h);
@@ -608,9 +641,9 @@
scanBtn.disabled = true;
setScanStatus("Loading crop editor…");
let cropped;
let cropResult;
try {
cropped = await openCropper(file);
cropResult = await openCropper(file);
} catch (e) {
console.error(e);
setScanStatus(`Crop editor failed: ${e.message || e}`, { error: true });
@@ -618,19 +651,20 @@
scanInput.value = "";
return;
}
if (!cropped) {
if (!cropResult) {
// User cancelled
setScanStatus("");
scanBtn.disabled = false;
scanInput.value = "";
return;
}
const { blob: cropped, invert } = cropResult;
setScanStatus("Preparing image…");
const t0 = performance.now();
try {
const { blob, w, h, scaled, origLong } = await preprocessForOCR(cropped);
const { blob, w, h, scaled, origLong } = await preprocessForOCR(cropped, { invert });
if (scaled) {
setScanStatus(`Preprocessed ${origLong}px → ${Math.max(w, h)}px (binarised)`);
} else {
@@ -670,7 +704,93 @@
}
}
scanBtn.addEventListener("click", () => scanInput.click());
// Live camera capture via getUserMedia — bypasses the OS camera's
// "retake / use" confirm step that <input type=file capture> shows.
// Tap the shutter → frame is captured and crop modal opens immediately.
// Returns the captured Blob, "file" if the user wants the file picker,
// or null on cancel.
async function captureFromCamera() {
let stream;
try {
stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: { ideal: "environment" } },
audio: false,
});
} catch (e) {
console.warn("Camera unavailable:", e);
return "file"; // fall through to file picker
}
return new Promise((resolve) => {
const backdrop = document.createElement("div");
backdrop.className = "modal-backdrop";
const modal = document.createElement("div");
modal.className = "modal";
modal.innerHTML = `
<div class="modal-header">Aim at the text and tap the shutter</div>
<div class="modal-body cam-body"><video autoplay playsinline muted></video></div>
<div class="modal-footer">
<div class="group">
<button class="modal-btn icon" data-act="file" title="Pick from file instead"><span class="material-symbols-outlined">photo_library</span></button>
</div>
<div class="group">
<button class="modal-btn" data-act="cancel">Cancel</button>
<button class="modal-btn primary icon" data-act="shoot" title="Capture"><span class="material-symbols-outlined">photo_camera</span></button>
</div>
</div>
`;
backdrop.appendChild(modal);
document.body.appendChild(backdrop);
const video = modal.querySelector("video");
video.srcObject = stream;
const cleanup = () => {
stream.getTracks().forEach(t => t.stop());
video.srcObject = null;
backdrop.remove();
document.removeEventListener("keydown", onKey);
};
const finish = (r) => { cleanup(); resolve(r); };
const onKey = (e) => {
if (e.key === "Escape") finish(null);
};
document.addEventListener("keydown", onKey);
modal.addEventListener("click", (e) => {
const act = e.target.closest("[data-act]")?.dataset.act;
if (act === "cancel") finish(null);
else if (act === "file") finish("file");
else if (act === "shoot") {
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext("2d").drawImage(video, 0, 0);
canvas.toBlob(blob => finish(blob), "image/jpeg", 0.92);
}
});
backdrop.addEventListener("click", (e) => {
if (e.target === backdrop) finish(null);
});
});
}
async function startScan() {
// On a touch device with getUserMedia, go straight to the live
// viewfinder. On desktop / no-camera, use the file picker.
if (isCoarsePointer && navigator.mediaDevices?.getUserMedia) {
const result = await captureFromCamera();
if (result === "file") {
scanInput.click();
} else if (result) {
runOCR(result);
}
return;
}
scanInput.click();
}
scanBtn.addEventListener("click", startScan);
scanInput.addEventListener("change", (e) => {
const file = e.target.files?.[0];
if (file) runOCR(file);