From a63b11e4676f428fff34a2d75e222f13a9a9bbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Battistella=20Nadas?= Date: Tue, 9 Jun 2026 19:27:34 +0200 Subject: [PATCH] feat(ocr): live camera viewfinder + invert-colours toggle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four wins: 1. Replace with a getUserMedia viewfinder modal on mobile. Tapping the shutter goes straight to the crop modal — skips the OS "retake/use" confirm step. Falls back to file picker if camera permission is denied or the API is unavailable. "Pick from file" icon (photo_library) inside the camera modal opens the gallery on demand. 2. Drop the Reset button — with no default crop box, "reset" is just "no box again," which is the starting state anyway. 3. The check (Use this) button is now larger and more prominent — wider padding, 24px icon, accent fill. 4. Invert-colours toggle (invert_colors icon) in the crop modal. White-on-dark text (signs, menus, screenshots) breaks Tesseract because the binariser was made for ink on paper. Toggling the button flips the grayscale buffer before adaptive thresholding so the binary output ends up black-on-white. State plumbed cleanly: openCropper now resolves with {blob, invert}; preprocessForOCR takes an {invert} option; binarizeAdaptive applies the flip. Also: drop `capture=environment` from the file . Without it the file path correctly opens the gallery/file dialog instead of re-opening the OS camera when the user picks "Pick from file". --- CHANGELOG.md | 6 +- static/index.html | 152 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 141 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ed6c3..1fe4886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,10 +18,14 @@ Versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - "Scan image" button replaced with a floating action button at the bottom-right of the viewport, using a Material Symbols Outlined camera glyph (`photo_camera`) for consistent rendering across platforms (was a small text button in the input pane header). - Crop modal: - No default crop box. User drags on the image to draw the region; falls back to full image if they confirm without drawing. - - Rotate left / Rotate right / Reset / Use this buttons replaced with Material Symbols icons (`rotate_left`, `rotate_right`, `restart_alt`, `check`). Cancel kept as text. + - Buttons are now Material Symbols icons: `rotate_left`, `rotate_right`, `invert_colors` (new — see below), `check`. Cancel kept as text. The `check` (Use this) button is larger and more prominent. The Reset button was removed (rarely useful with no default box). - Inner padding bumped to 2 rem so cropperjs's corner handles never sit on the right/bottom edges where Android's edge-swipe back gesture fires. iOS safe-area insets honoured on the backdrop and FAB. - Header text is now a hint ("Drag on the image to select the text region") instead of a title. +### Added +- Camera viewfinder modal on mobile (via `navigator.mediaDevices.getUserMedia` with `facingMode: environment`) — replaces the `` flow. Tapping the shutter goes directly to the crop modal, skipping the OS-level "retake / use" confirm step. Falls back to the file picker if camera access is denied or unavailable; a "Pick from file" icon (`photo_library`) inside the camera modal also routes to the picker on demand. +- Invert-colours toggle (`invert_colors`) in the crop modal. Tap to flip light-on-dark text into the dark-on-light polarity Tesseract expects. The grayscale buffer is inverted before adaptive thresholding so the binarised output is correct. + ### Fixed - "Hot: …" status no longer claims "unloads in X" when the server's `keep_alive` is `-1` (Ollama returns an absurd year-4001 `expires_at`). Threshold: any expiry >7 days drops the suffix and just shows the model name. diff --git a/static/index.html b/static/index.html index 81b989d..cf3b714 100644 --- a/static/index.html +++ b/static/index.html @@ -269,13 +269,36 @@ .modal-btn .material-symbols-outlined { font-size: 20px; } .modal-btn.icon { padding: 0.35rem 0.55rem; min-width: 2.4rem; } .modal-btn:hover { border-color: var(--accent); color: var(--fg); } + .modal-btn.active { + background: var(--accent); + color: #fff; + border-color: var(--accent); + } .modal-btn.primary { background: var(--accent); color: #fff; border-color: var(--accent); } + .modal-btn.primary.icon { + padding: 0.55rem 1.1rem; + min-width: 3.2rem; + } + .modal-btn.primary.icon .material-symbols-outlined { font-size: 24px; } .modal-btn.primary:hover { color: #fff; } + /* Camera viewfinder modal */ + .cam-body { + display: flex; + align-items: center; + justify-content: center; + padding: 0 !important; + } + .cam-body video { + max-width: 100%; + max-height: 70vh; + display: block; + } + /* CodeMirror editor host — matches .output styling so input & output align */ .cm-editor { min-height: 180px; @@ -330,7 +353,7 @@
- +
@@ -443,7 +466,7 @@
- +
@@ -465,13 +488,15 @@ checkOrientation: true, }); + let invert = false; + const cleanup = () => { cropper.destroy(); URL.revokeObjectURL(url); backdrop.remove(); document.removeEventListener("keydown", onKey); }; - const finish = (blob) => { cleanup(); resolve(blob); }; + const finish = (result) => { cleanup(); resolve(result); }; const onKey = (e) => { if (e.key === "Escape") finish(null); @@ -479,10 +504,13 @@ }; document.addEventListener("keydown", onKey); - function emit(act) { + function emit(act, btn) { if (act === "rotL") cropper.rotate(-90); else if (act === "rotR") cropper.rotate(90); - else if (act === "reset") cropper.reset(); + else if (act === "invert") { + invert = !invert; + btn?.classList.toggle("active", invert); + } else if (act === "cancel") finish(null); else if (act === "use") { // With autoCrop:false, if the user never drew a region, force a @@ -492,14 +520,14 @@ maxWidth: OCR_MAX_DIM * 2, maxHeight: OCR_MAX_DIM * 2, }); - if (canvas) canvas.toBlob(blob => finish(blob), "image/png"); + if (canvas) canvas.toBlob(blob => finish({ blob, invert }), "image/png"); else finish(null); } } modal.addEventListener("click", (e) => { - const act = e.target.closest("[data-act]")?.dataset.act; - if (act) emit(act); + const btn = e.target.closest("[data-act]"); + if (btn) emit(btn.dataset.act, btn); }); // Click on backdrop (outside the modal) = cancel backdrop.addEventListener("click", (e) => { @@ -516,7 +544,7 @@ const OCR_MAX_DIM = 1600; const OCR_CONFIDENCE_MIN = 60; // word-level confidence (0-100) - async function preprocessForOCR(file) { + async function preprocessForOCR(file, { invert = false } = {}) { const bitmap = await createImageBitmap(file); const long = Math.max(bitmap.width, bitmap.height); const scale = Math.min(1, OCR_MAX_DIM / long); @@ -530,7 +558,7 @@ ctx.drawImage(bitmap, 0, 0, w, h); bitmap.close?.(); - binarizeAdaptive(ctx, w, h); + binarizeAdaptive(ctx, w, h, { invert }); // PNG, not JPEG — JPEG would re-blur the freshly-sharp B/W edges. const blob = await new Promise(resolve => canvas.toBlob(resolve, "image/png")); @@ -540,7 +568,9 @@ // Bradley/Roth adaptive threshold: compare each pixel against the // mean of a local window. Computed in O(N) total via an integral // image. Output is a 3-channel B/W image (R=G=B=0 or 255). - function binarizeAdaptive(ctx, w, h) { + // invert=true flips the grayscale before thresholding so white-on-dark + // text comes out as the expected black-on-white for Tesseract. + function binarizeAdaptive(ctx, w, h, { invert = false } = {}) { const imgData = ctx.getImageData(0, 0, w, h); const data = imgData.data; @@ -549,6 +579,9 @@ for (let i = 0, j = 0; i < data.length; i += 4, j++) { gray[j] = (data[i] * 0.299 + data[i + 1] * 0.587 + data[i + 2] * 0.114) | 0; } + if (invert) { + for (let i = 0; i < gray.length; i++) gray[i] = 255 - gray[i]; + } // 2. Integral image (summed-area table) const integral = new Float64Array(w * h); @@ -608,9 +641,9 @@ scanBtn.disabled = true; setScanStatus("Loading crop editor…"); - let cropped; + let cropResult; try { - cropped = await openCropper(file); + cropResult = await openCropper(file); } catch (e) { console.error(e); setScanStatus(`Crop editor failed: ${e.message || e}`, { error: true }); @@ -618,19 +651,20 @@ scanInput.value = ""; return; } - if (!cropped) { + if (!cropResult) { // User cancelled setScanStatus(""); scanBtn.disabled = false; scanInput.value = ""; return; } + const { blob: cropped, invert } = cropResult; setScanStatus("Preparing image…"); const t0 = performance.now(); try { - const { blob, w, h, scaled, origLong } = await preprocessForOCR(cropped); + const { blob, w, h, scaled, origLong } = await preprocessForOCR(cropped, { invert }); if (scaled) { setScanStatus(`Preprocessed ${origLong}px → ${Math.max(w, h)}px (binarised)`); } else { @@ -670,7 +704,93 @@ } } - scanBtn.addEventListener("click", () => scanInput.click()); + // Live camera capture via getUserMedia — bypasses the OS camera's + // "retake / use" confirm step that shows. + // Tap the shutter → frame is captured and crop modal opens immediately. + // Returns the captured Blob, "file" if the user wants the file picker, + // or null on cancel. + async function captureFromCamera() { + let stream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + video: { facingMode: { ideal: "environment" } }, + audio: false, + }); + } catch (e) { + console.warn("Camera unavailable:", e); + return "file"; // fall through to file picker + } + return new Promise((resolve) => { + const backdrop = document.createElement("div"); + backdrop.className = "modal-backdrop"; + const modal = document.createElement("div"); + modal.className = "modal"; + modal.innerHTML = ` +
Aim at the text and tap the shutter
+
+
+
+ +
+
+ + +
+
+ `; + backdrop.appendChild(modal); + document.body.appendChild(backdrop); + + const video = modal.querySelector("video"); + video.srcObject = stream; + + const cleanup = () => { + stream.getTracks().forEach(t => t.stop()); + video.srcObject = null; + backdrop.remove(); + document.removeEventListener("keydown", onKey); + }; + const finish = (r) => { cleanup(); resolve(r); }; + + const onKey = (e) => { + if (e.key === "Escape") finish(null); + }; + document.addEventListener("keydown", onKey); + + modal.addEventListener("click", (e) => { + const act = e.target.closest("[data-act]")?.dataset.act; + if (act === "cancel") finish(null); + else if (act === "file") finish("file"); + else if (act === "shoot") { + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + canvas.getContext("2d").drawImage(video, 0, 0); + canvas.toBlob(blob => finish(blob), "image/jpeg", 0.92); + } + }); + backdrop.addEventListener("click", (e) => { + if (e.target === backdrop) finish(null); + }); + }); + } + + async function startScan() { + // On a touch device with getUserMedia, go straight to the live + // viewfinder. On desktop / no-camera, use the file picker. + if (isCoarsePointer && navigator.mediaDevices?.getUserMedia) { + const result = await captureFromCamera(); + if (result === "file") { + scanInput.click(); + } else if (result) { + runOCR(result); + } + return; + } + scanInput.click(); + } + + scanBtn.addEventListener("click", startScan); scanInput.addEventListener("change", (e) => { const file = e.target.files?.[0]; if (file) runOCR(file);