v0.3.0: OCR via camera, FAB UX, side drawer #4

Merged
jpnadas merged 16 commits from feature/ocr-camera into main 2026-06-09 18:38:34 +00:00
2 changed files with 577 additions and 0 deletions
Showing only changes of commit a63b11e467 - Show all commits

View File

@@ -18,10 +18,14 @@ Versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- "Scan image" button replaced with a floating action button at the bottom-right of the viewport, using a Material Symbols Outlined camera glyph (`photo_camera`) for consistent rendering across platforms (was a small text button in the input pane header). - "Scan image" button replaced with a floating action button at the bottom-right of the viewport, using a Material Symbols Outlined camera glyph (`photo_camera`) for consistent rendering across platforms (was a small text button in the input pane header).
- Crop modal: - Crop modal:
- No default crop box. User drags on the image to draw the region; falls back to full image if they confirm without drawing. - No default crop box. User drags on the image to draw the region; falls back to full image if they confirm without drawing.
- Rotate left / Rotate right / Reset / Use this buttons replaced with Material Symbols icons (`rotate_left`, `rotate_right`, `restart_alt`, `check`). Cancel kept as text. - Buttons are now Material Symbols icons: `rotate_left`, `rotate_right`, `invert_colors` (new — see below), `check`. Cancel kept as text. The `check` (Use this) button is larger and more prominent. The Reset button was removed (rarely useful with no default box).
- Inner padding bumped to 2 rem so cropperjs's corner handles never sit on the right/bottom edges where Android's edge-swipe back gesture fires. iOS safe-area insets honoured on the backdrop and FAB. - Inner padding bumped to 2 rem so cropperjs's corner handles never sit on the right/bottom edges where Android's edge-swipe back gesture fires. iOS safe-area insets honoured on the backdrop and FAB.
- Header text is now a hint ("Drag on the image to select the text region") instead of a title. - Header text is now a hint ("Drag on the image to select the text region") instead of a title.
### Added
- Camera viewfinder modal on mobile (via `navigator.mediaDevices.getUserMedia` with `facingMode: environment`) — replaces the `<input type=file capture>` flow. Tapping the shutter goes directly to the crop modal, skipping the OS-level "retake / use" confirm step. Falls back to the file picker if camera access is denied or unavailable; a "Pick from file" icon (`photo_library`) inside the camera modal also routes to the picker on demand.
- Invert-colours toggle (`invert_colors`) in the crop modal. Tap to flip light-on-dark text into the dark-on-light polarity Tesseract expects. The grayscale buffer is inverted before adaptive thresholding so the binarised output is correct.
### Fixed ### Fixed
- "Hot: …" status no longer claims "unloads in X" when the server's `keep_alive` is `-1` (Ollama returns an absurd year-4001 `expires_at`). Threshold: any expiry >7 days drops the suffix and just shows the model name. - "Hot: …" status no longer claims "unloads in X" when the server's `keep_alive` is `-1` (Ollama returns an absurd year-4001 `expires_at`). Threshold: any expiry >7 days drops the suffix and just shows the model name.

View File

@@ -269,13 +269,36 @@
.modal-btn .material-symbols-outlined { font-size: 20px; } .modal-btn .material-symbols-outlined { font-size: 20px; }
.modal-btn.icon { padding: 0.35rem 0.55rem; min-width: 2.4rem; } .modal-btn.icon { padding: 0.35rem 0.55rem; min-width: 2.4rem; }
.modal-btn:hover { border-color: var(--accent); color: var(--fg); } .modal-btn:hover { border-color: var(--accent); color: var(--fg); }
.modal-btn.active {
background: var(--accent);
color: #fff;
border-color: var(--accent);
}
.modal-btn.primary { .modal-btn.primary {
background: var(--accent); background: var(--accent);
color: #fff; color: #fff;
border-color: var(--accent); border-color: var(--accent);
} }
.modal-btn.primary.icon {
padding: 0.55rem 1.1rem;
min-width: 3.2rem;
}
.modal-btn.primary.icon .material-symbols-outlined { font-size: 24px; }
.modal-btn.primary:hover { color: #fff; } .modal-btn.primary:hover { color: #fff; }
/* Camera viewfinder modal */
.cam-body {
display: flex;
align-items: center;
justify-content: center;
padding: 0 !important;
}
.cam-body video {
max-width: 100%;
max-height: 70vh;
display: block;
}
/* CodeMirror editor host — matches .output styling so input & output align */ /* CodeMirror editor host — matches .output styling so input & output align */
.cm-editor { .cm-editor {
min-height: 180px; min-height: 180px;
@@ -330,7 +353,7 @@
<div> <div>
<label for="input">Input</label> <label for="input">Input</label>
<div id="input"></div> <div id="input"></div>
<input type="file" id="scan-input" accept="image/*" capture="environment" hidden> <input type="file" id="scan-input" accept="image/*" hidden>
<div id="scan-status" class="scan-status"></div> <div id="scan-status" class="scan-status"></div>
</div> </div>
<div> <div>
@@ -443,7 +466,7 @@
<div class="group"> <div class="group">
<button class="modal-btn icon" data-act="rotL" title="Rotate left"><span class="material-symbols-outlined">rotate_left</span></button> <button class="modal-btn icon" data-act="rotL" title="Rotate left"><span class="material-symbols-outlined">rotate_left</span></button>
<button class="modal-btn icon" data-act="rotR" title="Rotate right"><span class="material-symbols-outlined">rotate_right</span></button> <button class="modal-btn icon" data-act="rotR" title="Rotate right"><span class="material-symbols-outlined">rotate_right</span></button>
<button class="modal-btn icon" data-act="reset" title="Reset"><span class="material-symbols-outlined">restart_alt</span></button> <button class="modal-btn icon" data-act="invert" title="Invert colours (for light text on dark background)"><span class="material-symbols-outlined">invert_colors</span></button>
</div> </div>
<div class="group"> <div class="group">
<button class="modal-btn" data-act="cancel">Cancel</button> <button class="modal-btn" data-act="cancel">Cancel</button>
@@ -465,13 +488,15 @@
checkOrientation: true, checkOrientation: true,
}); });
let invert = false;
const cleanup = () => { const cleanup = () => {
cropper.destroy(); cropper.destroy();
URL.revokeObjectURL(url); URL.revokeObjectURL(url);
backdrop.remove(); backdrop.remove();
document.removeEventListener("keydown", onKey); document.removeEventListener("keydown", onKey);
}; };
const finish = (blob) => { cleanup(); resolve(blob); }; const finish = (result) => { cleanup(); resolve(result); };
const onKey = (e) => { const onKey = (e) => {
if (e.key === "Escape") finish(null); if (e.key === "Escape") finish(null);
@@ -479,10 +504,13 @@
}; };
document.addEventListener("keydown", onKey); document.addEventListener("keydown", onKey);
function emit(act) { function emit(act, btn) {
if (act === "rotL") cropper.rotate(-90); if (act === "rotL") cropper.rotate(-90);
else if (act === "rotR") cropper.rotate(90); else if (act === "rotR") cropper.rotate(90);
else if (act === "reset") cropper.reset(); else if (act === "invert") {
invert = !invert;
btn?.classList.toggle("active", invert);
}
else if (act === "cancel") finish(null); else if (act === "cancel") finish(null);
else if (act === "use") { else if (act === "use") {
// With autoCrop:false, if the user never drew a region, force a // With autoCrop:false, if the user never drew a region, force a
@@ -492,14 +520,14 @@
maxWidth: OCR_MAX_DIM * 2, maxWidth: OCR_MAX_DIM * 2,
maxHeight: OCR_MAX_DIM * 2, maxHeight: OCR_MAX_DIM * 2,
}); });
if (canvas) canvas.toBlob(blob => finish(blob), "image/png"); if (canvas) canvas.toBlob(blob => finish({ blob, invert }), "image/png");
else finish(null); else finish(null);
} }
} }
modal.addEventListener("click", (e) => { modal.addEventListener("click", (e) => {
const act = e.target.closest("[data-act]")?.dataset.act; const btn = e.target.closest("[data-act]");
if (act) emit(act); if (btn) emit(btn.dataset.act, btn);
}); });
// Click on backdrop (outside the modal) = cancel // Click on backdrop (outside the modal) = cancel
backdrop.addEventListener("click", (e) => { backdrop.addEventListener("click", (e) => {
@@ -516,7 +544,7 @@
const OCR_MAX_DIM = 1600; const OCR_MAX_DIM = 1600;
const OCR_CONFIDENCE_MIN = 60; // word-level confidence (0-100) const OCR_CONFIDENCE_MIN = 60; // word-level confidence (0-100)
async function preprocessForOCR(file) { async function preprocessForOCR(file, { invert = false } = {}) {
const bitmap = await createImageBitmap(file); const bitmap = await createImageBitmap(file);
const long = Math.max(bitmap.width, bitmap.height); const long = Math.max(bitmap.width, bitmap.height);
const scale = Math.min(1, OCR_MAX_DIM / long); const scale = Math.min(1, OCR_MAX_DIM / long);
@@ -530,7 +558,7 @@
ctx.drawImage(bitmap, 0, 0, w, h); ctx.drawImage(bitmap, 0, 0, w, h);
bitmap.close?.(); bitmap.close?.();
binarizeAdaptive(ctx, w, h); binarizeAdaptive(ctx, w, h, { invert });
// PNG, not JPEG — JPEG would re-blur the freshly-sharp B/W edges. // PNG, not JPEG — JPEG would re-blur the freshly-sharp B/W edges.
const blob = await new Promise(resolve => canvas.toBlob(resolve, "image/png")); const blob = await new Promise(resolve => canvas.toBlob(resolve, "image/png"));
@@ -540,7 +568,9 @@
// Bradley/Roth adaptive threshold: compare each pixel against the // Bradley/Roth adaptive threshold: compare each pixel against the
// mean of a local window. Computed in O(N) total via an integral // mean of a local window. Computed in O(N) total via an integral
// image. Output is a 3-channel B/W image (R=G=B=0 or 255). // image. Output is a 3-channel B/W image (R=G=B=0 or 255).
function binarizeAdaptive(ctx, w, h) { // invert=true flips the grayscale before thresholding so white-on-dark
// text comes out as the expected black-on-white for Tesseract.
function binarizeAdaptive(ctx, w, h, { invert = false } = {}) {
const imgData = ctx.getImageData(0, 0, w, h); const imgData = ctx.getImageData(0, 0, w, h);
const data = imgData.data; const data = imgData.data;
@@ -549,6 +579,9 @@
for (let i = 0, j = 0; i < data.length; i += 4, j++) { for (let i = 0, j = 0; i < data.length; i += 4, j++) {
gray[j] = (data[i] * 0.299 + data[i + 1] * 0.587 + data[i + 2] * 0.114) | 0; gray[j] = (data[i] * 0.299 + data[i + 1] * 0.587 + data[i + 2] * 0.114) | 0;
} }
if (invert) {
for (let i = 0; i < gray.length; i++) gray[i] = 255 - gray[i];
}
// 2. Integral image (summed-area table) // 2. Integral image (summed-area table)
const integral = new Float64Array(w * h); const integral = new Float64Array(w * h);
@@ -608,9 +641,9 @@
scanBtn.disabled = true; scanBtn.disabled = true;
setScanStatus("Loading crop editor…"); setScanStatus("Loading crop editor…");
let cropped; let cropResult;
try { try {
cropped = await openCropper(file); cropResult = await openCropper(file);
} catch (e) { } catch (e) {
console.error(e); console.error(e);
setScanStatus(`Crop editor failed: ${e.message || e}`, { error: true }); setScanStatus(`Crop editor failed: ${e.message || e}`, { error: true });
@@ -618,19 +651,20 @@
scanInput.value = ""; scanInput.value = "";
return; return;
} }
if (!cropped) { if (!cropResult) {
// User cancelled // User cancelled
setScanStatus(""); setScanStatus("");
scanBtn.disabled = false; scanBtn.disabled = false;
scanInput.value = ""; scanInput.value = "";
return; return;
} }
const { blob: cropped, invert } = cropResult;
setScanStatus("Preparing image…"); setScanStatus("Preparing image…");
const t0 = performance.now(); const t0 = performance.now();
try { try {
const { blob, w, h, scaled, origLong } = await preprocessForOCR(cropped); const { blob, w, h, scaled, origLong } = await preprocessForOCR(cropped, { invert });
if (scaled) { if (scaled) {
setScanStatus(`Preprocessed ${origLong}px → ${Math.max(w, h)}px (binarised)`); setScanStatus(`Preprocessed ${origLong}px → ${Math.max(w, h)}px (binarised)`);
} else { } else {
@@ -670,7 +704,93 @@
} }
} }
scanBtn.addEventListener("click", () => scanInput.click()); // Live camera capture via getUserMedia — bypasses the OS camera's
// "retake / use" confirm step that <input type=file capture> shows.
// Tap the shutter → frame is captured and crop modal opens immediately.
// Returns the captured Blob, "file" if the user wants the file picker,
// or null on cancel.
async function captureFromCamera() {
let stream;
try {
stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: { ideal: "environment" } },
audio: false,
});
} catch (e) {
console.warn("Camera unavailable:", e);
return "file"; // fall through to file picker
}
return new Promise((resolve) => {
const backdrop = document.createElement("div");
backdrop.className = "modal-backdrop";
const modal = document.createElement("div");
modal.className = "modal";
modal.innerHTML = `
<div class="modal-header">Aim at the text and tap the shutter</div>
<div class="modal-body cam-body"><video autoplay playsinline muted></video></div>
<div class="modal-footer">
<div class="group">
<button class="modal-btn icon" data-act="file" title="Pick from file instead"><span class="material-symbols-outlined">photo_library</span></button>
</div>
<div class="group">
<button class="modal-btn" data-act="cancel">Cancel</button>
<button class="modal-btn primary icon" data-act="shoot" title="Capture"><span class="material-symbols-outlined">photo_camera</span></button>
</div>
</div>
`;
backdrop.appendChild(modal);
document.body.appendChild(backdrop);
const video = modal.querySelector("video");
video.srcObject = stream;
const cleanup = () => {
stream.getTracks().forEach(t => t.stop());
video.srcObject = null;
backdrop.remove();
document.removeEventListener("keydown", onKey);
};
const finish = (r) => { cleanup(); resolve(r); };
const onKey = (e) => {
if (e.key === "Escape") finish(null);
};
document.addEventListener("keydown", onKey);
modal.addEventListener("click", (e) => {
const act = e.target.closest("[data-act]")?.dataset.act;
if (act === "cancel") finish(null);
else if (act === "file") finish("file");
else if (act === "shoot") {
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext("2d").drawImage(video, 0, 0);
canvas.toBlob(blob => finish(blob), "image/jpeg", 0.92);
}
});
backdrop.addEventListener("click", (e) => {
if (e.target === backdrop) finish(null);
});
});
}
async function startScan() {
// On a touch device with getUserMedia, go straight to the live
// viewfinder. On desktop / no-camera, use the file picker.
if (isCoarsePointer && navigator.mediaDevices?.getUserMedia) {
const result = await captureFromCamera();
if (result === "file") {
scanInput.click();
} else if (result) {
runOCR(result);
}
return;
}
scanInput.click();
}
scanBtn.addEventListener("click", startScan);
scanInput.addEventListener("change", (e) => { scanInput.addEventListener("change", (e) => {
const file = e.target.files?.[0]; const file = e.target.files?.[0];
if (file) runOCR(file); if (file) runOCR(file);