mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
fix(voice): preserve pointer crops in viewport escalation diagnostics
Reconcile origin #835 with merged voice core #837. Classify pointer crop separately from full screenshots, retain existing session/camera freshness guards, and log bounded static reason codes. Add dispatch regressions and changelog fragment. Credit: asturksever identified the incompatibility; preserve #837 authorship.
This commit is contained in:
commit
8c02f5852f
1068 files changed
+148229
-8636
No files matched your search
+269
-52
@@ -2,6 +2,12 @@ import {
|
||||
shouldSendViewportImage,
|
||||
hasStructuredViewIdentity,
|
||||
} from './realtimeProtocol.js';
|
||||
import { cameraPoseKey } from './pointerContext.js';
|
||||
|
||||
/** The live viewer's camera pose identity (Cesium-free read). */
|
||||
export function readViewerCameraKey() {
|
||||
return cameraPoseKey(globalThis.window?.__godsEyeView?.viewer);
|
||||
}
|
||||
// Viewport-screenshot size guards (M13). The old code clamped WIDTH only, so a
|
||||
// tall portrait window produced an oversized capture whose dc.send could throw.
|
||||
// Cap total pixels (clamps both dimensions) and drop the image entirely if the
|
||||
@@ -11,31 +17,6 @@ export const VIEWPORT_MAX_PIXELS = 1200 * 900;
|
||||
// ~1.08 MP, matches the old 1200px-wide landscape budget
|
||||
export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
|
||||
|
||||
/**
|
||||
* Explain whether the existing Realtime path should escalate structured entity
|
||||
* context to a viewport image. This is instrumentation vocabulary only; it
|
||||
* preserves the current send policy exactly.
|
||||
*/
|
||||
export function viewportContextDecision(result, { channelOpen = true } = {}) {
|
||||
if (result?.action !== 'get_entity_context')
|
||||
return { send: false, reason: 'not-entity-context' };
|
||||
if (!channelOpen) return { send: false, reason: 'channel-closed' };
|
||||
const viewScale = result.scene?.basemap?.viewScale ?? null;
|
||||
if (!shouldSendViewportImage(viewScale))
|
||||
return { send: false, reason: 'view-scale-not-local', viewScale };
|
||||
if (hasStructuredViewIdentity(result))
|
||||
return {
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale,
|
||||
};
|
||||
return {
|
||||
send: true,
|
||||
reason: 'structured-identity-missing',
|
||||
viewScale,
|
||||
};
|
||||
}
|
||||
|
||||
export async function captureViewportImage() {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
const source =
|
||||
@@ -84,6 +65,174 @@ export async function captureViewportImage() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Side of the point-and-ask crop, in CSS pixels, and its output size. */
|
||||
export const POINTER_CROP_SIZE = 512;
|
||||
/** A pointer snapshot older than this no longer matches the frame. */
|
||||
export const POINTER_CROP_MAX_AGE_MS = 20000;
|
||||
/** View scales where a crop shows buildings and terrain, not a smear. */
|
||||
export const POINTER_CROP_SCALES = new Set(['local', 'city']);
|
||||
|
||||
/**
|
||||
* Reason-coded explanation of the existing visual escalation policy.
|
||||
* Pointer crops remain a separate path from full-frame screenshots.
|
||||
* These reasons are static diagnostics, never model-generated claims.
|
||||
*/
|
||||
export function viewportContextDecision(result, { channelOpen = true } = {}) {
|
||||
if (result?.action !== 'get_entity_context')
|
||||
return { send: false, reason: 'not-entity-context' };
|
||||
if (!channelOpen) return { send: false, reason: 'channel-closed' };
|
||||
const viewScale = result.scene?.basemap?.viewScale ?? null;
|
||||
if (result.scope === 'pointer') {
|
||||
if (result.selected)
|
||||
return { send: false, reason: 'pointer-identity-present', viewScale };
|
||||
if (!result.pointer)
|
||||
return { send: false, reason: 'pointer-missing', viewScale };
|
||||
if (!POINTER_CROP_SCALES.has(viewScale))
|
||||
return { send: false, reason: 'pointer-scale-not-useful', viewScale };
|
||||
return {
|
||||
send: true,
|
||||
kind: 'crop',
|
||||
reason: 'pointer-ground-crop',
|
||||
viewScale,
|
||||
};
|
||||
}
|
||||
if (!shouldSendViewportImage(viewScale))
|
||||
return { send: false, reason: 'view-scale-not-local', viewScale };
|
||||
if (hasStructuredViewIdentity(result))
|
||||
return {
|
||||
send: false,
|
||||
reason: 'structured-identity-sufficient',
|
||||
viewScale,
|
||||
};
|
||||
return {
|
||||
send: true,
|
||||
kind: 'viewport',
|
||||
reason: 'structured-identity-missing',
|
||||
viewScale,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Square source rectangle around a canvas point, clamped inside the canvas,
|
||||
* in drawing-buffer pixels, plus where the point lands in the output image.
|
||||
* Pure; exported for tests.
|
||||
*/
|
||||
export function pointerCropRect({
|
||||
x,
|
||||
y,
|
||||
cssWidth,
|
||||
cssHeight,
|
||||
pixelWidth,
|
||||
pixelHeight,
|
||||
size = POINTER_CROP_SIZE,
|
||||
}) {
|
||||
const values = [x, y, cssWidth, cssHeight, pixelWidth, pixelHeight, size];
|
||||
if (
|
||||
values.some((value) => !Number.isFinite(value)) ||
|
||||
cssWidth <= 0 ||
|
||||
cssHeight <= 0 ||
|
||||
pixelWidth <= 0 ||
|
||||
pixelHeight <= 0 ||
|
||||
size <= 0 ||
|
||||
x < 0 ||
|
||||
x > cssWidth ||
|
||||
y < 0 ||
|
||||
y > cssHeight
|
||||
)
|
||||
return null;
|
||||
const side = Math.min(size, cssWidth, cssHeight);
|
||||
const left = Math.min(Math.max(0, x - side / 2), cssWidth - side);
|
||||
const top = Math.min(Math.max(0, y - side / 2), cssHeight - side);
|
||||
const scaleX = pixelWidth / cssWidth;
|
||||
const scaleY = pixelHeight / cssHeight;
|
||||
// Keep the provider-facing image contract stable even on a narrow canvas.
|
||||
// drawImage may upscale a smaller source crop, but the retained image and
|
||||
// pointer-ring coordinates are always expressed in one 512 px square.
|
||||
const output = Math.round(size);
|
||||
return {
|
||||
sx: Math.round(left * scaleX),
|
||||
sy: Math.round(top * scaleY),
|
||||
sw: Math.round(side * scaleX),
|
||||
sh: Math.round(side * scaleY),
|
||||
output,
|
||||
pointX: Math.round(((x - left) / side) * output),
|
||||
pointY: Math.round(((y - top) / side) * output),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A 512 px crop of a freshly rendered frame around the pointer, with a ring
|
||||
* at the exact spot, for "what is this building / this hill" on bare ground.
|
||||
* The crop is the map canvas only.
|
||||
* @param {{x: number, y: number}} screenPx Canvas CSS pixels.
|
||||
* @returns {Promise<{dataUrl: string, rect: object}|null>} JPEG data URL and
|
||||
* the crop's normalized bounds in the full canvas, or null.
|
||||
*/
|
||||
export async function capturePointerCrop(screenPx) {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
const source = viewer?.scene?.canvas;
|
||||
if (!source?.width || !source?.height || !screenPx) return null;
|
||||
const fresh = await renderFreshPointerFrame(viewer);
|
||||
if (!fresh) return null;
|
||||
const rect = pointerCropRect({
|
||||
x: screenPx.x,
|
||||
y: screenPx.y,
|
||||
cssWidth: source.clientWidth || source.width,
|
||||
cssHeight: source.clientHeight || source.height,
|
||||
pixelWidth: source.width,
|
||||
pixelHeight: source.height,
|
||||
});
|
||||
if (!rect) return null;
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = rect.output;
|
||||
canvas.height = rect.output;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return null;
|
||||
try {
|
||||
ctx.drawImage(
|
||||
source,
|
||||
rect.sx,
|
||||
rect.sy,
|
||||
rect.sw,
|
||||
rect.sh,
|
||||
0,
|
||||
0,
|
||||
rect.output,
|
||||
rect.output,
|
||||
);
|
||||
if (isNearlyBlackFrame(ctx, rect.output, rect.output)) return null;
|
||||
ctx.strokeStyle = 'rgba(0, 230, 255, 0.95)';
|
||||
ctx.lineWidth = 2;
|
||||
ctx.beginPath();
|
||||
ctx.arc(rect.pointX, rect.pointY, 16, 0, Math.PI * 2);
|
||||
ctx.stroke();
|
||||
const dataUrl = canvas.toDataURL('image/jpeg', 0.75);
|
||||
if (estimateDataUrlBytes(dataUrl) > VIEWPORT_MAX_ENCODED_BYTES) return null;
|
||||
return {
|
||||
dataUrl,
|
||||
rect: {
|
||||
x: rect.sx / source.width,
|
||||
y: rect.sy / source.height,
|
||||
w: rect.sw / source.width,
|
||||
h: rect.sh / source.height,
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pointer crops get one bounded recovery request for a slow render loop.
|
||||
* Full-viewport captures intentionally keep their single 400 ms attempt.
|
||||
*/
|
||||
export async function renderFreshPointerFrame(viewer) {
|
||||
if (await renderFreshCesiumFrame(viewer)) return true;
|
||||
if (typeof document !== 'undefined' && document.hidden) return false;
|
||||
return renderFreshCesiumFrame(viewer);
|
||||
}
|
||||
|
||||
// Scale (w, h) down so w*h <= maxPixels while preserving aspect ratio. Never
|
||||
// upscales. Both dimensions shrink together, so portrait and landscape are
|
||||
// treated equally (M13). Pure + deterministic → unit-tested (exported below).
|
||||
@@ -193,9 +342,28 @@ export function isBenignViewportDeleteError(payload, pendingDeleteIds = null) {
|
||||
|
||||
/** Own the one retained viewport image and bounded pending deletion identities. */
|
||||
export class RealtimeViewport {
|
||||
constructor({ readChannel, operations, capture = captureViewportImage }) {
|
||||
Object.assign(this, { readChannel, capture }, operations);
|
||||
constructor({
|
||||
readChannel,
|
||||
readPointer = () => null,
|
||||
operations,
|
||||
capture = captureViewportImage,
|
||||
capturePointer = capturePointerCrop,
|
||||
readCameraKey = readViewerCameraKey,
|
||||
now = () => Date.now(),
|
||||
}) {
|
||||
Object.assign(
|
||||
this,
|
||||
{ readChannel, readPointer, capture, capturePointer, readCameraKey, now },
|
||||
operations,
|
||||
);
|
||||
this.generation = 0;
|
||||
// Bumped by every capture request and every new user turn: only the
|
||||
// newest request of the current turn may publish.
|
||||
this.captureSeq = 0;
|
||||
this.lastPointerCrop = null;
|
||||
// What the retained image shows: `{kind:'viewport'|'crop', rect, cameraKey}`.
|
||||
// Pixel targeting (screenX/screenY) is read against it.
|
||||
this.retainedFrame = null;
|
||||
this.pendingViewportDeletes = new Set();
|
||||
this.lastViewportItemId = null;
|
||||
}
|
||||
@@ -210,25 +378,74 @@ export class RealtimeViewport {
|
||||
});
|
||||
this.debugLog?.('viewport_context.decision', decision);
|
||||
if (!decision.send) return false;
|
||||
// #837's pointer crop is separate from the full viewport: preserve it.
|
||||
if (decision.kind === 'crop')
|
||||
return this.sendPointerCrop(this.readPointer(), {
|
||||
viewScale: decision.viewScale,
|
||||
});
|
||||
return this.sendRetainedImage(
|
||||
() => this.capture(),
|
||||
"Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.",
|
||||
'client.viewport_context',
|
||||
{ kind: 'viewport' },
|
||||
);
|
||||
}
|
||||
|
||||
/** A new user turn: captures still in flight from the last one are dropped. */
|
||||
beginTurn() {
|
||||
this.captureSeq++;
|
||||
}
|
||||
|
||||
/**
|
||||
* Push the crop around a ground pointer snapshot, once per snapshot, into
|
||||
* the one retained image slot.
|
||||
* @param {object|null} snapshot Fresh pointer snapshot on bare ground.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async sendPointerCrop(snapshot, { viewScale } = {}) {
|
||||
if (!snapshot?.fresh || snapshot.target !== 'ground' || !snapshot.screenPx)
|
||||
return false;
|
||||
// Too far out, the crop is a smear: every path shares this guard.
|
||||
if (!POINTER_CROP_SCALES.has(viewScale)) return false;
|
||||
if (snapshot === this.lastPointerCrop) return false;
|
||||
const capturedAt = Number(snapshot.capturedAt) || 0;
|
||||
if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS) return false;
|
||||
// The snapshot's pixels only name its own camera pose.
|
||||
const snapshotCameraKey = snapshot.cameraKey;
|
||||
if (!snapshotCameraKey) return false;
|
||||
const samePose = () =>
|
||||
snapshotCameraKey === this.readCameraKey() &&
|
||||
this.now() - capturedAt <= POINTER_CROP_MAX_AGE_MS;
|
||||
if (!samePose()) return false;
|
||||
this.lastPointerCrop = snapshot;
|
||||
const sent = await this.sendRetainedImage(
|
||||
() => this.capturePointer(snapshot.screenPx),
|
||||
"Crop of the map around the user's pointer; the ring marks the exact spot they mean by this or here. Describe only what is clearly visible.",
|
||||
'client.pointer_crop',
|
||||
{ kind: 'crop', valid: samePose },
|
||||
);
|
||||
if (!sent && this.lastPointerCrop === snapshot) this.lastPointerCrop = null;
|
||||
return sent;
|
||||
}
|
||||
|
||||
/** Capture one image and make it the single retained image in context. */
|
||||
async sendRetainedImage(captureImage, text, label, frame = {}) {
|
||||
if (!this.dc || this.dc.readyState !== 'open') return false;
|
||||
const generation = this.generation;
|
||||
const imageUrl = await this.capture();
|
||||
if (!imageUrl) {
|
||||
this.debugLog?.('viewport_context.skipped', {
|
||||
reason: 'capture-unavailable',
|
||||
});
|
||||
return false;
|
||||
}
|
||||
const request = ++this.captureSeq;
|
||||
const channel = this.dc;
|
||||
const captured = await captureImage();
|
||||
const imageUrl =
|
||||
typeof captured === 'string' ? captured : captured?.dataUrl || null;
|
||||
if (
|
||||
!imageUrl ||
|
||||
generation !== this.generation ||
|
||||
request !== this.captureSeq ||
|
||||
this.dc !== channel ||
|
||||
channel.readyState !== 'open'
|
||||
) {
|
||||
this.debugLog?.('viewport_context.skipped', {
|
||||
reason: 'session-changed-during-capture',
|
||||
});
|
||||
channel.readyState !== 'open' ||
|
||||
(typeof frame.valid === 'function' && !frame.valid())
|
||||
)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Keep at most one viewport screenshot in context. Images are the single
|
||||
// most expensive item (re-billed every turn they linger), so we proactively
|
||||
@@ -266,10 +483,7 @@ export class RealtimeViewport {
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_text',
|
||||
text: "Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.",
|
||||
},
|
||||
{ type: 'input_text', text },
|
||||
{
|
||||
type: 'input_image',
|
||||
image_url: imageUrl,
|
||||
@@ -283,21 +497,24 @@ export class RealtimeViewport {
|
||||
// (it no longer throws — M13); we then leave lastViewportItemId pointing at
|
||||
// the item we just deleted as null and fall through so the caller still
|
||||
// issues queueResponseCreate WITHOUT the image, instead of stranding the turn.
|
||||
const sent = this.sendRealtimeEvent(
|
||||
contextEvent,
|
||||
'client.viewport_context',
|
||||
);
|
||||
const sent = this.sendRealtimeEvent(contextEvent, label);
|
||||
this.lastViewportItemId = sent ? newItemId : null;
|
||||
this.debugLog?.('viewport_context.result', {
|
||||
sent,
|
||||
reason: sent ? 'sent' : 'data-channel-send-failed',
|
||||
});
|
||||
this.retainedFrame = sent
|
||||
? {
|
||||
kind: frame.kind || 'viewport',
|
||||
rect: captured?.rect || null,
|
||||
cameraKey: this.readCameraKey(),
|
||||
}
|
||||
: null;
|
||||
return sent;
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.generation++;
|
||||
this.captureSeq++;
|
||||
this.lastViewportItemId = null;
|
||||
this.lastPointerCrop = null;
|
||||
this.retainedFrame = null;
|
||||
this.pendingViewportDeletes.clear();
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user