/** * Resolve deictic tool arguments before an action runs. * * The model never copies coordinates or ids for "this", "here" or "the second * one". It passes a sentinel and the runner fills in the real target: * * track_entity {query:'pointer'} | {referent:n} * fly_to_location {query:'pointer'} | {referent:n} * get_entity_context {scope:'pointer'} | {referent:n} * analyst_query {scope:{kind:'pointer', km?}} * annotate_map annotations[].target / toTarget / points[].target = 'pointer' * find_imagery {area:'pointer'} * * The pointer is the current turn's snapshot (see pointerContext.js); the * referents are the latest result's numbered list (see referents.js). * Resolution is pure: it returns rewritten arguments, what they resolved * from, or a plain error the model can say. */ import { finiteOrNull } from './pointerContext.js'; const POINTER_SENTINEL = 'pointer'; const TRACKABLE_LAYERS = new Set([ 'flights', 'military', 'ais-live-vessels', 'satellites', ]); const DEFAULT_POINTER_RADIUS_KM = 50; const NO_POINTER = 'Nothing is under the pointer. Ask the user to point at it or name it.'; function isPointer(value) { return ( typeof value === 'string' && value.trim().toLowerCase() === POINTER_SENTINEL ); } function hasPosition(target) { return ( finiteOrNull(target?.lat) !== null && finiteOrNull(target?.lon) !== null ); } function referentIndex(args) { if (args?.referent === undefined || args?.referent === null) return null; const n = Number(args.referent); return Number.isInteger(n) ? n : NaN; } function withoutKeys(args, keys) { const out = { ...args }; for (const key of keys) delete out[key]; return out; } /** The pointer as a target record, or null. */ function pointerTarget(pointer) { if (!pointer?.fresh) return null; if (pointer.entity) { return { ...pointer.entity, lat: pointer.lat, lon: pointer.lon, }; } if (hasPosition(pointer)) return { lat: pointer.lat, lon: pointer.lon, label: null }; return null; } function lookupReferent(referents, n) { if (Number.isNaN(n)) return { error: 'referent must be a whole number.' }; const ref = referents?.get?.(n) || null; if (!ref) return { error: `There is no item ${n} in the last list.`, }; return { ref }; } function resolveTrack(args, { pointer, referents }) { const n = referentIndex(args); let target = null; let source = null; if (n !== null) { const { ref, error } = lookupReferent(referents, n); if (error) return { error }; target = ref; source = 'referent'; } else if (isPointer(args.query)) { target = pointerTarget(pointer); if (!target) return { error: NO_POINTER }; if (!target.layerId) return { error: 'The pointer is on the ground, not on an aircraft, ship or satellite.', }; source = 'pointer'; } else { return null; } if (!TRACKABLE_LAYERS.has(target.layerId)) return { error: `${target.label || 'That'} cannot be followed; only aircraft, ships and satellites can.`, }; return { args: { ...withoutKeys(args, ['referent']), query: String(target.id || target.label), layerId: target.layerId, }, used: { source, label: target.label || null, layerId: target.layerId }, }; } function resolveFly(args, { pointer, referents }) { const n = referentIndex(args); let target = null; let source = null; if (n !== null) { const { ref, error } = lookupReferent(referents, n); if (error) return { error }; target = ref; source = 'referent'; } else if (isPointer(args.query)) { target = pointerTarget(pointer); if (!target) return { error: NO_POINTER }; source = 'pointer'; } else { return null; } const rest = withoutKeys(args, ['referent', 'query', 'locationId']); const used = { source, label: target.label || null }; if (target.layerId && target.id && source === 'referent') { // Contacts move: locate them when the flight starts, not from a list. return { args: { ...rest, entity: { layerId: target.layerId, id: target.id, // The answer names it, not its id ("near cam-9"). ...(target.label ? { label: target.label } : {}), ...(hasPosition(target) ? { lat: target.lat, lon: target.lon } : {}), }, }, used, }; } if (hasPosition(target)) return { args: { ...rest, latitude: target.lat, longitude: target.lon }, used, }; if (target.label) return { args: { ...rest, query: target.label }, used }; return { error: 'That item has no position to fly to.' }; } function resolveContext(args, { pointer, referents }) { const n = referentIndex(args); if (n !== null) { const { ref, error } = lookupReferent(referents, n); if (error) return { error }; return { args: { ...withoutKeys(args, ['referent']), scope: 'target', target: ref, }, used: { source: 'referent', label: ref.label || null }, }; } if (String(args.scope || '').toLowerCase() !== POINTER_SENTINEL) return null; const target = pointerTarget(pointer); if (!target) { // A read-only lookup falls back to the usual scope instead of failing. return { args: { ...args, scope: 'auto' }, used: { source: 'pointer', label: null, missing: true }, }; } return { args: { ...args, scope: 'target', target }, used: { source: 'pointer', label: target.label || null, }, }; } function resolveAnalyst(args, { pointer }) { const scope = args.scope; if (!scope || String(scope.kind || '').toLowerCase() !== POINTER_SENTINEL) return null; const target = pointerTarget(pointer); if (!target || !hasPosition(target)) return { error: NO_POINTER }; const asked = finiteOrNull(scope.km); const km = asked > 0 ? asked : finiteOrNull(pointer.radiusKm) > 0 ? pointer.radiusKm : DEFAULT_POINTER_RADIUS_KM; const nextScope = withoutKeys(scope, ['name']); return { args: { ...args, scope: { ...nextScope, kind: 'radius', center: { lat: target.lat, lon: target.lon }, km, }, }, used: { source: 'pointer', label: target.label || null, km }, }; } function resolveAnnotate(args, { pointer }) { const list = Array.isArray(args.annotations) ? args.annotations : null; if (!list) return null; const mentions = list.some( (spec) => isPointer(spec?.target) || isPointer(spec?.toTarget) || (Array.isArray(spec?.points) && spec.points.some((point) => isPointer(point?.target))), ); if (!mentions) return null; const target = pointerTarget(pointer); if (!target || !hasPosition(target)) return { error: NO_POINTER }; const annotations = list.map((spec) => { if (!spec || typeof spec !== 'object') return spec; let next = spec; if (isPointer(spec.target)) { next = withoutKeys(next, ['target']); next.latitude = target.lat; next.longitude = target.lon; if (!next.label && target.label) next.label = target.label; } if (isPointer(spec.toTarget)) { next = withoutKeys(next, ['toTarget']); next.toLatitude = target.lat; next.toLongitude = target.lon; } if (Array.isArray(spec.points)) { next = { ...next, points: spec.points.map((point) => isPointer(point?.target) ? { ...withoutKeys(point, ['target']), latitude: target.lat, longitude: target.lon, } : point, ), }; } return next; }); return { args: { ...args, annotations }, used: { source: 'pointer', label: target.label || null }, }; } function resolveImagery(args, { pointer }) { if (!isPointer(args.area)) return null; const target = pointerTarget(pointer); if (!target || !hasPosition(target)) return { error: NO_POINTER }; return { args: { ...withoutKeys(args, ['area']), center: { lat: target.lat, lon: target.lon }, ...(target.label ? { centerLabel: target.label } : {}), }, used: { source: 'pointer', label: target.label || null }, }; } const SCREEN_PAIRS = [ ['screenX', 'screenY'], ['toScreenX', 'toScreenY'], ]; function hasScreenPoint(spec) { return ( SCREEN_PAIRS.some( ([x, y]) => spec?.[x] !== undefined || spec?.[y] !== undefined, ) || (Array.isArray(spec?.points) && spec.points.some( (point) => point?.screenX !== undefined || point?.screenY !== undefined, )) ); } /** * screenX/screenY are normalized against the latest image the model saw, but * the annotation resolver reads them against the full canvas. When that image * is the pointer crop, map crop coordinates back to the canvas; when the * camera moved since, no pixel names anything any more. */ function resolveCropPixels(args, { imageFrame, cameraKey }) { const list = Array.isArray(args.annotations) ? args.annotations : null; if (!list || imageFrame?.kind !== 'crop' || !list.some(hasScreenPoint)) return null; const rect = imageFrame.rect; if ( !rect || (imageFrame.cameraKey && cameraKey && imageFrame.cameraKey !== cameraKey) ) return { error: 'The view has moved since that image, so its pixels no longer match. Name the place or point at it.', }; const map = (value, origin, span) => { const n = finiteOrNull(value); return n === null ? value : origin + Math.min(1, Math.max(0, n)) * span; }; const remap = (spec) => { const next = { ...spec }; for (const [xKey, yKey] of SCREEN_PAIRS) { if (xKey in next) next[xKey] = map(next[xKey], rect.x, rect.w); if (yKey in next) next[yKey] = map(next[yKey], rect.y, rect.h); } return next; }; return { args: { ...args, annotations: list.map((spec) => { if (!spec || typeof spec !== 'object') return spec; const next = remap(spec); if (Array.isArray(spec.points)) next.points = spec.points.map(remap); return next; }), }, }; } const RESOLVERS = Object.freeze({ track_entity: resolveTrack, fly_to_location: resolveFly, show_street_level: resolveFly, get_entity_context: resolveContext, analyst_query: resolveAnalyst, annotate_map: resolveAnnotate, find_imagery: resolveImagery, }); /** * @param {string} name Tool name. * @param {object} args Model arguments. * @param {object} context * @param {object|null} [context.pointer] Current turn's pointer snapshot. * @param {{get: Function}|null} [context.referents] Referent registry. * @param {object|null} [context.imageFrame] What the retained image shows. * @param {string|null} [context.cameraKey] Current camera pose identity. * @returns {{args: object, used: object|null, error: string|null}} */ export function resolveDeicticArgs( name, args, { pointer = null, referents = null, imageFrame = null, cameraKey = null, } = {}, ) { let input = args && typeof args === 'object' ? args : {}; if (name === 'annotate_map') { const pixels = resolveCropPixels(input, { imageFrame, cameraKey }); if (pixels?.error) return { args: input, used: null, error: pixels.error }; if (pixels) input = pixels.args; } const resolver = RESOLVERS[name]; if (!resolver) return { args: input, used: null, error: null }; const resolved = resolver(input, { pointer, referents }); if (!resolved) return { args: input, used: null, error: null }; if (resolved.error) return { args: input, used: null, error: resolved.error }; return { args: resolved.args, used: resolved.used || null, error: null }; }