Merge wind and ECMWF candidates into walkthrough train

This commit is contained in:
Bilawal Sidhu committed 2026-09-15 11:02:26 -05:00
commit 055bcda1c3
38 files changed
+1998 -6

No files matched your search

+2
View File
@@ -20,6 +20,7 @@ import { trackBackfillProxies } from './aircraft/tracks.js';
import { openAiRealtimeProxy } from './openai.js';
import { googlePlacesContextProxy } from './places.js';
import { keySetupEndpoint } from '../standalone/key-setup.js';
import { windProxy } from './wind.js';
/** Construct the local provider plugins in their established order. */
function localProviderPlugins() {
@@ -45,6 +46,7 @@ function localProviderPlugins() {
trackBackfillProxies(),
openAiRealtimeProxy(),
googlePlacesContextProxy(),
windProxy(),
keySetupEndpoint(),
];
}
+176
View File
@@ -0,0 +1,176 @@
import { fetchGfsWind } from './wind/gfs.js';
import { fetchIfsWind } from './wind/ifs.js';
import { decodeWindGribMessage } from './wind/decode.js';
/** Serve bounded, single-flight, per-model forecast snapshots from fixed providers. */
export function windProxy({
fetchImpl = fetch,
now = () => Date.now(),
decodeImpl = decodeWindGribMessage,
targetDx = 1,
ttlMs = 3600_000,
timeoutMs = 40_000,
models = { gfs: fetchGfsWind, ifs: fetchIfsWind },
} = {}) {
const caches = new Map();
const loadings = new Map();
const grids = new Map();
const attempts = new Map();
const unavailable = (model) => ({
manifest: {
model,
schemaVersion: 1,
unavailable: true,
stale: true,
reason: 'Wind upstream unavailable',
},
});
const sendJson = (res, value, status = 200) => {
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify(value));
};
function refresh(model) {
const controller = new AbortController();
const operation = { controller, waiters: 0, promise: null };
loadings.set(model, operation);
attempts.set(model, now());
operation.promise = (async () => {
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const value = await models[model]({
fetchImpl,
now,
decodeImpl,
targetDx,
signal: controller.signal,
});
controller.signal.throwIfAborted();
const { grid, cycle } = value;
if (!grid.u.every(Number.isFinite) || !grid.v.every(Number.isFinite))
throw new Error('Invalid wind grid');
const id = `${model}-${cycle.date}-${cycle.hour}-f${cycle.forecastHour || 0}-${targetDx}`;
const manifest = {
schemaVersion: 1,
model,
cycle,
fetchedAt: now(),
level: value.level,
units: value.units,
grid: {
nx: grid.nx,
ny: grid.ny,
lo1: grid.lo1,
la1: grid.la1,
dx: grid.dx,
dy: grid.dy,
},
stale: false,
unavailable: false,
reason: null,
gridUrl: `/api/wind/grid/${id}.bin?model=${model}`,
};
const state = { id, grid, manifest, fetchedAt: now() };
caches.set(model, state);
// Retain the previous issued grid as well to cover a manifest/grid rollover.
const history = grids.get(model) || new Map();
history.set(id, state);
while (history.size > 2) history.delete(history.keys().next().value);
grids.set(model, history);
return state;
} catch {
if (controller.signal.aborted && operation.waiters === 0) {
attempts.delete(model);
return unavailable(model);
}
const old = caches.get(model);
if (old) {
old.manifest = {
...old.manifest,
stale: true,
reason: 'Wind upstream unavailable',
};
return old;
}
return unavailable(model);
} finally {
clearTimeout(timer);
controller.abort(); // Cancel a sibling request if the other component failed.
if (loadings.get(model) === operation) loadings.delete(model);
}
})();
return operation;
}
const handler = async (req, res) => {
const url = new URL(req.url, 'http://localhost');
const model = url.searchParams.get('model') || 'gfs';
if (req.method !== 'GET')
return sendJson(res, { error: 'method_not_allowed' }, 405);
if (!['gfs', 'ifs'].includes(model) || !Object.hasOwn(models, model))
return sendJson(res, { error: 'unknown_model' }, 400);
if (url.pathname.startsWith('/grid/')) {
const id = url.pathname.slice(6).replace(/\.bin$/, '');
const state =
url.pathname === `/grid/${id}.bin` && grids.get(model)?.get(id);
if (!state) return sendJson(res, { error: 'unknown_grid' }, 404);
const bytes = Buffer.concat([
Buffer.from(
state.grid.u.buffer,
state.grid.u.byteOffset,
state.grid.u.byteLength,
),
Buffer.from(
state.grid.v.buffer,
state.grid.v.byteOffset,
state.grid.v.byteLength,
),
]);
res.writeHead(200, {
'Content-Type': 'application/octet-stream',
'Cache-Control': 'public, max-age=3600, immutable',
});
return res.end(bytes);
}
if (!['/', '/manifest', '/status'].includes(url.pathname))
return sendJson(res, { error: 'not_found' }, 404);
let state = caches.get(model);
if (!state || now() - state.fetchedAt >= ttlMs) {
let operation = loadings.get(model);
if (!operation && now() - (attempts.get(model) ?? -Infinity) >= 60_000)
operation = refresh(model);
if (operation) {
let disconnected = false;
operation.waiters += 1;
const close = () => {
disconnected = true;
if (--operation.waiters === 0) operation.controller.abort();
};
res.once?.('close', close);
try {
state = await operation.promise;
} finally {
res.removeListener?.('close', close);
if (!disconnected) operation.waiters -= 1;
}
if (disconnected) return;
}
}
const manifest = state?.manifest || unavailable(model).manifest;
if (url.pathname === '/status') {
const { gridUrl, ...status } = manifest;
return sendJson(res, status);
}
return sendJson(res, manifest);
};
return {
name: 'wind',
configureServer({ middlewares }) {
middlewares.use('/api/wind', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/wind', handler);
},
};
}
+34
View File
@@ -0,0 +1,34 @@
export const GFS_BUCKET = 'noaa-gfs-bdp-pds';
/** Build the public GFS object key. */
export function gfsObjectKey({ date, hour, forecastHour = 0 }) {
const hh = String(hour).padStart(2, '0');
return `gfs.${date}/${hh}/atmos/gfs.t${hh}z.pgrb2.0p25.f${String(forecastHour).padStart(3, '0')}`;
}
/** Select the latest GFS cycle that should be available. */
export function selectLatestGfsCycle(
nowMs,
{ availabilityLagMs = 5 * 3600_000 } = {},
) {
const d = new Date(nowMs - availabilityLagMs);
const date = new Date(
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()),
);
const hour = Math.floor(d.getUTCHours() / 6) * 6;
date.setUTCHours(hour);
return { date: date.toISOString().slice(0, 10).replaceAll('-', ''), hour };
}
/**
* Round a time offset to the nearest published GFS forecast step so the field
* shown is the one valid closest to now, not always the analysis (f000).
* GFS 0.25° is hourly through f120, then every 3 h through f384.
* @param {number} hours - Hours elapsed since the cycle run time.
* @returns {number} A valid forecast step.
*/
export function nearestGfsStep(hours) {
const value = Math.max(0, Number.isFinite(hours) ? hours : 0);
const step = value <= 120 ? Math.round(value) : Math.round(value / 3) * 3;
return Math.max(0, Math.min(384, step));
}
+51
View File
@@ -0,0 +1,51 @@
/** Lazy shared ecCodes WASM instance, created only for an active wind request. */
let eccodesPromise = null;
function loadEccodes() {
eccodesPromise ??= import('@meri-imperiumi/eccodes-wasm')
.then((module) => module.createEccodes())
.catch((error) => {
eccodesPromise = null;
throw error;
});
return eccodesPromise;
}
/** Decode one bounded message in WASM memory without mounting the host filesystem. */
export async function decodeWindGribMessage(buffer, { eccodesModule } = {}) {
if (buffer.byteLength > 8 * 1024 * 1024)
throw new Error('Wind GRIB exceeds byte budget');
const eccodes = eccodesModule ?? (await loadEccodes());
// Everything after this await is synchronous. A shared virtual file is safe
// across U/V and model requests and its storage is released after each decode.
const file = '/gev-wind.grib2';
let handle = null;
try {
eccodes.writeFile(file, buffer);
handle = eccodes.openGrib(file);
const ni = handle.getLong('Ni');
const nj = handle.getLong('Nj');
if (
!Number.isInteger(ni) ||
!Number.isInteger(nj) ||
ni < 1 ||
nj < 1 ||
ni * nj > 2_000_000
)
throw new Error('Invalid wind GRIB geometry');
return {
ni,
nj,
lo1: handle.getDouble('longitudeOfFirstGridPointInDegrees'),
la1: handle.getDouble('latitudeOfFirstGridPointInDegrees'),
di: handle.getDouble('iDirectionIncrementInDegrees'),
dj: handle.getDouble('jDirectionIncrementInDegrees'),
values: handle.getDoubleArray('values'),
shortName: handle.getString('shortName'),
level: handle.getLong('level'),
units: handle.getString('units'),
};
} finally {
handle?.delete();
eccodes.writeFile(file, new Uint8Array(0));
}
}
+165
View File
@@ -0,0 +1,165 @@
import { readWindBody } from '../../../src/sources/windBody.js';
/**
* Parse a GFS `.idx` inventory into ordered message records.
*
* Each line is `msgNumber:byteOffset:d=YYYYMMDDHH:VAR:LEVEL:...:`. Malformed
* lines are skipped; a payload with no valid lines is rejected so callers never
* treat an HTML error page as an empty inventory.
*
* @param {string} text - Raw `.idx` body.
* @returns {{ messages: Array<{index: number, offset: number, variable: string, level: string}> }}
*/
export function parseGfsIdx(text) {
const messages = [];
for (const line of String(text).split(/\r?\n/)) {
if (!line) continue;
const parts = line.split(':');
if (
parts.length < 6 ||
!/^\d+$/.test(parts[0]) ||
!/^\d+$/.test(parts[1]) ||
!parts[3] ||
!parts[4]
)
continue;
messages.push({
index: Number(parts[0]),
offset: Number(parts[1]),
variable: parts[3],
level: parts[4],
});
}
if (!messages.length) throw new Error('malformed GFS index');
return { messages };
}
/**
* Byte ranges for the two 10 m wind components. A message runs from its own
* offset to one byte before the next message in file order.
*
* @param {{messages: Array<object>}} parsed - {@link parseGfsIdx} result.
* @param {{level?: string}} [options]
* @returns {{u: {start: number, end: number}, v: {start: number, end: number}}}
*/
export function windMessageRanges(
parsed,
{ level = '10 m above ground' } = {},
) {
const range = (variable) => {
const index = parsed.messages.findIndex(
(message) => message.variable === variable && message.level === level,
);
const message = parsed.messages[index];
const next = parsed.messages[index + 1];
if (!message || !next) throw new Error(`missing ${variable} at ${level}`);
return { start: message.offset, end: next.offset - 1 };
};
return { u: range('UGRD'), v: range('VGRD') };
}
/** Fetch a bounded text response as a Buffer. */
export async function fetchText({ url, fetchImpl = fetch, signal }) {
const response = await fetchImpl(url, { signal, redirect: 'error' });
if (!response.ok) {
await response.body?.cancel();
throw new Error('Wind upstream unavailable');
}
return Buffer.from(await readWindBody(response, 2 * 1024 * 1024, signal));
}
/**
* Fetch one bounded byte range. Rejects oversized ranges, non-206 responses
* (except a whole-object 200 starting at zero), and short bodies.
*/
export async function fetchRange({
url,
start,
end,
fetchImpl = fetch,
signal,
maxBytes = 8 * 1024 * 1024,
}) {
if (
!Number.isSafeInteger(start) ||
!Number.isSafeInteger(end) ||
start < 0 ||
end < start ||
end - start + 1 > maxBytes
)
throw new Error('range too large');
const response = await fetchImpl(url, {
signal,
redirect: 'error',
headers: { Range: `bytes=${start}-${end}` },
});
const validStatus =
response.status === 206 || (response.status === 200 && start === 0);
if (!validStatus || !response.ok) {
await response.body?.cancel();
throw new Error('Wind upstream unavailable');
}
const buffer = Buffer.from(
await readWindBody(response, end - start + 1, signal),
);
if (buffer.length !== end - start + 1)
throw new Error('invalid range length');
return buffer;
}
import {
GFS_BUCKET,
gfsObjectKey,
selectLatestGfsCycle,
nearestGfsStep,
} from './catalog.js';
import { decodeWindGribMessage } from './decode.js';
import { resampleWindGrid } from './grid.js';
/** Fetch and decode the GFS 10 m wind field valid closest to now. */
export async function fetchGfsWind({
fetchImpl = fetch,
now = () => Date.now(),
targetDx = 1,
decodeImpl = decodeWindGribMessage,
signal,
} = {}) {
const nowMs = now();
const cycle = selectLatestGfsCycle(nowMs);
const runMs = Date.UTC(
Number(cycle.date.slice(0, 4)),
Number(cycle.date.slice(4, 6)) - 1,
Number(cycle.date.slice(6, 8)),
cycle.hour,
);
// Pick the forecast step whose valid time is closest to now, so the layer
// shows the freshest field the cycle offers instead of the analysis.
const forecastHour = nearestGfsStep((nowMs - runMs) / 3600_000);
const base = `https://${GFS_BUCKET}.s3.amazonaws.com/${gfsObjectKey({ ...cycle, forecastHour })}`;
const index = await fetchText({ url: `${base}.idx`, fetchImpl, signal });
const ranges = windMessageRanges(parseGfsIdx(index.toString()));
const [uBuffer, vBuffer] = await Promise.all([
fetchRange({ url: base, ...ranges.u, fetchImpl, signal }),
fetchRange({ url: base, ...ranges.v, fetchImpl, signal }),
]);
const [u, v] = await Promise.all([decodeImpl(uBuffer), decodeImpl(vBuffer)]);
const grid = resampleWindGrid({
u: u.values,
v: v.values,
ni: u.ni,
nj: u.nj,
lo1: u.lo1,
la1: u.la1,
di: u.di,
dj: u.dj,
dx: targetDx,
dy: targetDx,
});
const runIso = new Date(runMs).toISOString();
const validIso = new Date(runMs + forecastHour * 3600_000).toISOString();
return {
cycle: { ...cycle, forecastHour, runIso, validIso },
level: '10 m above ground',
units: 'm/s',
grid,
};
}
+68
View File
@@ -0,0 +1,68 @@
/**
* Resample a GRIB wind grid into a compact equirectangular grid.
*
* Source grids are north-to-south (`la1` at the top, `dj` increasing toward the
* south) and start at `lo1`. Longitude wraps across the seam; latitude clamps.
*
* @param {{u: ArrayLike<number>, v: ArrayLike<number>, ni: number, nj: number,
* lo1: number, la1: number, di: number, dj: number, dx?: number, dy?: number}} input
* @returns {{nx: number, ny: number, lo1: number, la1: number, dx: number,
* dy: number, u: Float32Array, v: Float32Array}}
*/
export function resampleWindGrid({
u,
v,
ni,
nj,
lo1,
la1,
di,
dj,
dx = 1,
dy = 1,
}) {
if (
!Number.isInteger(ni) ||
!Number.isInteger(nj) ||
ni < 1 ||
nj < 1 ||
ni * nj > 2_000_000 ||
u?.length !== ni * nj ||
v?.length !== ni * nj ||
![lo1, la1, di, dj, dx, dy].every(Number.isFinite) ||
di <= 0 ||
dj <= 0 ||
dx < 0.25 ||
dy < 0.25 ||
dx > 180 ||
dy > 180
)
throw new Error('Invalid wind grid geometry');
const nx = Math.round(360 / dx);
const ny = Math.round(180 / dy) + 1;
const outU = new Float32Array(nx * ny);
const outV = new Float32Array(nx * ny);
const sample = (source, lon, lat) => {
const x = ((lon - lo1) / di + ni) % ni;
const y = Math.max(0, Math.min(nj - 1, (la1 - lat) / dj));
const x0 = Math.floor(x);
const x1 = (x0 + 1) % ni;
const y0 = Math.floor(y);
const y1 = Math.min(nj - 1, y0 + 1);
const fx = x - x0;
const fy = y - y0;
const top = source[y0 * ni + x0] * (1 - fx) + source[y0 * ni + x1] * fx;
const bottom = source[y1 * ni + x0] * (1 - fx) + source[y1 * ni + x1] * fx;
return top * (1 - fy) + bottom * fy;
};
for (let y = 0; y < ny; y += 1) {
for (let x = 0; x < nx; x += 1) {
const index = y * nx + x;
const lon = x * dx;
const lat = 90 - y * dy;
outU[index] = sample(u, lon, lat);
outV[index] = sample(v, lon, lat);
}
}
return { nx, ny, lo1: 0, la1: 90, dx, dy, u: outU, v: outV };
}
+148
View File
@@ -0,0 +1,148 @@
import { fetchText, fetchRange } from './gfs.js';
import { decodeWindGribMessage } from './decode.js';
import { resampleWindGrid } from './grid.js';
/** Keyless ECMWF Open Data root for real-time IFS forecasts. */
export const IFS_BASE = 'https://data.ecmwf.int/forecasts';
/**
* Select the latest IFS cycle that should be available.
* @param {number} nowMs - Reference epoch milliseconds.
* @param {{availabilityLagMs?: number}} [options]
* @returns {{date: string, hour: number}}
*/
export function selectLatestIfsCycle(
nowMs,
{ availabilityLagMs = 6 * 3600_000 } = {},
) {
const shifted = new Date(nowMs - availabilityLagMs);
return {
date: shifted.toISOString().slice(0, 10).replaceAll('-', ''),
hour: Math.floor(shifted.getUTCHours() / 6) * 6,
};
}
/**
* Build the IFS GRIB2 and `.index` URLs for one cycle and step.
* @param {{date: string, hour: number, step?: number}} cycle
* @returns {{grib: string, index: string}}
*/
export function ifsObjectUrls({ date, hour, step = 0 }) {
const hh = String(hour).padStart(2, '0');
const stem = `${IFS_BASE}/${date}/${hh}z/ifs/0p25/oper/${date}${hh}0000-${step}h-oper-fc`;
// The inventory sits beside the GRIB2 as `<stem>.index`, NOT `<grib2>.index`.
return { grib: `${stem}.grib2`, index: `${stem}.index` };
}
/**
* Parse the IFS JSON Lines inventory into `{param, offset, length}` records.
* Malformed lines are skipped; a payload with no valid lines is rejected.
* @param {string} text - Raw `.index` body.
* @returns {Array<{param: string, offset: number, length: number}>}
*/
export function parseIfsIndex(text) {
const entries = [];
for (const line of String(text).split(/\r?\n/)) {
if (!line) continue;
let value;
try {
value = JSON.parse(line);
} catch {
continue; // non-JSON line in the index
}
if (
value?.param &&
Number.isFinite(value._offset) &&
Number.isFinite(value._length)
)
entries.push({
param: value.param,
offset: value._offset,
length: value._length,
});
}
if (!entries.length) throw new Error('malformed IFS index');
return entries;
}
/**
* Byte ranges for the IFS 10 m wind components.
* @param {Array<{param: string, offset: number, length: number}>} entries
* @returns {{u: {start: number, end: number}, v: {start: number, end: number}}}
*/
export function ifsWindRanges(entries) {
const range = (param) => {
const item = entries.find((entry) => entry.param === param);
if (!item) throw new Error(`missing ${param}`);
return { start: item.offset, end: item.offset + item.length - 1 };
};
return { u: range('10u'), v: range('10v') };
}
/**
* Round a time offset to the nearest published IFS forecast step. IFS 0.25°
* oper is 3-hourly through f144, then 6-hourly to f360 for 00/12z.
* @param {number} hours - Hours elapsed since the cycle run time.
* @param {number} cycleHour - The cycle hour (0, 6, 12 or 18).
* @returns {number} A valid forecast step.
*/
export function nearestIfsStep(hours, cycleHour) {
const max = cycleHour === 6 || cycleHour === 18 ? 144 : 360;
const value = Math.max(0, Number.isFinite(hours) ? hours : 0);
const step =
value <= 144 ? Math.round(value / 3) * 3 : Math.round(value / 6) * 6;
return Math.max(0, Math.min(max, step));
}
/**
* Fetch and decode the IFS 10 m wind field valid closest to now.
* @param {{fetchImpl?: Function, now?: Function, targetDx?: number,
* decodeImpl?: Function}} [options]
* @returns {Promise<{cycle: object, level: string, units: string, grid: object}>}
*/
export async function fetchIfsWind({
fetchImpl = fetch,
now = () => Date.now(),
targetDx = 1,
decodeImpl = decodeWindGribMessage,
signal,
} = {}) {
const nowMs = now();
const cycle = selectLatestIfsCycle(nowMs);
const runMs = Date.UTC(
Number(cycle.date.slice(0, 4)),
Number(cycle.date.slice(4, 6)) - 1,
Number(cycle.date.slice(6, 8)),
cycle.hour,
);
// Show the field valid closest to now rather than always the analysis.
const forecastHour = nearestIfsStep((nowMs - runMs) / 3600_000, cycle.hour);
const urls = ifsObjectUrls({ ...cycle, step: forecastHour });
const index = await fetchText({ url: urls.index, fetchImpl, signal });
const ranges = ifsWindRanges(parseIfsIndex(index.toString()));
const [uBuffer, vBuffer] = await Promise.all([
fetchRange({ url: urls.grib, ...ranges.u, fetchImpl, signal }),
fetchRange({ url: urls.grib, ...ranges.v, fetchImpl, signal }),
]);
const [u, v] = await Promise.all([decodeImpl(uBuffer), decodeImpl(vBuffer)]);
const grid = resampleWindGrid({
u: u.values,
v: v.values,
ni: u.ni,
nj: u.nj,
lo1: u.lo1,
la1: u.la1,
di: u.di,
dj: u.dj,
dx: targetDx,
dy: targetDx,
});
const runIso = new Date(runMs).toISOString();
const validIso = new Date(runMs + forecastHour * 3600_000).toISOString();
return {
cycle: { ...cycle, forecastHour, runIso, validIso },
level: '10 m above ground',
units: 'm/s',
grid,
};
}