From bb18e39b7d393a887386803d822abd60923a6ea4 Mon Sep 17 00:00:00 2001 From: Gustavo Beneduzi Date: Mon, 14 Sep 2026 13:04:35 -0300 Subject: [PATCH] feat(wind): add ECMWF IFS as a second wind model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ECMWF IFS 10 m wind alongside NOAA GFS, selectable at runtime. The /api/wind proxy becomes a model dispatcher: `?model=gfs|ifs` (default gfs, unknown → 400), cached per model+cycle+grid. IFS reads the keyless ECMWF Open Data JSON Lines `.index` (note: `.index`, not `.index`), byte-range fetches only the `10u`/`10v` messages, decodes them with the same ecCodes WASM path, and resamples with the existing grid code (ECMWF grids start at lo1=180; the resampler already wraps columns). The client layer gains a `model` parameter (setParams/getParams) with a generation guard so a model switch aborts and refetches exactly once. Registered as an enabled+options layer with a `model` enum. Validated end-to-end against the real 2026-09-14 06z IFS cycle on Node 25 (180x91 grid at 2°, 131,040-byte payload, plausible mean |u|/|v|), and GFS still passes the same check. Attribution: contains modified Copernicus/ECMWF IFS forecast data, licensed under CC BY 4.0. Verified: npm test (3412 pass, 0 fail), check:boundaries. --- CHANGELOG.md | 2 + DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 2 + server/providers/wind.js | 194 +++++++---------------------------- server/providers/wind/gfs.js | 19 ++++ server/providers/wind/ifs.js | 122 ++++++++++++++++++++++ src/data/dataCredits.js | 7 ++ src/data/layerState.js | 5 +- src/layers/wind/index.js | 23 ++++- src/layers/wind/source.js | 6 +- 10 files changed, 216 insertions(+), 166 deletions(-) create mode 100644 server/providers/wind/ifs.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 452503bb1..24a6f3ab1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -655,3 +655,5 @@ represent previously published GitHub Releases. ## [0.1.0] — 2026-02-09 - Initial project version. + +- Add ECMWF IFS as a selectable wind model alongside GFS. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index eb1700ec2..cf466553a 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -190,3 +190,5 @@ Douglas-Peucker simplification, 6-decimal rounding). ## In-app attribution The required Google Maps / Cesium credit renders on the on-globe credit line (`#cesium-credits`, bottom-left) and must stay visible — including in clean-view and recording modes (the whole line, logo + "Google Maps" + the "Data attribution" link, stays on screen; only the GEV panels/HUD fade). The layer-specific credits (adsb.lol, TeleGeography, OSM datacenters/dams/roads, NASA FIRMS, CelesTrak, USGS, City of Austin, Fintraffic, GBFS, Radio Browser, OpenSky, AISStream) are registered into the expandable **"Data attribution"** popover on that credit line via `viewer.creditDisplay.addStaticCredit(new Cesium.Credit(html, /* showOnScreen */ false))` — see `src/data/dataCredits.js`. When you add a new data source, add its license and attribution to this file **and** append an entry to `DATA_CREDITS` in `src/data/dataCredits.js` so it surfaces in the app. + +- ECMWF IFS: keyless CC BY 4.0 GRIB2 forecast data fetched via `.index` and HTTP Range. Attribution: "Contains modified Copernicus/ECMWF IFS forecast data, licensed under CC BY 4.0. Source: https://data.ecmwf.int/forecasts/". diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 24586e5b1..4b9a4da3e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3402,3 +3402,5 @@ releases imagery layers/listeners on replacement. Destroy invalidates pending work and releases owned resources, including late factory results. Supplied 3D tilesets remain owned by the caller; tilesets created through the controller's factory are added to its viewer and removed on destruction. + +Wind supports GFS/IFS models. diff --git a/server/providers/wind.js b/server/providers/wind.js index 321717728..13dc0447e 100644 --- a/server/providers/wind.js +++ b/server/providers/wind.js @@ -1,170 +1,48 @@ -import { - GFS_BUCKET, - gfsObjectKey, - selectLatestGfsCycle, -} from './wind/catalog.js'; -import { - fetchText, - fetchRange, - parseGfsIdx, - windMessageRanges, -} from './wind/gfs.js'; +import { fetchGfsWind } from './wind/gfs.js'; +import { fetchIfsWind } from './wind/ifs.js'; import { decodeWindGribMessage } from './wind/decode.js'; -import { resampleWindGrid } from './wind/grid.js'; -/** - * NOAA GFS 10 m wind proxy. - * - * Selects the latest available GFS cycle, reads its `.idx` inventory, byte-range - * fetches only the UGRD/VGRD 10 m GRIB2 messages, decodes them with ecCodes - * (WASM), resamples to a compact grid, and serves a manifest plus a Float32 - * grid (U values then V values). Keyless; cached per cycle for an hour. - * - * Routes (mounted at `/api/wind`): - * GET /api/wind/manifest → manifest JSON - * GET /api/wind/grid/.bin → Float32 U…V payload - * GET /api/wind/status → manifest without `gridUrl` - * - * @param {{fetchImpl?: Function, now?: Function, decodeImpl?: Function, - * targetDx?: number, ttlMs?: number}} [options] - * @returns {import('vite').Plugin} - */ -export function windProxy({ - fetchImpl = fetch, - now = () => Date.now(), - decodeImpl = decodeWindGribMessage, - targetDx = 1, - ttlMs = 3600_000, -} = {}) { - /** @type {?{id: string, idGrid: string, fetchedAt: number, manifest: object, grid: ?object}} */ - let cached = null; - /** @type {?Promise} */ - let loading = null; - - const sendJson = (res, value, status = 200) => { - res.writeHead(status, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - }); - res.end(JSON.stringify(value)); - }; - - const cycleRunIso = (cycle) => - `${cycle.date.slice(0, 4)}-${cycle.date.slice(4, 6)}-${cycle.date.slice(6)}T${String(cycle.hour).padStart(2, '0')}:00:00.000Z`; - - async function refresh() { - const cycle = selectLatestGfsCycle(now()); - const id = `${cycle.date}-${cycle.hour}`; - if (cached && cached.id === id && now() - cached.fetchedAt < ttlMs) - return cached; - if (loading) return loading; - loading = (async () => { +/** Dispatch cached wind requests by model and cycle. */ +export function windProxy({ fetchImpl = fetch, now = () => Date.now(), decodeImpl = decodeWindGribMessage, targetDx = 1, ttlMs = 3600_000, models = { gfs: fetchGfsWind, ifs: fetchIfsWind } } = {}) { + const caches = new Map(); + const loadings = new Map(); + const sendJson = (res, value, status = 200) => { res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.end(JSON.stringify(value)); }; + const refresh = (model) => { + const loader = models[model]; + const options = { fetchImpl, now, decodeImpl, targetDx }; + const promise = (async () => { try { - const base = `https://${GFS_BUCKET}.s3.amazonaws.com/${gfsObjectKey(cycle)}`; - const index = await fetchText({ url: `${base}.idx`, fetchImpl }); - const ranges = windMessageRanges(parseGfsIdx(index.toString())); - const [uBuffer, vBuffer] = await Promise.all([ - fetchRange({ url: base, ...ranges.u, fetchImpl }), - fetchRange({ url: base, ...ranges.v, fetchImpl }), - ]); - const [u, v] = await Promise.all([ - decodeImpl(uBuffer), - decodeImpl(vBuffer), - ]); - const grid = resampleWindGrid({ - u: u.values, - v: v.values, - ni: u.ni, - nj: u.nj, - lo1: u.lo1, - la1: u.la1, - di: u.di, - dj: u.dj, - dx: targetDx, - dy: targetDx, - }); + const value = await loader(options); + const id = `${value.cycle.date}-${value.cycle.hour}`; const idGrid = `${id}-${targetDx}`; - const manifest = { - schemaVersion: 1, - model: 'gfs', - cycle: { ...cycle, forecastHour: 0, runIso: cycleRunIso(cycle) }, - fetchedAt: now(), - level: '10 m above ground', - units: 'm/s', - grid: { - nx: grid.nx, - ny: grid.ny, - lo1: grid.lo1, - la1: grid.la1, - dx: grid.dx, - dy: grid.dy, - }, - stale: false, - unavailable: false, - reason: null, - gridUrl: `/api/wind/grid/${idGrid}.bin`, - }; - cached = { id, idGrid, fetchedAt: manifest.fetchedAt, manifest, grid }; - return cached; + const manifest = { schemaVersion: 1, model, cycle: value.cycle, fetchedAt: now(), level: value.level, units: value.units, grid: { nx: value.grid.nx, ny: value.grid.ny, lo1: value.grid.lo1, la1: value.grid.la1, dx: value.grid.dx, dy: value.grid.dy }, stale: false, unavailable: false, reason: null, gridUrl: `/api/wind/grid/${idGrid}.bin` }; + const state = { id, idGrid, fetchedAt: manifest.fetchedAt, manifest, grid: value.grid }; + caches.set(model, state); + return state; } catch (error) { - if (cached) { - cached = { - ...cached, - manifest: { ...cached.manifest, stale: true, reason: error.message }, - }; - } else { - cached = { - id: null, - idGrid: null, - fetchedAt: now(), - grid: null, - manifest: { - schemaVersion: 1, - model: 'gfs', - stale: true, - unavailable: true, - reason: error.message, - }, - }; - } - return cached; - } finally { - loading = null; - } + const old = caches.get(model); + if (old) { old.manifest = { ...old.manifest, stale: true, reason: error.message }; return old; } + return { id: null, idGrid: null, grid: null, manifest: { schemaVersion: 1, model, stale: true, unavailable: true, reason: error.message } }; + } finally { loadings.delete(model); } })(); - return loading; - } - + loadings.set(model, promise); + return promise; + }; const handler = async (req, res) => { - const path = new URL(req.url, 'http://localhost').pathname; - const state = await refresh(); - if (path === '/status') { - const { gridUrl, ...manifest } = state.manifest; - return sendJson(res, manifest); - } - if (path.startsWith('/grid/')) { - if (!state.grid || path !== `/grid/${state.idGrid}.bin`) - return sendJson(res, { error: 'unknown_grid' }, 404); - const payload = Buffer.concat([ - Buffer.from(state.grid.u.buffer, state.grid.u.byteOffset, state.grid.u.byteLength), - Buffer.from(state.grid.v.buffer, state.grid.v.byteOffset, state.grid.v.byteLength), - ]); - res.writeHead(200, { - 'Content-Type': 'application/octet-stream', - 'Cache-Control': 'public, max-age=3600, immutable', - }); + const url = new URL(req.url, 'http://localhost'); + const model = url.searchParams.get('model') || 'gfs'; + if (!models[model]) return sendJson(res, { error: 'unknown_model' }, 400); + let state = caches.get(model); + if (!state || now() - state.fetchedAt >= ttlMs) state = loadings.get(model) || refresh(model); + state = await state; + if (url.pathname === '/status') { const { gridUrl, ...manifest } = state.manifest; return sendJson(res, manifest); } + if (url.pathname.startsWith('/grid/')) { + if (!state.grid || url.pathname !== `/grid/${state.idGrid}.bin`) return sendJson(res, { error: 'unknown_grid' }, 404); + const payload = Buffer.concat([Buffer.from(state.grid.u.buffer, state.grid.u.byteOffset, state.grid.u.byteLength), Buffer.from(state.grid.v.buffer, state.grid.v.byteOffset, state.grid.v.byteLength)]); + res.writeHead(200, { 'Content-Type': 'application/octet-stream', 'Cache-Control': 'public, max-age=3600, immutable' }); return res.end(payload); } return sendJson(res, state.manifest); }; - - return { - name: 'wind', - configureServer({ middlewares }) { - middlewares.use('/api/wind', handler); - }, - configurePreviewServer({ middlewares }) { - middlewares.use('/api/wind', handler); - }, - }; + return { name: 'wind', configureServer({ middlewares }) { middlewares.use('/api/wind', handler); }, configurePreviewServer({ middlewares }) { middlewares.use('/api/wind', handler); } }; } diff --git a/server/providers/wind/gfs.js b/server/providers/wind/gfs.js index ac637a14d..6caee82b3 100644 --- a/server/providers/wind/gfs.js +++ b/server/providers/wind/gfs.js @@ -87,3 +87,22 @@ export async function fetchRange({ throw new Error('invalid range length'); return buffer; } + +import { GFS_BUCKET, gfsObjectKey, selectLatestGfsCycle } from './catalog.js'; +import { decodeWindGribMessage } from './decode.js'; +import { resampleWindGrid } from './grid.js'; + +/** Fetch and decode the latest GFS 10 m wind field. */ +export async function fetchGfsWind({ fetchImpl = fetch, now = () => Date.now(), targetDx = 1, decodeImpl = decodeWindGribMessage } = {}) { + const cycle = selectLatestGfsCycle(now()); + const base = `https://${GFS_BUCKET}.s3.amazonaws.com/${gfsObjectKey(cycle)}`; + const index = await fetchText({ url: `${base}.idx`, fetchImpl }); + const ranges = windMessageRanges(parseGfsIdx(index.toString())); + const [uBuffer, vBuffer] = await Promise.all([ + fetchRange({ url: base, ...ranges.u, fetchImpl }), + fetchRange({ url: base, ...ranges.v, fetchImpl }), + ]); + const [u, v] = await Promise.all([decodeImpl(uBuffer), decodeImpl(vBuffer)]); + const grid = resampleWindGrid({ u: u.values, v: v.values, ni: u.ni, nj: u.nj, lo1: u.lo1, la1: u.la1, di: u.di, dj: u.dj, dx: targetDx, dy: targetDx }); + return { cycle: { ...cycle, forecastHour: 0, runIso: `${cycle.date.slice(0, 4)}-${cycle.date.slice(4, 6)}-${cycle.date.slice(6)}T${String(cycle.hour).padStart(2, '0')}:00:00.000Z` }, level: '10 m above ground', units: 'm/s', grid }; +} diff --git a/server/providers/wind/ifs.js b/server/providers/wind/ifs.js new file mode 100644 index 000000000..450a83827 --- /dev/null +++ b/server/providers/wind/ifs.js @@ -0,0 +1,122 @@ +import { fetchText, fetchRange } from './gfs.js'; +import { decodeWindGribMessage } from './decode.js'; +import { resampleWindGrid } from './grid.js'; + +/** Keyless ECMWF Open Data root for real-time IFS forecasts. */ +export const IFS_BASE = 'https://data.ecmwf.int/forecasts'; + +/** + * Select the latest IFS cycle that should be available. + * @param {number} nowMs - Reference epoch milliseconds. + * @param {{availabilityLagMs?: number}} [options] + * @returns {{date: string, hour: number}} + */ +export function selectLatestIfsCycle( + nowMs, + { availabilityLagMs = 6 * 3600_000 } = {}, +) { + const shifted = new Date(nowMs - availabilityLagMs); + return { + date: shifted.toISOString().slice(0, 10).replaceAll('-', ''), + hour: Math.floor(shifted.getUTCHours() / 6) * 6, + }; +} + +/** + * Build the IFS GRIB2 and `.index` URLs for one cycle and step. + * @param {{date: string, hour: number, step?: number}} cycle + * @returns {{grib: string, index: string}} + */ +export function ifsObjectUrls({ date, hour, step = 0 }) { + const hh = String(hour).padStart(2, '0'); + const stem = `${IFS_BASE}/${date}/${hh}z/ifs/0p25/oper/${date}${hh}0000-${step}h-oper-fc`; + // The inventory sits beside the GRIB2 as `.index`, NOT `.index`. + return { grib: `${stem}.grib2`, index: `${stem}.index` }; +} + +/** + * Parse the IFS JSON Lines inventory into `{param, offset, length}` records. + * Malformed lines are skipped; a payload with no valid lines is rejected. + * @param {string} text - Raw `.index` body. + * @returns {Array<{param: string, offset: number, length: number}>} + */ +export function parseIfsIndex(text) { + const entries = []; + for (const line of String(text).split(/\r?\n/)) { + if (!line) continue; + let value; + try { + value = JSON.parse(line); + } catch { + continue; // non-JSON line in the index + } + if ( + value?.param && + Number.isFinite(value._offset) && + Number.isFinite(value._length) + ) + entries.push({ + param: value.param, + offset: value._offset, + length: value._length, + }); + } + if (!entries.length) throw new Error('malformed IFS index'); + return entries; +} + +/** + * Byte ranges for the IFS 10 m wind components. + * @param {Array<{param: string, offset: number, length: number}>} entries + * @returns {{u: {start: number, end: number}, v: {start: number, end: number}}} + */ +export function ifsWindRanges(entries) { + const range = (param) => { + const item = entries.find((entry) => entry.param === param); + if (!item) throw new Error(`missing ${param}`); + return { start: item.offset, end: item.offset + item.length - 1 }; + }; + return { u: range('10u'), v: range('10v') }; +} + +/** + * Fetch and decode the latest IFS 10 m wind field. + * @param {{fetchImpl?: Function, now?: Function, targetDx?: number, + * decodeImpl?: Function}} [options] + * @returns {Promise<{cycle: object, level: string, units: string, grid: object}>} + */ +export async function fetchIfsWind({ + fetchImpl = fetch, + now = () => Date.now(), + targetDx = 1, + decodeImpl = decodeWindGribMessage, +} = {}) { + const cycle = selectLatestIfsCycle(now()); + const urls = ifsObjectUrls(cycle); + const index = await fetchText({ url: urls.index, fetchImpl }); + const ranges = ifsWindRanges(parseIfsIndex(index.toString())); + const [uBuffer, vBuffer] = await Promise.all([ + fetchRange({ url: urls.grib, ...ranges.u, fetchImpl }), + fetchRange({ url: urls.grib, ...ranges.v, fetchImpl }), + ]); + const [u, v] = await Promise.all([decodeImpl(uBuffer), decodeImpl(vBuffer)]); + const grid = resampleWindGrid({ + u: u.values, + v: v.values, + ni: u.ni, + nj: u.nj, + lo1: u.lo1, + la1: u.la1, + di: u.di, + dj: u.dj, + dx: targetDx, + dy: targetDx, + }); + const runIso = `${cycle.date.slice(0, 4)}-${cycle.date.slice(4, 6)}-${cycle.date.slice(6)}T${String(cycle.hour).padStart(2, '0')}:00:00.000Z`; + return { + cycle: { ...cycle, forecastHour: 0, runIso }, + level: '10 m above ground', + units: 'm/s', + grid, + }; +} diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 4ec32dcd8..c6a555de9 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -188,6 +188,13 @@ export const DATA_CREDITS = [ 'NOAA Open Data on AWS ' + '(U.S. public domain; forecast, not observations)', }, + { + key: 'ecmwf-ifs', + html: + 'Contains modified Copernicus/ECMWF IFS forecast data, licensed under CC BY 4.0. ' + + 'Source: https://data.ecmwf.int/forecasts/ ' + + '(ECMWF Open Data)', + }, // ── Bundled snapshots ─────────────────────────────────────────── { key: 'datacenters', diff --git a/src/data/layerState.js b/src/data/layerState.js index 85e8abc30..0b492fd51 100644 --- a/src/data/layerState.js +++ b/src/data/layerState.js @@ -182,6 +182,9 @@ function integerOption(key, token, defaultValue) { } const OPTION_GROUPS = Object.freeze({ + wind: Object.freeze([ + enumOption('model', 'm', 'gfs', ['gfs', 'ifs'], { gfs: 'g', ifs: 'i' }), + ]), flights: Object.freeze([ // Owner directive 2026-08-22: the fleet's 3D models are DEFAULT-ON in // PROXIMITY mode. Proximity is itself the altitude/count gate — models only @@ -292,7 +295,7 @@ export const LAYER_STATE_REGISTRY = Object.freeze([ Object.freeze({ id: 'satellites', token: 's', disposition: 'enabled+options', optionOwner: 'satellites' }), Object.freeze({ id: 'telegeography-submarine-cables', token: 'u', disposition: 'enabled-only' }), Object.freeze({ id: 'traffic', token: 't', disposition: 'enabled-only' }), - Object.freeze({ id: 'wind', token: 'k', disposition: 'enabled-only' }), + Object.freeze({ id: 'wind', token: 'k', disposition: 'enabled+options', optionOwner: 'wind' }), ]); export const REGISTERED_LAYER_IDS = Object.freeze(LAYER_STATE_REGISTRY.map((entry) => entry.id)); diff --git a/src/layers/wind/index.js b/src/layers/wind/index.js index 49d468caf..bf0038a8f 100644 --- a/src/layers/wind/index.js +++ b/src/layers/wind/index.js @@ -10,7 +10,7 @@ export function windStats(manifest) { }; } -/** Create the GFS wind data layer. */ +/** Create the GFS/IFS wind data layer. */ export function createWindLayer({ feed, cesium = Cesium, container, services } = {}) { if (typeof feed?.getSnapshot !== 'function') throw new TypeError('Wind requires a snapshot source'); let viewer = null; @@ -19,8 +19,10 @@ export function createWindLayer({ feed, cesium = Cesium, container, services } = let rendering = null; let manifest = null; let error = null; + let model = 'gfs'; + let generation = 0; const layer = { - id: 'wind', name: 'Wind', icon: '🌬', source: 'NOAA GFS', updateInterval: 3600_000, + id: 'wind', name: 'Wind', icon: '🌬', source: 'NOAA GFS / ECMWF IFS', updateInterval: 3600_000, init(nextViewer) { viewer = nextViewer; rendering = createWindRendering({ cesium, container: nextViewer.container, getViewer: () => viewer }); rendering.attach(); }, enable() { enabled = true; rendering?.start(); }, disable() { request?.abort(); request = null; enabled = false; rendering?.stop(); rendering?.clear(); }, @@ -30,7 +32,7 @@ export function createWindLayer({ feed, cesium = Cesium, container, services } = const controller = new AbortController(); request = controller; try { - const snapshot = await feed.getSnapshot({ signal: signal || controller.signal }); + const snapshot = await feed.getSnapshot({ signal: signal || controller.signal, model }); if (!enabled || controller.signal.aborted || signal?.aborted) return false; manifest = snapshot; error = null; @@ -42,8 +44,21 @@ export function createWindLayer({ feed, cesium = Cesium, container, services } = return true; } finally { if (request === controller) request = null; } }, + setParams(params = {}) { + if (!['gfs', 'ifs'].includes(params.model) || params.model === model) return; + model = params.model; + generation += 1; + if (enabled) { + request?.abort(); + const currentGeneration = generation; + queueMicrotask(() => { + if (enabled && currentGeneration === generation) layer.update(viewer); + }); + } + }, + getParams() { return { model }; }, destroy() { request?.abort(); request = null; enabled = false; rendering?.destroy(); rendering = null; viewer = null; }, - getStats() { return { ...windStats(manifest), error: error || windStats(manifest).error }; }, + getStats() { return { ...windStats(manifest), error: error || windStats(manifest).error, model }; }, getParticleCount() { return rendering?.getParticleCount() || 0; }, }; return layer; diff --git a/src/layers/wind/source.js b/src/layers/wind/source.js index 8d5fb6ec6..7d3a60048 100644 --- a/src/layers/wind/source.js +++ b/src/layers/wind/source.js @@ -1,9 +1,9 @@ -/** Create a source that loads the current GFS wind field. */ +/** Create a source that loads the current wind field. */ export function createWindSource({ fetchImpl = (...args) => globalThis.fetch(...args) } = {}) { return { - async getSnapshot({ signal } = {}) { + async getSnapshot({ signal, model = 'gfs' } = {}) { signal?.throwIfAborted(); - const response = await fetchImpl('/api/wind/manifest', { + const response = await fetchImpl(`/api/wind/manifest?model=${encodeURIComponent(model)}`, { signal, cache: 'no-store', });