From 51e32a30f20ccbb5e7c7ec7878ac444fcd009dd1 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 13:08:45 -0700 Subject: [PATCH] fix(bikeshare): scope city failures to range and report partial outages A failure now ends when its city leaves range or the camera rises above the activation gate, so an empty map no longer reads LOAD FAILED. Late activation and status rejections are ignored like late successes. Partial failure reads DEGRADED, every in-range city failing over cached points reads STALE, and freshness advances only on a refresh that succeeded. Behavioural tests drive the real proximity, update and enable paths. Docs say no licence is published for Toronto and list Social Bicycles and PBSC. The merge also restored the #61 GBFS v3 changelog entry that an earlier merge on main dropped. --- CHANGELOG.md | 11 +- DATA_SOURCES.md | 4 +- docs/CURRENT-STATE.md | 2 +- src/layers/bikeshare/controls.js | 21 +- .../bikeshare/failureReporting.test.mjs | 296 ++++++++++++++++++ src/layers/bikeshare/ingestion.js | 19 +- src/layers/bikeshare/registry.test.mjs | 2 +- src/layers/bikeshare/state.js | 5 +- src/layers/bikeshare/viewport.js | 15 + 9 files changed, 351 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f232443..c851c88d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,19 @@ flag that only a successful *activation* cleared — and an already-loaded city never activates again. A dead feed re-entering camera range therefore latched LOAD FAILED permanently over a map still drawing every other city's stations. - Failure is now tracked per city and reported only when it leaves the layer - with nothing to draw; a city that recovers clears its own failure. + Failure is now tracked per city: a city that fails beside others still + drawing marks the layer DEGRADED, polls that all fail over previously drawn + points read STALE without advancing freshness, and LOAD FAILED is reported + only when nothing is left to draw. A city that recovers clears its own + failure, a failure is dropped once its city leaves range or the camera rises + above the activation altitude, and late results from superseded requests are + ignored. Repointed the three feeds whose operators had moved: Reddy Bikeshare (Buffalo) to Social Bicycles, Tugo (Tucson) and Trinity Metro (Fort Worth) to PBSC. Dropped CoGo (Columbus) and Broward B-cycle (Fort Lauderdale), whose systems have wound down — CoGo's feed reports no stations and last moved a bike in - March 2025. All 31 remaining systems were verified live. + March 2025. The three repointed feeds were verified live. - Add Bike Share Toronto (1,069 stations) to the bikeshare registry. The system publishes the same PBSC GBFS v2 feeds as the Austin, Honolulu and Chattanooga diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 157bab7b8..e983a972d 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -60,8 +60,8 @@ How to read this: | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **NYC DOT Traffic Management Center** (`webcams.nyctmc.org`) | CCTV camera catalog + frames, New York City five boroughs | NYC DOT requires developers to sign a [data-sharing agreement](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) for the camera feed. **Off by default**; enable only under your own agreement | "NYC DOT Traffic Management Center" + NYC DOT's required application disclaimer (in the Data attribution popover) | -| **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | -| **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. The City's open-data package declares no licence and the feed publishes no `license_url`, so it is treated as attribution-only like the other GBFS operators. | "Bike Share Toronto (Toronto Parking Authority)" | +| **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. No licence is published: the City's open-data package declares none and the feed has no `license_url`. Attribution is given to the operator. | "Bike Share Toronto (Toronto Parking Authority)" | | **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | | **Kolumbus / Farte Bysykkel** (Entur Mobility, `api.entur.io/mobility/v2/gbfs`) | Bikeshare availability, Kolumbus Bysykkel (Stavanger/Nord-Jæren) and Farte Bysykkel (Skien/Porsgrunn). Keyless GBFS; the proxy sends the `ET-Client-Name` header Entur requires and is limited to Entur's GBFS paths | [NLOD](https://data.norge.no/nlod/en/2.0) per the [Entur Mobility API docs](https://developer.entur.org/pages-mobility-docs-mobility-v2) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)" | | **JCDecaux Cyclocity** (`api.cyclocity.fr`) | Bikeshare availability, Bysykkel Lillestrøm. Keyless GBFS; the proxy is limited to JCDecaux contract paths | [Licence Ouverte / Etalab](https://developer.jcdecaux.com/files/Open-Licence-fr.pdf) (the feed's `license_url`) — attribution REQUIRED | "Bysykkel Lillestrøm: JCDecaux, Licence Ouverte" | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 10a11f234..0cc9c8ae5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3172,7 +3172,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + NYC DOT TMC (opt-in) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Bike Share Toronto + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/src/layers/bikeshare/controls.js b/src/layers/bikeshare/controls.js index 9e9389c58..58c5f003a 100644 --- a/src/layers/bikeshare/controls.js +++ b/src/layers/bikeshare/controls.js @@ -56,7 +56,7 @@ export function createControls({ state: layerState, services, parts, source }) { /** * Return current layer statistics for the UI status display. - * @returns {{ count: number, lastUpdate: number|null, loading: boolean, loadingLabel?: string, error?: string }} + * @returns {{ count: number, lastUpdate: number|null, loading: boolean, loadingLabel?: string, error?: string, degraded?: boolean, stale?: boolean }} */ getStats() { const stats = { @@ -71,12 +71,19 @@ export function createControls({ state: layerState, services, parts, source }) { : 'scanning nearby systems...'; } // A failed city is only the layer's failure when it left nothing on the - // map. While any city still renders, the layer is working and says so. - if ( - layerState._failedCityIds.size > 0 && - layerState._stationRenderMap.size === 0 - ) { - stats.error = 'GBFS fetch error'; + // map. While other cities still render, the partial failure reads + // DEGRADED; when every in-range city failed and only previously drawn + // points remain, they read STALE. + if (layerState._failedCityIds.size > 0) { + if (layerState._stationRenderMap.size === 0) { + stats.error = 'GBFS fetch error'; + } else { + stats.degraded = true; + const allFailed = Array.from(layerState._activeCityIds).every( + (cityId) => layerState._failedCityIds.has(cityId), + ); + if (allFailed) stats.stale = true; + } } return stats; }, diff --git a/src/layers/bikeshare/failureReporting.test.mjs b/src/layers/bikeshare/failureReporting.test.mjs index 63034078b..89a87bdad 100644 --- a/src/layers/bikeshare/failureReporting.test.mjs +++ b/src/layers/bikeshare/failureReporting.test.mjs @@ -78,3 +78,299 @@ test('a recovered city clears its own failure', () => { 'the failure outlived the city that caused it', ); }); + +// The cases above pin how getStats() reads failure state. The cases below +// drive the real proximity check, status poll and lifecycle against a fake +// GBFS source, so the failure state they read is the state the layer builds. + +import * as Cesium from 'cesium'; +import { CITY_BY_ID } from './registry.js'; +import { createIngestion } from './ingestion.js'; +import { createModel } from './model.js'; +import { createViewport } from './viewport.js'; +import { createRendering } from './rendering.js'; +import { createLifecycle } from './lifecycle.js'; + +const TORONTO = 'toronto-bike-share'; +const BUFFALO = 'buffalo-reddy'; +// Within range of both Toronto and Buffalo. +const BETWEEN = { lat: 43.25, lon: -79.1 }; +// Within range of Toronto only. +const TORONTO_ONLY = { lat: 43.7, lon: -79.6 }; +// Within range of Buffalo only. +const BUFFALO_ONLY = { lat: 42.5, lon: -78.0 }; +const LOW_M = 10_000; + +// disable() detaches a keydown listener; node has no document. +globalThis.document ??= { + addEventListener() {}, + removeEventListener() {}, +}; + +/** A GBFS source whose per-city answers each test can change. */ +function makeSource() { + const cityOf = (url) => { + for (const [id, city] of CITY_BY_ID) { + if (url === city.stationInformationUrl) return { id, kind: 'info' }; + if (url === city.stationStatusUrl) return { id, kind: 'status' }; + } + throw new Error(`unexpected GBFS url ${url}`); + }; + const source = { + dead: new Set(), + deadStatus: new Set(), + /** Optional per-call override: (cityId, kind) => Promise | undefined. */ + hook: null, + async getStations(url, { signal } = {}) { + const { id, kind } = cityOf(url); + const hooked = source.hook?.(id, kind); + if (hooked) return hooked; + await new Promise((resolve) => setTimeout(resolve, 1)); + signal?.throwIfAborted(); + if (source.dead.has(id)) throw new Error('GBFS HTTP 404'); + if (kind === 'status' && source.deadStatus.has(id)) + throw new Error('GBFS HTTP 503'); + const city = CITY_BY_ID.get(id); + if (kind === 'info') { + return { + data: { + stations: [ + { + station_id: `${id}-1`, + name: `${id} dock`, + lat: city.centerLat, + lon: city.centerLon, + capacity: 10, + }, + ], + }, + }; + } + return { + data: { + stations: [ + { + station_id: `${id}-1`, + num_bikes_available: 3, + num_docks_available: 7, + is_installed: true, + is_renting: true, + is_returning: true, + }, + ], + }, + }; + }, + }; + return source; +} + +/** Assemble the layer from its real parts around a camera the test moves. */ +function makeLiveLayer() { + const source = makeSource(); + const services = { + render: { governorRequestRender() {} }, + overlays: { + setOverlayEntries() {}, + setOverlaySourceVisible() {}, + clearOverlaySource() {}, + }, + sprites: { registerSpriteCollection() {}, restoreSpriteOrder() {} }, + picking: { registerPickOwner() {}, unregisterPickOwner() {} }, + }; + const state = createState({ services }); + const parts = {}; + const context = { state, services, parts, source }; + parts.ingestion = createIngestion(context); + parts.model = createModel(context); + parts.viewport = createViewport(context); + parts.rendering = createRendering(context); + // Selection owns a DOM click handler; nothing here selects a station. + parts.selection = { + _installClickHandler() {}, + _clearSelection() {}, + _onKeyDown() {}, + }; + parts.queries = { collectDetectableStations: () => [] }; + parts.lifecycle = createLifecycle(context); + const { methods: controls } = createControls(context); + + const camera = { ...BETWEEN, height: LOW_M }; + const viewer = { + camera: { + get positionCartographic() { + return Cesium.Cartographic.fromDegrees( + camera.lon, + camera.lat, + camera.height, + ); + }, + computeViewRectangle: () => undefined, + changed: { addEventListener() {}, removeEventListener() {} }, + }, + scene: { globe: {}, primitives: { add() {}, remove() {} } }, + }; + parts.lifecycle.methods.init(viewer); + + return { + state, + source, + stats: () => controls.getStats(), + moveTo(place, height = LOW_M) { + Object.assign(camera, place, { height }); + return parts.viewport.runProximityCheck(); + }, + async enable() { + parts.lifecycle.methods.enable(viewer); + await settle(); + }, + disable: () => parts.lifecycle.methods.disable(viewer), + poll: () => parts.ingestion.methods.update(), + }; +} + +/** Let fire-and-forget activation started by enable() finish. */ +async function settle() { + for (let i = 0; i < 5; i += 1) + await new Promise((resolve) => setTimeout(resolve, 2)); +} + +function deferred() { + let resolve, reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +test('a city that fails beside one that renders reads DEGRADED, not healthy', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + + const stats = layer.stats(); + assert.equal(stats.count, 1, 'Toronto should still draw'); + assert.equal(stats.error, undefined); + assert.equal(stats.degraded, true, 'the Buffalo failure was hidden'); + assert.equal(stats.stale, undefined); +}); + +test('a failed city that comes back clears its own failure', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + assert.equal(layer.stats().degraded, true); + + layer.source.dead.clear(); + await layer.moveTo(BETWEEN); + const stats = layer.stats(); + assert.equal(stats.count, 2); + assert.equal(stats.degraded, undefined); + assert.equal(stats.error, undefined); +}); + +test('a status poll that recovers clears the failure the last poll set', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(TORONTO_ONLY); + await layer.enable(); + layer.source.deadStatus.add(TORONTO); + await layer.poll(); + assert.equal(layer.stats().degraded, true); + + layer.source.deadStatus.clear(); + await layer.poll(); + assert.equal(layer.stats().degraded, undefined); + assert.equal(layer.stats().stale, undefined); +}); + +test('a failure does not outlive its city leaving range', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + await layer.moveTo(BUFFALO_ONLY); + assert.equal(layer.stats().error, 'GBFS fetch error'); + + // Low over the open Atlantic: nothing in range is empty, not broken. + await layer.moveTo({ lat: 30, lon: -40 }); + const stats = layer.stats(); + assert.equal(stats.count, 0); + assert.equal(stats.error, undefined, 'LOAD FAILED over empty ocean'); + assert.equal(stats.degraded, undefined); +}); + +test('zooming out above the altitude gate drops a city failure', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + await layer.moveTo(BUFFALO_ONLY); + assert.equal(layer.stats().error, 'GBFS fetch error'); + + await layer.moveTo(BUFFALO_ONLY, 200_000); + const stats = layer.stats(); + assert.equal(stats.count, 0); + assert.equal(stats.error, undefined, 'LOAD FAILED above the gate'); +}); + +test('a late rejection from before a disable/re-enable is ignored', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(BUFFALO_ONLY); + const first = deferred(); + // The first info request hangs and ignores its abort signal. + layer.source.hook = (id, kind) => { + if (id !== BUFFALO || kind !== 'info') return undefined; + layer.source.hook = null; + return first.promise; + }; + await layer.enable(); + layer.disable(); + await layer.enable(); + assert.equal(layer.stats().count, 1, 're-enable should draw Buffalo'); + + first.reject(new Error('GBFS HTTP 500')); + await settle(); + const stats = layer.stats(); + assert.equal(stats.count, 1, 'a stale rejection tore down live points'); + assert.equal(stats.error, undefined); + assert.equal(stats.degraded, undefined); +}); + +test('a late status poll rejection from before a disable/re-enable is ignored', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(TORONTO_ONLY); + await layer.enable(); + const first = deferred(); + layer.source.hook = (id, kind) => { + if (id !== TORONTO || kind !== 'status') return undefined; + layer.source.hook = null; + return first.promise; + }; + const poll = layer.poll(); + layer.disable(); + await layer.enable(); + + first.reject(new Error('GBFS HTTP 500')); + await poll; + const stats = layer.stats(); + assert.equal(stats.count, 1); + assert.equal(stats.degraded, undefined); + assert.equal(stats.error, undefined); +}); + +test('every status poll failing over cached points reads STALE, not fresh', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(TORONTO_ONLY); + await layer.enable(); + const loadedAt = layer.stats().lastUpdate; + assert.ok(loadedAt, 'activation should record freshness'); + + layer.source.deadStatus.add(TORONTO); + await new Promise((resolve) => setTimeout(resolve, 5)); + await layer.poll(); + const stats = layer.stats(); + assert.equal(stats.count, 1, 'the cached points stay drawn'); + assert.equal(stats.error, undefined); + assert.equal(stats.degraded, true); + assert.equal(stats.stale, true); + assert.equal(stats.lastUpdate, loadedAt, 'a failed poll advanced freshness'); +}); diff --git a/src/layers/bikeshare/ingestion.js b/src/layers/bikeshare/ingestion.js index cda677766..f2127e53e 100644 --- a/src/layers/bikeshare/ingestion.js +++ b/src/layers/bikeshare/ingestion.js @@ -171,20 +171,21 @@ export function createIngestion({ const generation = layerState._proximityGeneration; const cityIds = Array.from(layerState._activeCityIds); + const isCurrent = (cityId) => + layerState._enabled && + layerState._activeCityIds.has(cityId) && + generation === layerState._proximityGeneration; + let refreshed = false; await Promise.all( cityIds.map(async (cityId) => { try { const statusMap = await loadCityStationStatus(cityId, generation); - if ( - !layerState._enabled || - !layerState._activeCityIds.has(cityId) || - generation !== layerState._proximityGeneration - ) - return; + if (!isCurrent(cityId)) return; parts.rendering.applyStatusToPoints(cityId, statusMap); layerState._failedCityIds.delete(cityId); + refreshed = true; } catch (error) { - if (error?.name === 'AbortError') return; + if (error?.name === 'AbortError' || !isCurrent(cityId)) return; console.warn( `[Data:Bikeshare] ${cityId} status update error:`, error, @@ -195,7 +196,9 @@ export function createIngestion({ ); layerState._count = layerState._stationRenderMap.size; - if (layerState._count > 0) layerState._lastUpdate = Date.now(); + // Freshness advances only on a poll that actually refreshed a city. + if (refreshed && layerState._count > 0) + layerState._lastUpdate = Date.now(); }, }; diff --git a/src/layers/bikeshare/registry.test.mjs b/src/layers/bikeshare/registry.test.mjs index 30e708957..74a103079 100644 --- a/src/layers/bikeshare/registry.test.mjs +++ b/src/layers/bikeshare/registry.test.mjs @@ -23,7 +23,7 @@ test('every registered city is reachable through the GBFS proxy allowlist', () = `${city.id}: host ${url.hostname} is not allowlisted`, ); assert.equal( - isAllowedGbfsPath(url.pathname), + isAllowedGbfsPath(url.pathname, url.hostname), true, `${city.id}: path ${url.pathname} is not allowlisted`, ); diff --git a/src/layers/bikeshare/state.js b/src/layers/bikeshare/state.js index 728c1c124..3617914bf 100644 --- a/src/layers/bikeshare/state.js +++ b/src/layers/bikeshare/state.js @@ -104,8 +104,9 @@ export function createState({ services }) { * * Per city rather than one layer-wide flag: feeds are fetched independently, * so a retired operator must not mark a layer that other cities are still - * populating as failed. The layer reports failure only when these cost it - * everything it had to draw — see `getStats()` in controls.js. + * populating as failed. Entries are dropped when their city leaves range. + * The layer reports failure only when these cost it everything it had to + * draw, and degraded otherwise — see `getStats()` in controls.js. */ layerState._failedCityIds = new Set(); diff --git a/src/layers/bikeshare/viewport.js b/src/layers/bikeshare/viewport.js index b4e63e1dd..25a4df7d0 100644 --- a/src/layers/bikeshare/viewport.js +++ b/src/layers/bikeshare/viewport.js @@ -122,6 +122,9 @@ export function createViewport({ state: layerState, services, parts, source }) { const cityIds = Array.from(layerState._activeCityIds); for (const cityId of cityIds) deactivateCity(cityId); layerState._activeCityIds.clear(); + // A failure belongs to a city in range; none are once everything is off. + layerState._failedCityIds.clear(); + layerState._count = 0; } /** @@ -171,6 +174,14 @@ export function createViewport({ state: layerState, services, parts, source }) { layerState._failedCityIds.delete(cityId); } catch (error) { if (error?.name === 'AbortError') return; + // A rejection from superseded work must not mark (or tear down) a city + // that a newer proximity check or re-enable now owns. + if ( + !layerState._enabled || + !layerState._activeCityIds.has(cityId) || + generation !== layerState._proximityGeneration + ) + return; console.warn(`[Data:Bikeshare] ${cityId} activate error:`, error); layerState._failedCityIds.add(cityId); deactivateCity(cityId); @@ -208,6 +219,10 @@ export function createViewport({ state: layerState, services, parts, source }) { deactivateCity(cityId); } } + // A failure never outlives its city's range. + for (const cityId of layerState._failedCityIds) { + if (!nextActive.has(cityId)) layerState._failedCityIds.delete(cityId); + } layerState._activeCityIds = nextActive; if (layerState._activeCityIds.size === 0) {