Merge pull request #1509 from quanticsoul4772/upstream/route-outage-unavailable

fix(routes): report an upstream routing outage as unavailable

changelog.d/1509-route-outage-unavailable.md

@@ -0,0 +1,3 @@
+- Routing outages report "Routing service unavailable" in voice route planning
+  and "Routing unavailable" in Directions instead of "no route found"
+  (quanticsoul4772, #1509).

docs/CURRENT-STATE.md

@@ -5447,7 +5447,14 @@ service's usage policy states; past a bounded queue the answer is a 429 with a
 `Retry-After` rather than a queue that grows until everything times out. The
 upstream host is pinned and a redirect is refused rather than followed, and a
 rate limit from the routing service is reported as one instead of as "no route
-found". `src/data/routeSteps.js` turns OSRM maneuver type / modifier / exit /
+found". Upstream failure statuses except 400 and 429, network failures, and
+unreadable, oversized or non-JSON bodies return `502 routing service unavailable`.
+Voice hears `Routing service unavailable (HTTP 502)`; Directions shows
+`Routing unavailable (HTTP 502)`. OSRM 400 responses or successful responses
+without a usable route return "no route found". The 12 s upstream budget includes
+queue time: expiry before dispatch returns 429 "routing busy" when the queue slot
+opens; expiry during the upstream request returns 502.
+`src/data/routeSteps.js` turns OSRM maneuver type / modifier / exit /
 road name into one plain-English sentence per decision, folding
 `exit roundabout` steps into the roundabout they leave, and caps one route at
 200 maneuvers — a route with more is reported as cut off, in the row summary
@@ -6857,7 +6864,7 @@ apply to `public/` assets or built-preview serving.
 - `/api/google/text-search` keeps the Google key server-side for view-biased Places recovery used by annotation resolution.
 - `/api/overpass` is bounded by body/response caps, per-client/global rate limits, concurrency limits, operator-configured upstreams, in-flight dedupe, cache bounds, and static validation that every selector is spatially bounded.
 - `/api/military-installations` uses an independent limiter with the same 90-per-client/300-global one-minute bounds, so viewport installation refreshes never consume `/api/overpass` annotation capacity.
-- `/api/route` proxies bounded OSRM route requests for annotation routes, with profile allowlisting, distance caps, response caps, caching, and sanitized "no route found" errors.
+- `/api/route` proxies bounded OSRM route requests for annotation routes, with profile allowlisting, distance caps, response caps, caching, and sanitized "no route found" errors; routing outages return 502.
 - `/api/flights`, `/api/military`, and `/api/route` answer `405 Method Not Allowed` with `Allow: GET` to anything but `GET`, before rate limiting and any upstream call.
 - `/api/flights` negotiates gzip. The cached global snapshot and the regional adsb.lol fallback are each gzipped once per refresh (about 860 KB becomes about 250 KB). Clients that offer gzip get that frame; other clients, and bodies under 1 KiB or that gzip cannot shrink, get identity bytes. Both answers carry `Vary: Accept-Encoding`. Uncached bodies, such as auth failures, are never sent under a cached frame's encoding.
 - Track endpoints: `/api/vessels/track?mmsi=` (server-accumulated ring buffers; sub-route handled before the rows snapshot), `/api/flights/track?icao24=` (OAuth, 60s cache, sanitized errors, independent OpenSky credit bucket), `/api/military/track?hex=` (400 for an invalid hex, otherwise `503 history_unavailable` with `Retry-After: 1800` and `Cache-Control: no-store`, no upstream request; military tracking uses its locally accumulated trail).

server/providers/places/routes.js

@@ -72,6 +72,14 @@ class UpstreamRateLimitError extends Error {
   }
 }
 
+/** Thrown when the routing service failed or sent no JSON; answered as 502. */
+class UpstreamUnavailableError extends Error {
+  constructor() {
+    super('routing service unavailable');
+    this.name = 'UpstreamUnavailableError';
+  }
+}
+
 /**
  * Run one outbound request no sooner than the shared gate allows.
  * @template T
@@ -167,18 +175,26 @@ async function fetchRoute({
     `${base.replace(/\/$/, '')}/route/v1/${osrmProfile}/${coords}` +
     `?overview=full&geometries=geojson&alternatives=false&steps=${withSteps ? 'true' : 'false'}`;
   const controller = new AbortController();
+  // The 12 s budget includes queue wait, so Directions (15 s) hears back.
   const timer = setTimeout(() => controller.abort(), 12000);
   let osrm;
   try {
-    const upstreamRes = await throughUpstreamGate(() =>
+    const upstreamRes = await throughUpstreamGate(async () => {
-      fetchImpl(upstream, {
+      // Expiry before dispatch reflects our backlog, not an upstream outage.
-        signal: controller.signal,
+      if (controller.signal.aborted)
-        // The endpoint is configured above; a redirect is the one way out of
+        throw new RouteBusyError(_upstreamIntervalMs);
-        // it, so it is refused rather than followed.
+      try {
-        redirect: 'error',
+        return await fetchImpl(upstream, {
-        headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
+          signal: controller.signal,
-      }),
+          // The endpoint is configured above; a redirect is the one way out of
-    );
+          // it, so it is refused rather than followed.
+          redirect: 'error',
+          headers: { 'User-Agent': 'gods-eye-view/dev (local)' },
+        });
+      } catch {
+        throw new UpstreamUnavailableError();
+      }
+    });
     if (upstreamRes.status === 429) {
       await cancelBody(upstreamRes);
       const retryAfter = Number(upstreamRes.headers.get('retry-after'));
@@ -186,20 +202,28 @@ async function fetchRoute({
         Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : 5,
       );
     }
-    if (!upstreamRes.ok) {
+    // OSRM answers 400 for a route it cannot find; other failures are outages.
+    if (upstreamRes.status === 400) {
       await cancelBody(upstreamRes);
       return { payload: null, error: 'no route found' };
     }
+    if (!upstreamRes.ok) {
+      await cancelBody(upstreamRes);
+      throw new UpstreamUnavailableError();
+    }
     const ctype = upstreamRes.headers.get('content-type') || '';
     if (!ctype.includes('json')) {
       await cancelBody(upstreamRes);
-      return { payload: null, error: 'no route found' };
+      throw new UpstreamUnavailableError();
+    }
+    // Read and parse failures are outages; later ones stay "route proxy error".
+    try {
+      osrm = JSON.parse(
+        await readResponseTextCapped(upstreamRes, ROUTE_MAX_RESPONSE_BYTES),
+      );
+    } catch {
+      throw new UpstreamUnavailableError();
     }
-    const text = await readResponseTextCapped(
-      upstreamRes,
-      ROUTE_MAX_RESPONSE_BYTES,
-    );
-    osrm = JSON.parse(text);
   } finally {
     clearTimeout(timer);
   }
@@ -355,6 +379,11 @@ export function installRouteMiddleware(
             upstreamError.retryAfterSec,
           );
         }
+        if (upstreamError instanceof UpstreamUnavailableError) {
+          res.writeHead(502, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ ok: false, error: upstreamError.message }));
+          return;
+        }
         throw upstreamError;
       }
       if (error || !payload) return fail(error || 'no route found');

src/layers/directions/source.js

@@ -21,7 +21,13 @@ export function createDirectionsSource({
         signal,
         headers: { Accept: 'application/json' },
       });
-      const payload = await response.json();
+      // Proxy error pages may not be JSON.
+      let payload = null;
+      try {
+        payload = await response.json();
+      } catch (error) {
+        if (response.ok) throw error;
+      }
       signal?.throwIfAborted();
       // Failures keep their English message and carry the HTTP status and the
       // server's refusal text, so the layer can show them in the language.
@@ -39,7 +45,8 @@ export function createDirectionsSource({
           { status: 429, refusal },
         );
       }
-      if (!response.ok && !payload?.error)
+      // A 5xx body may mislabel an outage as a route refusal.
+      if (response.status >= 500 || (!response.ok && !payload?.error))
         throw Object.assign(
           new Error(`Routing unavailable (HTTP ${response.status})`),
           { status: response.status },

src/layers/directions/source.test.mjs

@@ -41,3 +41,15 @@ test('route failures are surfaced without fallback requests', async () => {
     assert.equal(calls, 1);
   }
 });
+
+test('a routing outage stays an outage when its body carries an error', async () => {
+  const source = createDirectionsSource({ fetchImpl: async () =>
+    Response.json({ ok: false, error: 'routing service unavailable' }, { status: 502 }) });
+  await assert.rejects(source.getRoute(query), { status: 502, message: 'Routing unavailable (HTTP 502)' });
+});
+
+test('a routing outage page that is not JSON still reads as an outage', async () => {
+  const source = createDirectionsSource({ fetchImpl: async () =>
+    new Response('<html>Bad Gateway</html>', { status: 502, headers: { 'content-type': 'text/html' } }) });
+  await assert.rejects(source.getRoute(query), { status: 502, message: 'Routing unavailable (HTTP 502)' });
+});

src/tooling/placeProviders.test.mjs

@@ -236,23 +236,108 @@ test('OSRM routing preserves aliases, cache, span guards and upstream failure be
     assert.equal((await request(route, url)).body.error, error);
   assert.equal(calls, 1);
   now += 600001;
-  t.mock.method(
+  t.mock.method(globalThis, 'fetch', async () =>
-    globalThis,
+    Response.json(
-    'fetch',
+      { code: 'NoRoute', message: 'Impossible route between points' },
-    async () => new Response('offline', { status: 503 }),
+      { status: 400 },
+    ),
   );
-  assert.deepEqual((await request(route, query)).body, {
+  const noRoute = await request(route, query);
-    ok: false,
+  assert.equal(noRoute.statusCode, 200);
-    error: 'no route found',
+  assert.deepEqual(noRoute.body, { ok: false, error: 'no route found' });
-  });
+  t.mock.method(globalThis, 'fetch', async () =>
-  t.mock.method(
+    Response.json({ code: 'NoRoute', routes: [] }),
-    globalThis,
-    'fetch',
-    async () => new Response('x', { headers: { 'content-type': 'text/html' } }),
   );
   assert.equal((await request(route, query)).body.error, 'no route found');
 });
 
+test('a routing outage is reported as one, not as "no route found"', async (t) => {
+  const route = '/api/route';
+  const query = '?profile=car&coords=-97,30;-97.01,30.01';
+  for (const response of [
+    () => new Response('offline', { status: 503 }),
+    () => new Response('blocked', { status: 403 }),
+    () => new Response('x', { headers: { 'content-type': 'text/html' } }),
+    () =>
+      new Response('<html>', {
+        headers: { 'content-type': 'application/json' },
+      }),
+    () => Promise.reject(new TypeError('fetch failed')),
+    () => Promise.reject(new DOMException('aborted', 'AbortError')),
+  ]) {
+    _resetRouteUpstreamForTest();
+    const request = install(installRouteMiddleware);
+    t.mock.method(globalThis, 'fetch', async () => response());
+    const result = await request(route, query);
+    assert.equal(result.statusCode, 502);
+    assert.deepEqual(result.body, {
+      ok: false,
+      error: 'routing service unavailable',
+    });
+  }
+});
+
+test('callers sharing one failed upstream call all hear the outage, and it is not cached', async (t) => {
+  _resetRouteUpstreamForTest();
+  _setRouteUpstreamIntervalForTest(0);
+  t.after(() => _resetRouteUpstreamForTest());
+  const request = install(installRouteMiddleware);
+  let calls = 0;
+  let release;
+  const gate = new Promise((resolve) => {
+    release = resolve;
+  });
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    await gate;
+    return new Response('offline', { status: 503 });
+  });
+  const route = '/api/route';
+  const query = '?profile=car&coords=-97,30;-97.01,30.01';
+  const all = [
+    request(route, `${query}&steps=1`),
+    request(route, `${query}&steps=1`),
+    request(route, query),
+  ];
+  await Promise.resolve();
+  release();
+  for (const result of await Promise.all(all)) {
+    assert.equal(result.statusCode, 502);
+    assert.equal(result.body.error, 'routing service unavailable');
+  }
+  assert.equal(calls, 1);
+  assert.equal((await request(route, query)).statusCode, 502);
+  assert.equal(calls, 2, 'the next request asks the routing service again');
+});
+
+test('a budget spent in the outbound queue is reported as busy, not as an outage', async (t) => {
+  _resetRouteUpstreamForTest();
+  // The second request waits 13 s for its slot, past the 12 s budget.
+  _setRouteUpstreamIntervalForTest(13_000);
+  t.after(() => _resetRouteUpstreamForTest());
+  t.mock.timers.enable({ apis: ['setTimeout', 'Date'], now: Date.now() });
+  const request = install(installRouteMiddleware);
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return Response.json(osrmRouteWithSteps(2));
+  });
+  const first = request(
+    '/api/route',
+    '?profile=car&coords=-97,30;-97.01,30.01',
+  );
+  const second = request(
+    '/api/route',
+    '?profile=foot&coords=-97,30;-97.01,30.01',
+  );
+  assert.equal((await first).body.ok, true);
+  t.mock.timers.tick(13_000);
+  const result = await second;
+  assert.equal(result.statusCode, 429);
+  assert.match(result.body.error, /routing busy/);
+  assert.equal(calls, 1, 'the spent request never reaches the routing service');
+});
+
 /** One OSRM route with `count` maneuvers, for the step tests. */
 function osrmRouteWithSteps(count) {
   return {

src/tools/places.js

@@ -189,7 +189,13 @@ export function createRouteService({
       } catch {
         /* the status below is authoritative */
       }
+      // Handle 429 and 5xx before route refusals, regardless of the body.
       if (response.status === 429) throw httpFailure(response, 'Routing');
+      if (response.status >= 500)
+        throw new ToolError(
+          'unavailable',
+          `Routing service unavailable (HTTP ${response.status})`,
+        );
       if (payload?.ok === false || response.status === 400)
         return { ok: false, error: payload?.error || 'no route found' };
       if (!response.ok) throw httpFailure(response, 'Routing');

src/tools/queries/places.test.mjs

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
 import { composeCatalog, coreTools } from '../index.js';
+import { createMcpServer } from '../mcp/index.js';
 import { createPlaceSearchService, createRouteService } from '../places.js';
 
 const place = (id, latitude, longitude, extra = {}) => ({
@@ -249,6 +250,91 @@ test('the place and route services read the app routes and translate failures',
   );
 });
 
+test('a routing outage is reported as unavailable, not as invalid arguments', async () => {
+  for (const fetchImpl of [
+    async () =>
+      Response.json(
+        { ok: false, error: 'routing service unavailable' },
+        { status: 502 },
+      ),
+    async () => {
+      throw new TypeError('fetch failed');
+    },
+  ]) {
+    const catalog = composeCatalog({
+      tools: coreTools,
+      services: { routing: createRouteService({ fetchImpl }), places },
+    });
+    const server = createMcpServer({
+      catalog,
+      name: 'test',
+      version: '1.0.0',
+    });
+    const { result } = await server.handle({
+      jsonrpc: '2.0',
+      id: 1,
+      method: 'tools/call',
+      params: {
+        name: 'plan_route',
+        arguments: { from: { place: 'Austin' }, to: { place: 'Dallas' } },
+      },
+    });
+    assert.equal(result.isError, true);
+    assert.equal(result.structuredContent.error, 'unavailable');
+    assert.doesNotMatch(result.content[0].text, /No walk route/);
+  }
+  // Voice receives only the message, so it must name the outage.
+  const outage = createRouteService({
+    fetchImpl: async () =>
+      Response.json(
+        { ok: false, error: 'routing service unavailable' },
+        { status: 502 },
+      ),
+  });
+  await assert.rejects(
+    outage.route(
+      [
+        { lat: 30, lon: -97 },
+        { lat: 30.01, lon: -97.01 },
+      ],
+      'foot',
+    ),
+    { message: 'Routing service unavailable (HTTP 502)' },
+  );
+});
+
+test('an HTML 502 gateway response tells voice the route service is unavailable', async () => {
+  const catalog = composeCatalog({
+    tools: coreTools,
+    services: {
+      routing: createRouteService({
+        fetchImpl: async () =>
+          new Response('<html>Bad Gateway</html>', {
+            status: 502,
+            headers: { 'content-type': 'text/html' },
+          }),
+      }),
+      places,
+    },
+  });
+  const server = createMcpServer({ catalog, name: 'test', version: '1.0.0' });
+  const { result } = await server.handle({
+    jsonrpc: '2.0',
+    id: 1,
+    method: 'tools/call',
+    params: {
+      name: 'plan_route',
+      arguments: { from: { place: 'Austin' }, to: { place: 'Dallas' } },
+    },
+  });
+  assert.equal(result.isError, true);
+  assert.equal(result.structuredContent.error, 'unavailable');
+  assert.equal(
+    result.content[0].text,
+    'Routing service unavailable (HTTP 502)',
+  );
+});
+
 test('place answers say when the search limits may have left places out', async () => {
   const five = Array.from({ length: 5 }, (_, index) =>
     place(`p${index}`, 30.27, -97.74),