Merge main into feat/voice-core and reconcile layer capabilities

src/data/cctvVegvesen.test.mjs

@@ -0,0 +1,202 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+
+import {
+  loadVegvesenSourcesFromOpenData,
+  vegvesenCameraToSource,
+} from '../../server/providers/cctv/sources.js';
+import {
+  DEFAULT_VEGVESEN_CCTV_URL,
+  VEGVESEN_IMAGE_ORIGIN,
+} from '../../server/providers/cctv/constants.js';
+import { createCctvCatalog } from '../../server/providers/cctv/catalog.js';
+
+/** One `datex_3_1:CctvSimple` feature, shaped like the live OGC payload. */
+const feature = (props = {}, coordinates = [5.459189, 61.832706]) => ({
+  type: 'Feature',
+  geometry: { type: 'Point', coordinates },
+  properties: {
+    stillImageUrl: 'https://kamera.atlas.vegvesen.no/api/images/3000063_1',
+    cameraId: '3000063_1',
+    roadNumber: 'F614',
+    'status.stillImageAvailability': 'videoOrImagesAvailable',
+    orientationDescription: 'Svelgen',
+    description: 'Langesi',
+    ...props,
+  },
+});
+
+test('a Vegvesen feature maps to a source on the pinned frame host', () => {
+  const source = vegvesenCameraToSource(feature());
+  assert.equal(source.id, 'no-vegvesen-3000063_1');
+  assert.equal(source.name, 'F614 Langesi → Svelgen');
+  assert.equal(source.city, 'Norway');
+  assert.equal(source.cityId, 'norway');
+  assert.equal(source.provider, 'Statens vegvesen');
+  assert.equal(source.lat, 61.832706);
+  assert.equal(source.lon, 5.459189);
+  assert.equal(source.headingConfidence, 'low');
+  assert.equal(source.sourceKind, 'vegvesen-datex');
+  assert.equal(source.code, 'LANGESI');
+  assert.ok(source.url.startsWith(VEGVESEN_IMAGE_ORIGIN));
+  assert.equal(source.snapshotUrl, source.url);
+});
+
+test('cameras that publish HLS get live video with the still as fallback', (t) => {
+  const video = {
+    videoServiceLevel: 1,
+    videoEncodingStandard: 'hls',
+    videoUrl: 'https://kamera.vegvesen.no/public/3000063_1/manifest.m3u8',
+  };
+  const source = vegvesenCameraToSource(feature(video));
+  assert.equal(source.feedType, 'hls');
+  assert.equal(source.url, video.videoUrl);
+  assert.equal(
+    source.snapshotUrl,
+    'https://kamera.atlas.vegvesen.no/api/images/3000063_1',
+  );
+  // A manifest anywhere but the camera's own path stays a still.
+  const offPath = vegvesenCameraToSource(
+    feature({ ...video, videoUrl: 'https://evil.test/x/manifest.m3u8' }),
+  );
+  assert.equal(offPath.feedType, 'image');
+  assert.equal(offPath.url, offPath.snapshotUrl);
+  // The kill switch keeps every camera on stills.
+  const saved = process.env.CCTV_VEGVESEN_VIDEO;
+  t.after(() => {
+    if (saved === undefined) delete process.env.CCTV_VEGVESEN_VIDEO;
+    else process.env.CCTV_VEGVESEN_VIDEO = saved;
+  });
+  process.env.CCTV_VEGVESEN_VIDEO = '0';
+  assert.equal(vegvesenCameraToSource(feature(video)).feedType, 'image');
+});
+
+test('faulty cameras, off-host frames and bad geometry are dropped', () => {
+  assert.equal(
+    vegvesenCameraToSource(
+      feature({
+        'status.stillImageAvailability':
+          'videoOrImagesUnavailableDueToCameraFault',
+      }),
+    ),
+    null,
+  );
+  assert.equal(
+    vegvesenCameraToSource(
+      feature({ stillImageUrl: 'https://evil.test/api/images/3000063_1' }),
+    ),
+    null,
+  );
+  // The frame URL must be exactly the camera's own image path.
+  assert.equal(
+    vegvesenCameraToSource(
+      feature({
+        stillImageUrl: 'https://kamera.atlas.vegvesen.no/api/images/999_1',
+      }),
+    ),
+    null,
+  );
+  assert.equal(vegvesenCameraToSource(feature({ cameraId: '../x' })), null);
+  // Copenhagen: a plausible coordinate, but outside the Norway box.
+  assert.equal(vegvesenCameraToSource(feature({}, [12.57, 55.68])), null);
+  assert.equal(vegvesenCameraToSource(feature({}, [5.4])), null);
+  assert.equal(vegvesenCameraToSource(null), null);
+});
+
+test('the loader dedupes, and failures degrade to an empty pack', async (t) => {
+  t.mock.method(console, 'log', () => {});
+  t.mock.method(console, 'warn', () => {});
+  const requested = [];
+  const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => {
+    requested.push(String(url));
+    return Response.json({
+      type: 'FeatureCollection',
+      features: [
+        feature(),
+        feature(),
+        feature(
+          {
+            cameraId: '0629001_1',
+            stillImageUrl:
+              'https://kamera.atlas.vegvesen.no/api/images/0629001_1',
+            description: 'Oslo S',
+          },
+          [10.752, 59.911],
+        ),
+      ],
+    });
+  });
+  const cameras = await loadVegvesenSourcesFromOpenData();
+  assert.deepEqual(requested, [DEFAULT_VEGVESEN_CCTV_URL]);
+  // Nearest-to-anchor first: Oslo leads.
+  assert.deepEqual(
+    cameras.map((c) => c.id),
+    ['no-vegvesen-0629001_1', 'no-vegvesen-3000063_1'],
+  );
+
+  fetchMock.mock.mockImplementation(
+    async () => new Response('nope', { status: 503 }),
+  );
+  assert.deepEqual(await loadVegvesenSourcesFromOpenData(), []);
+  fetchMock.mock.mockImplementation(async () => {
+    throw new Error('offline');
+  });
+  assert.deepEqual(await loadVegvesenSourcesFromOpenData(), []);
+});
+
+const runCatalog = async (t) => {
+  const requested = [];
+  t.mock.method(console, 'log', () => {});
+  t.mock.method(console, 'warn', () => {});
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    const href = String(url);
+    requested.push(href);
+    if (href === DEFAULT_VEGVESEN_CCTV_URL) {
+      return Response.json({
+        type: 'FeatureCollection',
+        features: [feature()],
+      });
+    }
+    return Response.json([]);
+  });
+  const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })();
+  return { requested, sources };
+};
+
+const withEnv = async (patch, fn) => {
+  const saved = { ...process.env };
+  try {
+    delete process.env.CCTV_SOURCES_FILE;
+    delete process.env.CCTV_SOURCES_JSON;
+    delete process.env.CCTV_VEGVESEN_ENABLED;
+    Object.assign(process.env, patch);
+    await fn();
+  } finally {
+    for (const key of Object.keys(process.env)) {
+      if (!(key in saved)) delete process.env[key];
+    }
+    Object.assign(process.env, saved);
+  }
+};
+
+test('the Vegvesen lane is wired into the catalog', async (t) => {
+  await withEnv({}, async () => {
+    const { requested, sources } = await runCatalog(t);
+    assert.ok(requested.includes(DEFAULT_VEGVESEN_CCTV_URL));
+    assert.deepEqual(
+      sources.filter((s) => s.cityId === 'norway').map((s) => s.id),
+      ['no-vegvesen-3000063_1'],
+    );
+  });
+});
+
+test('CCTV_VEGVESEN_ENABLED=0 keeps the lane from being loaded', async (t) => {
+  await withEnv({ CCTV_VEGVESEN_ENABLED: '0' }, async () => {
+    const { requested, sources } = await runCatalog(t);
+    assert.equal(requested.includes(DEFAULT_VEGVESEN_CCTV_URL), false);
+    assert.deepEqual(
+      sources.filter((s) => s.cityId === 'norway'),
+      [],
+    );
+  });
+});

src/data/dataCredits.js

@@ -240,6 +240,13 @@ export const DATA_CREDITS = [
       'Traffic cameras (Calgary): contains information licensed under the ' +
       '<a href="https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa" target="_blank" rel="noopener">Open Government Licence – City of Calgary</a>',
   },
+  {
+    key: 'vegvesen-cctv',
+    html:
+      'Road cameras (Norway): contains data under the ' +
+      '<a href="https://data.norge.no/nlod/en/2.0" target="_blank" rel="noopener">Norwegian licence for Open Government data (NLOD)</a> ' +
+      'distributed by Statens vegvesen',
+  },
   {
     key: 'gbfs',
     html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)',

src/data/layerState.js

@@ -410,6 +410,18 @@ const OPTION_GROUPS = Object.freeze({
     ),
     booleanOption('viirs', 'v', false),
   ]),
+  'street-level': Object.freeze([
+    // One switch per registered imagery provider (src/app/layers/streetLevel.js).
+    booleanOption('mapillary', 'm', true),
+    enumOption('pano', 'p', 'all', ['all', 'pano', 'flat'], {
+      all: 'a',
+      pano: 'p',
+      flat: 'f',
+    }),
+    // Relative days, so a link means the same next year. The max matches
+    // MAX_SINCE_DAYS in streetLevel/policy.js (pinned by layerState.test).
+    boundedIntegerOption('sinceDays', 's', 0, { min: 0, max: 36500 }),
+  ]),
   radio: Object.freeze([
     Object.freeze({
       key: 'filter',
@@ -631,6 +643,12 @@ export const LAYER_STATE_REGISTRY = Object.freeze([
     disposition: 'enabled+options',
     optionOwner: 'satellites',
   }),
+  Object.freeze({
+    id: 'street-level',
+    token: '0',
+    disposition: 'enabled+options',
+    optionOwner: 'street-level',
+  }),
   Object.freeze({
     id: 'telegeography-submarine-cables',
     token: 'u',

src/data/layerState.test.mjs

@@ -26,6 +26,8 @@ import {
 import { LayerStateCoordinator } from './layerStateCoordinator.js';
 import radioLayer from './radio.js';
 import { stampInitialShareGesture } from '../navigationPolicy.js';
+import { MAX_SINCE_DAYS } from '../layers/streetLevel/policy.js';
+import { SINCE_STOPS } from '../ui/streetLevelPresentation.js';
 
 function deferred() {
   let resolve;
@@ -199,8 +201,8 @@ function encode(state) {
 
 test('production registry is exact, canonical, and rejects incomplete contracts', async () => {
   assert.equal(validateLayerStateRegistry(), true);
-  assert.equal(REGISTERED_LAYER_IDS.length, 28);
-  assert.equal(new Set(REGISTERED_LAYER_IDS).size, 28);
+  assert.equal(REGISTERED_LAYER_IDS.length, 29);
+  assert.equal(new Set(REGISTERED_LAYER_IDS).size, 29);
   assert.ok(REGISTERED_LAYER_IDS.includes('transit'));
   assert.deepEqual(REGISTERED_LAYER_IDS, [...REGISTERED_LAYER_IDS].sort());
   assert.deepEqual(LEGACY_LAYER_STATE_TOKENS, {
@@ -239,15 +241,15 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
   for (const [id, token] of Object.entries(LEGACY_LAYER_STATE_TOKENS)) {
     assert.equal(LAYER_STATE_TOKEN_RESERVATIONS[id], token);
   }
-  assert.equal(nextLayerStateToken(), '0');
+  assert.equal(nextLayerStateToken(), '3');
   assert.equal(
-    nextLayerStateToken({ ...LAYER_STATE_TOKEN_RESERVATIONS, alpha: '0', bravo: '3' }),
-    '4',
+    nextLayerStateToken({ ...LAYER_STATE_TOKEN_RESERVATIONS, alpha: '3', bravo: '4' }),
+    '5',
   );
   const digitsExhausted = {
     ...LAYER_STATE_TOKEN_RESERVATIONS,
     ...Object.fromEntries(
-      [...'03456789'].map((digit) => [`prior-${digit}`, digit]),
+      [...'3456789'].map((digit) => [`prior-${digit}`, digit]),
     ),
   };
   assert.equal(nextLayerStateToken(digitsExhausted), '00');
@@ -298,7 +300,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
   assert.equal(
     validateLayerStateAllocations(
       LAYER_STATE_TOKEN_RESERVATIONS,
-      { ...LAYER_STATE_TOKEN_RESERVATIONS, future: '0', next: '3' },
+      { ...LAYER_STATE_TOKEN_RESERVATIONS, future: '3', next: '4' },
     ),
     true,
   );
@@ -307,7 +309,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
       ...LAYER_STATE_TOKEN_RESERVATIONS,
       future: '00',
     }),
-    /next free token 0/,
+    /next free token 3/,
   );
   const beforeLastDigit = { ...digitsExhausted };
   delete beforeLastDigit['prior-9'];
@@ -328,10 +330,10 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
   assert.throws(
     () =>
       validateLayerStateAllocations(
-        { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '0' },
-        { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '0', competing: '0' },
+        { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '3' },
+        { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '3', competing: '3' },
       ),
-    /next free token 3/,
+    /next free token 4/,
   );
   assert.throws(
     () => validateLayerStateAllocations({ future: '00' }, { future: '01' }),
@@ -2642,3 +2644,62 @@ test('fire perimeters uses digit 2 without colliding with wind or recent imagery
     decoded,
   );
 });
+
+/** Street Level options decoded from a v2 link with these `lo` assignments. */
+function streetLevelOptions(lo) {
+  const params = new URLSearchParams([
+    ['v', '2'],
+    ['l', '0'],
+  ]);
+  if (lo) params.set('lo', lo);
+  return decodeLayerStateParams(params).options['street-level'];
+}
+
+test('Street Level: every option round-trips through a share link and stored state', () => {
+  const state = createDefaultLayerState();
+  state.enabledLayerIds = ['street-level'];
+  state.options['street-level'] = {
+    mapillary: false,
+    pano: 'flat',
+    sinceDays: MAX_SINCE_DAYS,
+  };
+  const decoded = decodeLayerStateParams(new URLSearchParams(encode(state)));
+  assert.deepEqual(decoded.options['street-level'], {
+    mapillary: false,
+    pano: 'flat',
+    sinceDays: MAX_SINCE_DAYS,
+  });
+  assert.deepEqual(
+    parseStoredLayerState(serializeStoredLayerState(state)).options[
+      'street-level'
+    ],
+    state.options['street-level'],
+  );
+});
+
+test('Street Level: the link codec accepts exactly the windows the filter keeps', () => {
+  // A window past MAX_SINCE_DAYS reads as "any date".
+  assert.equal(
+    streetLevelOptions(`0.s.${MAX_SINCE_DAYS}`).sinceDays,
+    MAX_SINCE_DAYS,
+  );
+  assert.equal(streetLevelOptions(`0.s.${MAX_SINCE_DAYS + 1}`).sinceDays, 0);
+  // Every SINCE slider stop survives, the LAST 10 YEARS one included.
+  for (const { days } of SINCE_STOPS)
+    assert.equal(streetLevelOptions(`0.s.${days}`).sinceDays, days);
+  assert.equal(streetLevelOptions('0.s.3652').sinceDays, 3652);
+});
+
+test('Street Level: a link without the provider switch keeps Mapillary on', () => {
+  assert.deepEqual(streetLevelOptions(''), {
+    mapillary: true,
+    pano: 'all',
+    sinceDays: 0,
+  });
+  assert.equal(streetLevelOptions('0.p.f').mapillary, true);
+  assert.equal(streetLevelOptions('0.m.0').mapillary, false);
+  assert.equal(
+    createDefaultLayerState().options['street-level'].mapillary,
+    true,
+  );
+});

src/data/layerStateTokenLedger.test.mjs

@@ -122,10 +122,10 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed
     { ...parseLayerStateTokenReservations(reservationRows) },
     { ...LAYER_STATE_TOKEN_RESERVATIONS },
   );
-  assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 28);
+  assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 29);
   assert.deepEqual(
     { ...LAYER_STATE_TOKEN_RESERVATIONS },
-    { ...LEGACY_LAYER_STATE_TOKENS },
+    { ...LEGACY_LAYER_STATE_TOKENS, 'street-level': '0' },
   );
   assert.throws(
     () => parseLayerStateTokenReservations(reservationRows.slice(1)),
@@ -145,9 +145,9 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed
     /Duplicate layer-state token reservation/,
   );
   assert.equal(
-    parseLayerStateTokenReservations([...reservationRows, ['future', '0']])
+    parseLayerStateTokenReservations([...reservationRows, ['future', '3']])
       .future,
-    '0',
+    '3',
   );
   assert.throws(
     () =>
@@ -165,14 +165,14 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed
 
 test('published retired digits and competing PRs advance the merge-time allocation', () => {
   withPublishedBase(
-    { rows: [...reservationRows, ['retired-layer', '0']] },
+    { rows: [...reservationRows, ['retired-layer', '3']] },
     (cwd) => {
       const published = readPublishedLayerStateReservations('HEAD', cwd);
-      assert.equal(nextLayerStateToken(published), '3');
+      assert.equal(nextLayerStateToken(published), '4');
       assert.equal(
         validateLayerStateAllocations(published, {
           ...published,
-          newcomer: '3',
+          newcomer: '4',
         }),
         true,
       );
@@ -180,15 +180,15 @@ test('published retired digits and competing PRs advance the merge-time allocati
         () =>
           validateLayerStateAllocations(published, {
             ...published,
-            competing: '0',
+            competing: '3',
           }),
-        /next free token 3/,
+        /next free token 4/,
       );
       assert.throws(
         () =>
           validateLayerStateAllocations(published, {
             ...LAYER_STATE_TOKEN_RESERVATIONS,
-            newcomer: '0',
+            newcomer: '3',
           }),
         /changed or removed: retired-layer/,
       );
@@ -196,8 +196,8 @@ test('published retired digits and competing PRs advance the merge-time allocati
   );
 });
 
-test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => {
-  withPublishedBase({ rows: [...reservationRows, ['pr-a', '0']] }, (cwd) => {
+test('PR B manually replaces provisional 3 with 4 after PR A publishes 3', () => {
+  withPublishedBase({ rows: [...reservationRows, ['pr-a', '3']] }, (cwd) => {
     mkdirSync(path.join(cwd, 'scripts'));
     const fixtureChecker = path.join(
       cwd,
@@ -218,8 +218,8 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () =>
 
     writeCodecFixture(
       cwd,
-      [...reservationRows, ['pr-b', '0']],
-      [{ id: 'pr-b', token: '0' }],
+      [...reservationRows, ['pr-b', '3']],
+      [{ id: 'pr-b', token: '3' }],
     );
     const beforeRebase = runCheck();
     assert.equal(beforeRebase.status, 1);
@@ -227,10 +227,10 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () =>
 
     writeCodecFixture(
       cwd,
-      [...reservationRows, ['pr-a', '0'], ['pr-b', '0']],
+      [...reservationRows, ['pr-a', '3'], ['pr-b', '3']],
       [
-        { id: 'pr-a', token: '0' },
-        { id: 'pr-b', token: '0' },
+        { id: 'pr-a', token: '3' },
+        { id: 'pr-b', token: '3' },
       ],
     );
     const staleRebase = runCheck();
@@ -238,16 +238,16 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () =>
     assert.match(staleRebase.stderr, /Duplicate layer-state token reservation/);
 
     const published = readPublishedLayerStateReservations('HEAD', cwd);
-    assert.equal(nextLayerStateToken(published), '3');
-    const validBaselineRows = [...reservationRows, ['pr-a', '0']];
-    writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '0' }]);
+    assert.equal(nextLayerStateToken(published), '4');
+    const validBaselineRows = [...reservationRows, ['pr-a', '3']];
+    writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '3' }]);
     const helper = spawnSync(
       process.execPath,
       [realpathSync(fixtureNext), 'pr-b'],
       { cwd, encoding: 'utf8' },
     );
     assert.equal(helper.status, 0, helper.stderr);
-    assert.match(helper.stdout, /pr-b: 3/);
+    assert.match(helper.stdout, /pr-b: 4/);
     assert.deepEqual(
       JSON.parse(readFileSync(path.join(cwd, LEDGER_PATH), 'utf8')),
       validBaselineRows,
@@ -255,20 +255,20 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () =>
     );
     writeCodecFixture(
       cwd,
-      [...reservationRows, ['pr-a', '0'], ['pr-b', '3']],
+      [...reservationRows, ['pr-a', '3'], ['pr-b', '4']],
       [
-        { id: 'pr-a', token: '0' },
-        { id: 'pr-b', token: '3' },
+        { id: 'pr-a', token: '3' },
+        { id: 'pr-b', token: '4' },
       ],
     );
     const corrected = runCheck();
     assert.equal(corrected.status, 0, corrected.stderr);
-    assert.match(corrected.stdout, /29 published, 1 new/);
+    assert.match(corrected.stdout, /30 published, 1 new/);
   });
 });
 
 test('allocation batches cross the last digit and base-36 pair boundaries in order', () => {
-  const digitRows = [...'03456789'].map((digit) => [`retired-${digit}`, digit]);
+  const digitRows = [...'3456789'].map((digit) => [`retired-${digit}`, digit]);
   const allDigits = parseLayerStateTokenReservations([
     ...reservationRows,
     ...digitRows,
@@ -320,7 +320,7 @@ test('allocation batches cross the last digit and base-36 pair boundaries in ord
 test('an isolated valid two-character fixture round-trips an l field beyond the old 64-character cap', async () => {
   const cwd = mkdtempSync(path.join(tmpdir(), 'gev-layer-token-width-'));
   try {
-    const priorDigits = [...'03456789'].map((digit) => [
+    const priorDigits = [...'3456789'].map((digit) => [
       `qa-prior-digit-${digit}`,
       digit,
     ]);
@@ -357,7 +357,7 @@ test('an isolated valid two-character fixture round-trips an l field beyond the
     const params = new URLSearchParams([['v', '2']]);
     codec.encodeLayerStateParams(params, state);
     assert.ok(params.get('l').length > 64, 'fixture must cross the old cap');
-    assert.equal(params.get('l').length, 67);
+    assert.equal(params.get('l').length, 69);
     const restored = codec.decodeLayerStateParams(params);
     assert.deepEqual(restored?.enabledLayerIds, expectedLayerIds);
     assert.deepEqual(restored?.options, state.options);
@@ -375,7 +375,7 @@ test('checker reads a complete future base ledger and rejects retired-token reus
     },
     (cwd) => {
       const published = readPublishedLayerStateReservations('HEAD', cwd);
-      assert.equal(Object.keys(published).length, 29);
+      assert.equal(Object.keys(published).length, 30);
       assert.equal(published['retired-layer'], '00');
       assert.throws(
         () =>

src/data/layerStateTokenReservations.json

@@ -19,6 +19,7 @@
   ["recent-imagery", "1"],
   ["rocket-launches", "x"],
   ["satellites", "s"],
+  ["street-level", "0"],
   ["telegeography-submarine-cables", "u"],
   ["traffic", "t"],
   ["transit", "j"],