Merge branch 'main' into weather/source-names

src/data/dataCredits.js

@@ -700,6 +700,15 @@ export const NATURAL_EARTH_CREDIT = {
     '<a href="https://www.naturalearthdata.com" target="_blank" rel="noopener">Natural Earth</a> (public domain)',
 };
 
+/** Registered when a name from the bundled Wikidata places packs is first
+ * shown (CC0 — no attribution required; credited as a courtesy). */
+export const WIKIDATA_PLACES_CREDIT = {
+  key: 'wikidata-places',
+  html:
+    'City, town and village names from ' +
+    '<a href="https://www.wikidata.org" target="_blank" rel="noopener">Wikidata</a> (CC0)',
+};
+
 /** Registered when the first bundled US county or place outline resolves
  * (public domain — credited as a courtesy). */
 export const US_CENSUS_CREDIT = {

src/data/local_data/wikidata_cities/README.md

@@ -0,0 +1,42 @@
+# Wikidata cities pack
+
+World cities and towns for naming a spot (the nearest city), the places
+around it and finding a place by name, read by `src/data/wikidataCities.js`. The app reads this file; it
+does not call Wikidata or any place service for these.
+
+| File | Contents |
+|------|----------|
+| `cities.json` | Human settlements with coordinates and a population at or above the cut-off (see `meta.minPopulation`), largest first |
+
+Each row is `[name, lat, lon, population, country, wikidata]`: the English
+label, coordinates to three decimals, the largest stated population, the
+ISO 3166-1 alpha-2 country code (empty when Wikidata gives none) and the
+Wikidata id.
+
+**Source:** Wikidata, read once from the Wikidata Query Service by
+`scripts/build-wikidata-cities.mjs`. The build time and row count are in the
+file's `meta` header.
+
+**License:** CC0 1.0 (https://www.wikidata.org/wiki/Wikidata:Licensing). No
+attribution is legally required.
+
+**What is left out:** metropolitan areas, urban agglomerations, conurbations
+and similar groupings of settlements, so a view is named after the city
+rather than the area around it (an item of a narrower grouping kind that is
+also a city stays: Istanbul is a metropolitan municipality and a megacity);
+places that no longer stand (former entities, ghost towns, abandoned
+villages, archaeological sites) unless they are also a living settlement;
+and places Wikidata states as dissolved or abolished (P576) where another
+place within 15 km still names the spot. A dissolved unit with nothing
+else near it stays: many of Vietnam's provincial cities ended as units in
+2025 and still stand under their names. Also left out: places below the population cut-off and
+places without an English label.
+
+**Rebuild:** `node scripts/build-wikidata-cities.mjs [--min-population 15000]`.
+The build runs one query at a time with pauses and honours the service's
+Retry-After; it takes about ten minutes. It fails if a short list of
+well-known cities is missing from the result, or a known grouping gets in.
+
+**Limits:** populations are whatever Wikidata states and differ in date and
+definition between places (city proper, municipality, urban district). Some
+districts and suburbs appear as their own rows.

src/data/local_data/wikidata_cities/cities.json

@@ -0,0 +1 @@
+{"meta":{"source":"Wikidata (https://www.wikidata.org), via the Wikidata Query Service","license":"CC0 1.0","built":"2026-11-15T02:59:26.673Z","minPopulation":15000,"count":31827,"columns":["name","lat","lon","population","country","wikidata"]},"rows":[["Chongqing",29.55,106.507,32054159,"CN","Q11725"],["Delhi",28.667,77.217,26495000,"IN","Q1353"],["Shanghai",31.233,121.469,24870895,"CN","Q8686"],["Beijing",39.904,116.408,21893095,"CN","Q956"],["Chengdu",30.66,104.063,20937757,"CN","Q30002"],["Guangzhou",23.13,113.26,18676605,"CN","Q16572"],["Shenzhen",22.547,114.054,17494398,"CN","Q15174"],["Dhaka",23.729,90.394,16800000,"BD","Q1354"],["Istanbul",41.01,28.96,15655924,"TR","Q406"],["Mumbai",19.076,72.877,15414288,"IN","Q1156"],["Karachi",24.86,67.01,14910352,"PK","Q8660"],["Kinshasa",-4.322,15.312,14565700,"CD","Q3838"],["Tokyo",35.689,139.692,14264798,"JP","Q1490"],["Ho Chi Minh City",10.776,106.702,14002598,"VN","Q1854"],["Tianjin",39.147,117.206,13866009,"CN","Q11736"],["Moscow",55.751,37.617,13274285,"RU","Q649"],["Xi'an",34.261,108.942,12952907,"CN","Q5826"],["Suzhou",31.3,120.619,12748262,"CN","Q42622"],["Zhengzhou",34.749,113.661,12600574,"CN","Q30340"],["Bengaluru",12.979,77.591,12327000,"IN","Q1355"],["Wuhan",30.595,114.297,12326518,"CN","Q11746"],["Hangzhou",30.25,120.168,11936010,"CN","Q4970"],["Baoding",38.867,115.485,11544036,"CN","Q58584"],["São Paulo",-23.55,-46.634,11451999,"BR","Q174"],["Jakarta",-6.175,106.827,11135191,"ID","Q3630"],["Lahore",31.55,74.344,11126285,"PK","Q11739"],["Linyi",35.104,118.35,11018365,"CN","Q373346"],["Shijiazhuang",38.042,114.51,10640458,"CN","Q58401"],["Dongguan",23.047,113.749,10466625,"CN","Q59218"],["Qingdao",36.117,120.4,10071722,"CN","Q170322"],["Changsha",28.199,112.971,10047914,"CN","Q174091"],["Harbin",45.75,126.633,10009854,"CN","Q42956"],["Lima",-12.06,-77.037,9943800,"PE","Q2868"],["Cairo",30.044,31.236,9801536,"EG","Q85"],["Nanyang",32.999,112.529,9713112,"CN","Q404763"],["Seoul",37.56,126.99,9668465,"KR","Q8684"],["Tokyo",35.684,139.774,9640742,"JP","Q7473516"],["Wenzhou",27.999,120.656,9572903,"CN","Q42635"],["Foshan",23.029,113.106,9498863,"CN","Q34412"],["Handan",36.612,114.489,9413990,"CN","Q58570"],["Ningbo",29.875,121.549,9404283,"CN","Q42780"],["Weifang",36.717,119.1,9386705,"CN","Q217698"],["Hefei",31.864,117.281,9369881,"CN","Q185684"],["Nanjing",32.061,118.779,9314685,"CN","Q16666"],["Mexico City",19.354,-99.136,9209944,"MX","Q1489"],["Jinan",36.667,116.983,9202432,"CN","Q170247"],["Hyderabad",17.362,78.475,9191000,"IN","Q1361"],["Xuzhou",34.261,117.186,9083790,"CN","Q57719"],["Shenyang",41.803,123.428,9070093,"CN","Q11720"],["Changchun",43.88,125.323,9066906,"CN","Q92161"],["Zhoukou",33.625,114.642,9026015,"CN","Q198061"],["Kuala Lumpur",3.148,101.695,9000280,"MY","Q1865"],["Ganzhou",25.829,114.934,8970014,"CN","Q363166"],["New York City",40.713,-74.006,8804190,"US","Q60"],["London",51.507,-0.128,8799728,"GB","Q84"],["Heze",35.233,115.433,8795939,"CN","Q332753"],["Quanzhou",24.914,118.586,8782285,"CN","Q68695"],["Nanning",22.819,108.315,8741584,"CN","Q179608"],["Hanoi",21.024,105.841,8717600,"VN","Q1858"],["Tehran",35.689,51.39,8693706,"IR","Q3616"],["Kunming",25.043,102.706,8460088,"CN","Q182852"],["Jining",35.4,116.567,8357897,"CN","Q372791"],["Fuzhou",26.077,119.292,8291268,"CN","Q68481"],["Fuyang",32.899,115.804,8200264,"CN","Q360584"],["Baghdad",33.315,44.366,8126755,"IQ","Q1530"],["Veracruz",19.19,-96.153,8062579,"MX","Q173270"],["Bogotá",4.61,-74.082,8034649,"CO","Q2841"],["Shangqiu",34.426,115.647,7816831,"CN","Q404817"],["Nantong",31.983,120.887,7726635,"CN","Q57947"],["Tangshan",39.629,118.174,7717983,"CN","Q58422"],["Ahmedabad",23.022,72.58,7645000,"IN","Q1070"],["Wuxi",31.567,120.283,7462135,"CN","Q42651"],["Dalian",38.9,121.6,7450785,"CN","Q74881"],["Hong Kong",22.278,114.159,7413070,"CN","Q8646"],["Cangzhou",38.304,116.845,7300783,"CN","Q58695"],["Xingtai",37.066,114.475,7111106,"CN","Q58576"],["Yantai",37.464,121.448,7102116,"CN","Q210493"],["Luoyang",34.659,112.424,7056699,"CN","Q187136"],["Jinhua",29.105,119.649,7050683,"CN","Q58210"],["Riyadh",24.638,46.715,7009100,"SA","Q3692"],["Zhumadian",32.977,114.025,7008427,"CN","Q198240"],["Zhanjiang",21.197,110.403,6981236,"CN","Q59144"],["Bijie",27.302,105.286,6899636,"CN","Q486319"],["Yangon",16.795,96.16,6874000,"MM","Q37995"],["Yancheng",33.35,120.157,6709629,"CN","Q57866"],["Hengyang",26.897,112.586,6645243,"CN","Q144663"],["Taizhou",28.658,121.422,6622888,"CN","Q58273"],["Zunyi",27.705,106.934,6606675,"CN","Q28994"],["Chennai",13.082,80.275,6599000,"IN","Q1352"],["Shaoyang",27.242,111.472,6563520,"CN","Q194531"],["Shangrao",28.442,117.963,6491088,"CN","Q363479"],["Huanggang",30.45,114.875,6333000,"CN","Q71355"],["Moro",-6,144,6310129,"PG","Q103927888"],["Santiago",-33.438,-70.65,6257516,"CL","Q2887"],["Nanchang",28.684,115.887,6255007,"CN","Q171943"],["Xinxiang",35.299,113.885,6251929,"CN","Q404362"],["Xinyang",32.126,114.067,6234401,"CN","Q404873"],["Rio de Janeiro",-2Line truncated

src/data/wikidataCities.js

@@ -0,0 +1,244 @@
+/**
+ * The bundled world cities pack (Wikidata, CC0), read from
+ * `local_data/wikidata_cities/cities.json` (built by
+ * scripts/build-wikidata-cities.mjs). The file loads once, on first use, and
+ * every answer comes from it: no place service is called.
+ *
+ * Rows are `[name, lat, lon, population, country, wikidata]`, largest first.
+ */
+
+const CELL_DEG = 1;
+
+// Letters that are not a base letter plus an accent, so NFD leaves them.
+const PLAIN_LETTERS = {
+  ł: 'l',
+  ø: 'o',
+  đ: 'd',
+  ð: 'd',
+  ı: 'i',
+  ß: 'ss',
+  æ: 'ae',
+  œ: 'oe',
+  þ: 'th',
+};
+
+/**
+ * A name as the lookups compare it: lower case, without accents, so
+ * "sao paulo" finds São Paulo, "zurich" Zürich, "lodz" Łódź and "tromso"
+ * Tromsø.
+ */
+export const foldName = (name) =>
+  String(name || '')
+    .normalize('NFD')
+    .replace(/\p{M}/gu, '')
+    .toLocaleLowerCase('en')
+    .replace(/[łøđðıßæœþ]/g, (letter) => PLAIN_LETTERS[letter])
+    .trim();
+const EARTH_KM = 6371;
+const rad = (deg) => (deg * Math.PI) / 180;
+
+function distanceKm(lat1, lon1, lat2, lon2) {
+  const a =
+    Math.sin(rad(lat2 - lat1) / 2) ** 2 +
+    Math.cos(rad(lat1)) *
+      Math.cos(rad(lat2)) *
+      Math.sin(rad(lon2 - lon1) / 2) ** 2;
+  return 2 * EARTH_KM * Math.asin(Math.min(1, Math.sqrt(a)));
+}
+
+const cellKey = (lat, lon) =>
+  `${Math.floor(lat / CELL_DEG)},${Math.floor(((((lon % 360) + 540) % 360) - 180) / CELL_DEG)}`;
+
+/** Group rows by one-degree cell so a lookup reads only the cells near it. */
+export function indexCities(rows) {
+  const cells = new Map();
+  for (const row of rows) {
+    const key = cellKey(row[1], row[2]);
+    const cell = cells.get(key);
+    if (cell) cell.push(row);
+    else cells.set(key, [row]);
+  }
+  return cells;
+}
+
+/** Rows by folded name, each list in the rows' own order (largest first). */
+export function indexNames(rows) {
+  const names = new Map();
+  for (const row of rows) {
+    const key = foldName(row[0]);
+    const list = names.get(key);
+    if (list) list.push(row);
+    else names.set(key, [row]);
+  }
+  return names;
+}
+
+function toCity(row, km) {
+  return {
+    name: row[0],
+    latitude: row[1],
+    longitude: row[2],
+    population: row[3],
+    country: row[4] || null,
+    wikidata: row[5],
+    distanceKm: km,
+  };
+}
+
+/**
+ * Half the longitude a circle spans, in degrees: 180 when it reaches a pole,
+ * where every meridian passes through it.
+ */
+export function lonHalfSpan(latitude, radiusKm) {
+  const angle = radiusKm / EARTH_KM;
+  const fromPole = Math.PI / 2 - Math.abs(rad(latitude));
+  if (angle >= fromPole) return 180;
+  return (Math.asin(Math.sin(angle) / Math.cos(rad(latitude))) * 180) / Math.PI;
+}
+const latHalfSpan = (radiusKm) => ((radiusKm / EARTH_KM) * 180) / Math.PI;
+
+/** Rows within `radiusKm` of a point, each with its distance. */
+export function citiesWithin(cells, latitude, longitude, radiusKm) {
+  const latSpan = Math.ceil(latHalfSpan(radiusKm) / CELL_DEG);
+  const lonSpan = Math.ceil(lonHalfSpan(latitude, radiusKm) / CELL_DEG);
+  const found = [];
+  const seen = new Set();
+  for (let dLat = -latSpan; dLat <= latSpan; dLat++) {
+    const lat = latitude + dLat * CELL_DEG;
+    if (lat < -90 - CELL_DEG || lat > 90 + CELL_DEG) continue;
+    for (
+      let dLon = -Math.min(lonSpan, 180);
+      dLon <= Math.min(lonSpan, 180);
+      dLon++
+    ) {
+      const key = cellKey(lat, longitude + dLon * CELL_DEG);
+      if (seen.has(key)) continue;
+      seen.add(key);
+      for (const row of cells.get(key) || []) {
+        const km = distanceKm(latitude, longitude, row[1], row[2]);
+        if (km <= radiusKm) found.push(toCity(row, km));
+      }
+    }
+  }
+  return found;
+}
+
+// Each country in the pack by what a qualifier may call it: its code, its
+// English name and its name in the page language, folded.
+function countryNames(rows) {
+  const names = new Map();
+  for (const code of new Set(rows.map((row) => row[4]).filter(Boolean))) {
+    for (const name of [code, countryName(code, 'en'), countryName(code)])
+      names.set(foldName(name), code);
+  }
+  return names;
+}
+
+let loading = null;
+/** The indexed pack; a failed load is retried on the next call. */
+export function loadCities(fetchImpl = (...args) => globalThis.fetch(...args)) {
+  loading ??= fetchImpl(
+    new URL('./local_data/wikidata_cities/cities.json', import.meta.url),
+  )
+    .then((response) => {
+      if (!response.ok) throw new Error(`cities pack: HTTP ${response.status}`);
+      return response.json();
+    })
+    .then((pack) => ({
+      meta: pack.meta,
+      rows: pack.rows,
+      cells: indexCities(pack.rows),
+      names: indexNames(pack.rows),
+      countries: countryNames(pack.rows),
+    }))
+    .catch((error) => {
+      loading = null;
+      throw error;
+    });
+  return loading;
+}
+
+/**
+ * The city a point is in or nearest to: the largest within `nearKm`, else the
+ * nearest within `radiusKm`, else null. Preferring the largest nearby keeps a
+ * view over a metropolis from being named after one of its suburbs.
+ */
+export async function nearestCity(
+  latitude,
+  longitude,
+  { radiusKm = 60, nearKm = 12 } = {},
+) {
+  if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) return null;
+  const { cells } = await loadCities();
+  const found = citiesWithin(cells, latitude, longitude, radiusKm);
+  if (!found.length) return null;
+  const near = found.filter((city) => city.distanceKm <= nearKm);
+  if (near.length)
+    return near.reduce((a, b) => (b.population > a.population ? b : a));
+  return found.reduce((a, b) => (b.distanceKm < a.distanceKm ? b : a));
+}
+
+/** Cities around a point, most populous first. */
+export async function citiesAround({
+  latitude,
+  longitude,
+  radiusKm = 50,
+  limit = 8,
+  minPopulation = 0,
+}) {
+  if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) return [];
+  const { cells } = await loadCities();
+  return citiesWithin(cells, latitude, longitude, radiusKm)
+    .filter((city) => city.population >= minPopulation)
+    .sort((a, b) => b.population - a.population || a.distanceKm - b.distanceKm)
+    .slice(0, limit);
+}
+
+/** A country's name in the page language, from its ISO code. */
+export function countryName(
+  code,
+  language = globalThis.document?.documentElement?.lang || 'en',
+) {
+  if (!code) return null;
+  try {
+    return (
+      new Intl.DisplayNames([language], { type: 'region' }).of(code) || code
+    );
+  } catch {
+    return code;
+  }
+}
+
+/**
+ * The city with this name, or null. A qualifier after a comma that names a
+ * country ("Paris, France", "Paris, US") keeps only cities there, and finds
+ * nothing when none is; one that names no country ("Paris, Texas") is
+ * ignored. A city of that name within `radiusKm` of `near` (the view) wins,
+ * the nearest first, else the largest city of that name anywhere.
+ */
+export async function findPlace(
+  name,
+  { near = null, radiusKm = 150 } = {},
+  fetchImpl = (...args) => globalThis.fetch(...args),
+) {
+  const [head, ...rest] = String(name || '').split(',');
+  const wanted = foldName(head);
+  if (!wanted) return null;
+  const qualifier = foldName(rest.join(','));
+  const { names, countries } = await loadCities(fetchImpl);
+  const country = qualifier ? countries.get(qualifier) : undefined;
+  const cities = (names.get(wanted) || []).filter(
+    (row) => !country || row[4] === country,
+  );
+  if (!cities.length) return null;
+  if (Number.isFinite(near?.latitude) && Number.isFinite(near?.longitude)) {
+    let best = null;
+    for (const row of cities) {
+      const km = distanceKm(near.latitude, near.longitude, row[1], row[2]);
+      if (km <= radiusKm && (!best || km < best.distanceKm))
+        best = toCity(row, km);
+    }
+    if (best) return best;
+  }
+  return toCity(cities[0], 0);
+}

src/data/wikidataCities.test.mjs

@@ -0,0 +1,121 @@
+// src/data/wikidataCities.test.mjs — lookups over the bundled Wikidata
+// places packs, with the packs served from memory.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  citiesWithin,
+  foldName,
+  indexCities,
+  lonHalfSpan,
+} from './wikidataCities.js';
+
+const ROWS = [
+  ['São Paulo', -23.55, -46.633, 11451999, 'BR', 'Q174'],
+  ['Austin', 30.267, -97.743, 961855, 'US', 'Q16559'],
+  ['Round Rock', 30.508, -97.679, 119468, 'US', 'Q128269'],
+  ['Zürich', 47.374, 8.541, 421878, 'CH', 'Q72'],
+];
+
+test('names compare without case or accents', () => {
+  assert.equal(foldName('São Paulo'), 'sao paulo');
+  assert.equal(foldName('  ZÜRICH '), 'zurich');
+  assert.equal(foldName(null), '');
+});
+
+test('a lookup reads the cells around a point and measures each place', () => {
+  const cells = indexCities(ROWS);
+  const near = citiesWithin(cells, 30.27, -97.74, 40).sort(
+    (a, b) => a.distanceKm - b.distanceKm,
+  );
+  assert.deepEqual(
+    near.map((city) => city.name),
+    ['Austin', 'Round Rock'],
+  );
+  assert.ok(near[0].distanceKm < 1);
+  assert.equal(near[1].country, 'US');
+  assert.deepEqual(citiesWithin(cells, 0, 0, 50), []);
+});
+
+test('findPlace finds a city by its name without accents', async () => {
+  // A fresh module, so its pack cache starts empty.
+  const { findPlace } = await import(`./wikidataCities.js?case=${Date.now()}`);
+  const fetchImpl = async (url) => ({
+    ok: true,
+    json: async () =>
+      String(url).includes('cities.json') ? { meta: {}, rows: ROWS } : null,
+  });
+  const found = await findPlace('sao paulo, Brazil', {}, fetchImpl);
+  assert.equal(found?.wikidata, 'Q174');
+  assert.equal(await findPlace('Atlantis', {}, fetchImpl), null);
+});
+
+test('names fold letters that carry no separate accent', () => {
+  assert.equal(foldName('Łódź'), 'lodz');
+  assert.equal(foldName('Tromsø'), 'tromso');
+  assert.equal(foldName('Đà Nẵng'), 'da nang');
+  assert.equal(foldName('Diyarbakır'), 'diyarbakir');
+  assert.equal(foldName('Straße'), 'strasse');
+});
+
+const PARIS = [
+  ['Paris', 48.857, 2.352, 2145906, 'FR', 'Q90'],
+  ['Paris', 33.661, -95.556, 24476, 'US', 'Q830149'],
+];
+
+async function freshPack(rows) {
+  const module = await import(
+    `./wikidataCities.js?case=${Date.now()}-${Math.random()}`
+  );
+  const fetchImpl = async () => ({
+    ok: true,
+    json: async () => ({ meta: {}, rows }),
+  });
+  return { ...module, fetchImpl };
+}
+
+test('findPlace takes a country qualifier and prefers a namesake in view', async () => {
+  const { findPlace, fetchImpl } = await freshPack(PARIS);
+  const texas = { latitude: 33.2, longitude: -96.0 };
+  assert.equal((await findPlace('Paris', {}, fetchImpl)).wikidata, 'Q90');
+  assert.equal(
+    (await findPlace('Paris', { near: texas }, fetchImpl)).wikidata,
+    'Q830149',
+  );
+  assert.equal(
+    (await findPlace('Paris, United States', {}, fetchImpl)).wikidata,
+    'Q830149',
+  );
+  assert.equal(
+    (await findPlace('paris, us', {}, fetchImpl)).wikidata,
+    'Q830149',
+  );
+  // A qualifier the packs cannot place is ignored.
+  assert.equal(
+    (await findPlace('Paris, Texas', { near: texas }, fetchImpl)).wikidata,
+    'Q830149',
+  );
+});
+
+test('a circle that reaches a pole spans every longitude', () => {
+  assert.equal(lonHalfSpan(89, 150), 180);
+  assert.ok(lonHalfSpan(60, 150) < 3);
+  const cells = indexCities([['Pole camp', 89.9, 179, 1, '', 'Q1']]);
+  assert.equal(citiesWithin(cells, 89, 0, 150).length, 1);
+});
+
+test('a country qualifier that matches no namesake finds nothing', async () => {
+  const { findPlace, fetchImpl } = await freshPack([
+    ...PARIS,
+    ['Berlin', 52.52, 13.405, 3755251, 'DE', 'Q64'],
+  ]);
+  assert.equal(await findPlace('Paris, Germany', {}, fetchImpl), null);
+  assert.equal(
+    (await findPlace('Paris, France', {}, fetchImpl)).wikidata,
+    'Q90',
+  );
+});
+
+test('a lookup wraps at the antimeridian', () => {
+  const cells = indexCities([['Taveuni', -16.8, -179.9, 15000, 'FJ', 'Q2']]);
+  assert.equal(citiesWithin(cells, -16.8, 179.9, 40).length, 1);
+});