mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
Merge branch 'main' into prereq-10-osm-overlays
This commit is contained in:
424 files changed
+25311
-956
No files matched your search
@@ -111,8 +111,8 @@ const cacheDir = path.resolve(
|
||||
);
|
||||
const only = args.includes('--only') ? args[args.indexOf('--only') + 1] : null;
|
||||
|
||||
async function fetchPinned(name) {
|
||||
const { url, sha256 } = SOURCES[name];
|
||||
export async function fetchPinned(name, source = SOURCES[name]) {
|
||||
const { url, sha256 } = source;
|
||||
const file = path.join(cacheDir, path.basename(new URL(url).pathname));
|
||||
let bytes = await readFile(file).catch(() => null);
|
||||
if (!bytes) {
|
||||
@@ -137,7 +137,7 @@ const EARTH_RADIUS_KM = 6371;
|
||||
const toRad = (d) => (d * Math.PI) / 180;
|
||||
|
||||
/** Spherical ring area (km²); matches src/data/naturalEarthRegions.js. */
|
||||
function ringAreaKm2(ring) {
|
||||
export function ringAreaKm2(ring) {
|
||||
const n = ring.length;
|
||||
if (n < 3) return 0;
|
||||
let sum = 0;
|
||||
@@ -151,7 +151,7 @@ function ringAreaKm2(ring) {
|
||||
}
|
||||
|
||||
/** Signed planar (shoelace) area: positive counter-clockwise in lon/lat. */
|
||||
function signedArea(ring) {
|
||||
export function signedArea(ring) {
|
||||
let sum = 0;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++)
|
||||
sum += ring[j][0] * ring[i][1] - ring[i][0] * ring[j][1];
|
||||
@@ -209,15 +209,16 @@ function douglasPeucker(points, tolerance) {
|
||||
* Simplify a closed ring to an OPEN rounded ring. The ring is split at its
|
||||
* farthest vertex from the start so Douglas-Peucker has two anchored halves
|
||||
* (a closed ring has no chord to measure against). Returns null when fewer
|
||||
* than four distinct vertices survive.
|
||||
* than `minVertices` distinct vertices survive (four by default; a pack that
|
||||
* keeps triangles passes three).
|
||||
*/
|
||||
function simplifyRing(ring, tolerance, decimals) {
|
||||
function simplifyRing(ring, tolerance, decimals, minVertices = 4) {
|
||||
let open = ring.slice();
|
||||
const first = open[0];
|
||||
const last = open[open.length - 1];
|
||||
if (open.length > 1 && first[0] === last[0] && first[1] === last[1])
|
||||
open = open.slice(0, -1);
|
||||
if (open.length < 4) return null;
|
||||
if (open.length < minVertices) return null;
|
||||
let far = 1;
|
||||
let farDist = -1;
|
||||
for (let i = 1; i < open.length; i++) {
|
||||
@@ -247,7 +248,7 @@ function simplifyRing(ring, tolerance, decimals) {
|
||||
out[0][1] === out[out.length - 1][1]
|
||||
)
|
||||
out.pop();
|
||||
return out.length >= 4 ? out : null;
|
||||
return out.length >= minVertices ? out : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -255,16 +256,17 @@ function simplifyRing(ring, tolerance, decimals) {
|
||||
* area floors (the largest part always stays), orient outers
|
||||
* counter-clockwise and holes clockwise, sort parts by area descending.
|
||||
*/
|
||||
function simplifyPolygons(polygons, params, tolerance, decimals) {
|
||||
export function simplifyPolygons(polygons, params, tolerance, decimals) {
|
||||
const minVertices = params.minRingVertices ?? 4;
|
||||
const parts = [];
|
||||
for (const poly of polygons) {
|
||||
const outer = simplifyRing(poly[0], tolerance, decimals);
|
||||
const outer = simplifyRing(poly[0], tolerance, decimals, minVertices);
|
||||
if (!outer) continue;
|
||||
const area = ringAreaKm2(outer);
|
||||
if (signedArea(outer) < 0) outer.reverse();
|
||||
const holes = [];
|
||||
for (const hole of poly.slice(1)) {
|
||||
const ring = simplifyRing(hole, tolerance, decimals);
|
||||
const ring = simplifyRing(hole, tolerance, decimals, minVertices);
|
||||
if (!ring || ringAreaKm2(ring) < params.minHoleKm2) continue;
|
||||
if (signedArea(ring) > 0) ring.reverse();
|
||||
holes.push(ring);
|
||||
@@ -287,18 +289,22 @@ function simplifyPolygons(polygons, params, tolerance, decimals) {
|
||||
* and a fifth of the minimum tolerance, or failing that unsimplified at two
|
||||
* more decimals; the caller records `d`.
|
||||
*/
|
||||
function simplifyUnit(polygons, params) {
|
||||
export function unitTolerance(polygons, params) {
|
||||
const sourceArea = polygons.reduce(
|
||||
(sum, poly) => sum + ringAreaKm2(poly[0]),
|
||||
0,
|
||||
);
|
||||
const tolerance = Math.min(
|
||||
return Math.min(
|
||||
params.maxToleranceDeg,
|
||||
Math.max(
|
||||
params.minToleranceDeg,
|
||||
(params.toleranceFactor * Math.sqrt(sourceArea)) / 111.32,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function simplifyUnit(polygons, params) {
|
||||
const tolerance = unitTolerance(polygons, params);
|
||||
const coarse = simplifyPolygons(polygons, params, tolerance, params.decimals);
|
||||
if (coarse.length) return { polygons: coarse, decimals: params.decimals };
|
||||
const fine = simplifyPolygons(
|
||||
@@ -334,7 +340,7 @@ function encodeRing(ring, decimals) {
|
||||
return out;
|
||||
}
|
||||
|
||||
function encodeUnit({ polygons, decimals }) {
|
||||
export function encodeUnit({ polygons, decimals }) {
|
||||
return polygons.map((poly) => poly.map((ring) => encodeRing(ring, decimals)));
|
||||
}
|
||||
|
||||
@@ -362,7 +368,7 @@ function bboxOf(polygons) {
|
||||
// ── zip / shapefile / dbf ───────────────────────────────────────────────
|
||||
|
||||
/** Read named entries from a zip archive (stored or deflated). */
|
||||
function unzip(buffer) {
|
||||
export function unzip(buffer) {
|
||||
let eocd = buffer.length - 22;
|
||||
while (eocd >= 0 && buffer.readUInt32LE(eocd) !== 0x06054b50) eocd--;
|
||||
if (eocd < 0) throw new Error('zip: end of central directory not found');
|
||||
@@ -394,7 +400,7 @@ function unzip(buffer) {
|
||||
}
|
||||
|
||||
/** Polygon records (shape type 5) as arrays of closed [lon, lat] rings. */
|
||||
function readShp(buffer) {
|
||||
export function readShp(buffer) {
|
||||
const shapes = [];
|
||||
let offset = 100;
|
||||
while (offset < buffer.length) {
|
||||
@@ -431,7 +437,7 @@ function readShp(buffer) {
|
||||
}
|
||||
|
||||
/** dBASE records as plain objects (character and numeric fields). */
|
||||
function readDbf(buffer) {
|
||||
export function readDbf(buffer) {
|
||||
const count = buffer.readUInt32LE(4);
|
||||
const headerLength = buffer.readUInt16LE(8);
|
||||
const recordLength = buffer.readUInt16LE(10);
|
||||
@@ -458,7 +464,7 @@ function readDbf(buffer) {
|
||||
}
|
||||
|
||||
/** Group shapefile rings into polygons: clockwise outers, holes assigned by containment. */
|
||||
function shpPolygons(rings) {
|
||||
export function shpPolygons(rings) {
|
||||
const outers = [];
|
||||
const holes = [];
|
||||
for (const ring of rings) {
|
||||
@@ -475,7 +481,7 @@ function shpPolygons(rings) {
|
||||
return polygons;
|
||||
}
|
||||
|
||||
function pointInRing(ring, x, y) {
|
||||
export function pointInRing(ring, x, y) {
|
||||
let inside = false;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++) {
|
||||
const [xi, yi] = ring[i];
|
||||
@@ -742,18 +748,24 @@ async function writePack(relative, pack) {
|
||||
);
|
||||
}
|
||||
|
||||
if (!only || only === 'countries')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/countries.json',
|
||||
await buildCountries(),
|
||||
);
|
||||
if (!only || only === 'admin1')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/states_provinces.json',
|
||||
await buildAdmin1(),
|
||||
);
|
||||
if (!only || only === 'counties')
|
||||
await writePack(
|
||||
'src/data/local_data/us_census_counties/counties.json',
|
||||
await buildCounties(),
|
||||
);
|
||||
const isMain =
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
|
||||
if (isMain) {
|
||||
if (!only || only === 'countries')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/countries.json',
|
||||
await buildCountries(),
|
||||
);
|
||||
if (!only || only === 'admin1')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/states_provinces.json',
|
||||
await buildAdmin1(),
|
||||
);
|
||||
if (!only || only === 'counties')
|
||||
await writePack(
|
||||
'src/data/local_data/us_census_counties/counties.json',
|
||||
await buildCounties(),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the bundled US places pack read by `src/data/placeBoundaries.js`:
|
||||
*
|
||||
* src/data/local_data/us_census_places/index.json state list, boxes, meta
|
||||
* src/data/local_data/us_census_places/<ST>.json one file per state
|
||||
* src/data/local_data/us_census_places/files.js their URLs, for bundlers
|
||||
*
|
||||
* Source: US Census Bureau 2025 cartographic boundary places
|
||||
* (cb_2025_us_place_500k, public domain), pinned by URL and SHA-256, so a
|
||||
* rerun reproduces the pack byte for byte. The download is cached outside the
|
||||
* repository (default: the system temp directory).
|
||||
*
|
||||
* Geometry uses the county pack's simplification and encoding
|
||||
* (scripts/build-admin-packs.mjs) with two differences: triangles are kept,
|
||||
* and a unit whose simplified rings cross themselves or each other is
|
||||
* simplified again at half the tolerance (finer precision last). Then
|
||||
* `scripts/pack_geometry.py repair census` checks every feature as a whole
|
||||
* MultiPolygon under GEOS and repairs the ones that are still invalid (rings
|
||||
* touching themselves, overlapping parts), so it needs Python with
|
||||
* shapely==2.1.2 on GEOS 3.13.1.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/build-census-places.mjs [--cache <dir>] [--python <exe>]
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
PARAMS as ADMIN_PARAMS,
|
||||
encodeUnit,
|
||||
fetchPinned,
|
||||
readDbf,
|
||||
readShp,
|
||||
shpPolygons,
|
||||
simplifyPolygons,
|
||||
unitTolerance,
|
||||
unzip,
|
||||
} from './build-admin-packs.mjs';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const OUT = path.join(ROOT, 'src/data/local_data/us_census_places');
|
||||
|
||||
export const SOURCE = Object.freeze({
|
||||
url: 'https://www2.census.gov/geo/tiger/GENZ2025/shp/cb_2025_us_place_500k.zip',
|
||||
sha256: 'ce0e4019ecd4123d03d53aaa936eed0459b82e3e14b89a3dcd4d5e8b3308627d',
|
||||
});
|
||||
|
||||
/** County-pack parameters; triangles are valid rings here. */
|
||||
export const PARAMS = Object.freeze({
|
||||
...ADMIN_PARAMS.counties,
|
||||
minRingVertices: 3,
|
||||
maxRepairSteps: 8,
|
||||
});
|
||||
|
||||
/** Census LSAD codes for census designated places (unincorporated). */
|
||||
const CDP_LSAD = new Set(['55', '57', '62']);
|
||||
|
||||
// ── validity ─────────────────────────────────────────────────────────────
|
||||
|
||||
function orient(ax, ay, bx, by, cx, cy) {
|
||||
const v = (bx - ax) * (cy - ay) - (by - ay) * (cx - ax);
|
||||
return v > 0 ? 1 : v < 0 ? -1 : 0;
|
||||
}
|
||||
|
||||
function onSegment(ax, ay, bx, by, px, py) {
|
||||
return (
|
||||
Math.min(ax, bx) <= px &&
|
||||
px <= Math.max(ax, bx) &&
|
||||
Math.min(ay, by) <= py &&
|
||||
py <= Math.max(ay, by)
|
||||
);
|
||||
}
|
||||
|
||||
/** Whether segments ab and cd share any point (touching counts). */
|
||||
export function segmentsIntersect(a, b, c, d) {
|
||||
const o1 = orient(a[0], a[1], b[0], b[1], c[0], c[1]);
|
||||
const o2 = orient(a[0], a[1], b[0], b[1], d[0], d[1]);
|
||||
const o3 = orient(c[0], c[1], d[0], d[1], a[0], a[1]);
|
||||
const o4 = orient(c[0], c[1], d[0], d[1], b[0], b[1]);
|
||||
if (o1 !== o2 && o3 !== o4) return true;
|
||||
if (o1 === 0 && onSegment(a[0], a[1], b[0], b[1], c[0], c[1])) return true;
|
||||
if (o2 === 0 && onSegment(a[0], a[1], b[0], b[1], d[0], d[1])) return true;
|
||||
if (o3 === 0 && onSegment(c[0], c[1], d[0], d[1], a[0], a[1])) return true;
|
||||
if (o4 === 0 && onSegment(c[0], c[1], d[0], d[1], b[0], b[1])) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether segments of two different rings cross or overlap. Rings may touch
|
||||
* at a point (a hole meeting its outer ring at a vertex is valid).
|
||||
*/
|
||||
function segmentsCross(s, t) {
|
||||
const [a, b, c, d] = [s.a, s.b, t.a, t.b];
|
||||
const o1 = orient(a[0], a[1], b[0], b[1], c[0], c[1]);
|
||||
const o2 = orient(a[0], a[1], b[0], b[1], d[0], d[1]);
|
||||
const o3 = orient(c[0], c[1], d[0], d[1], a[0], a[1]);
|
||||
const o4 = orient(c[0], c[1], d[0], d[1], b[0], b[1]);
|
||||
if (o1 && o2 && o3 && o4) return o1 !== o2 && o3 !== o4;
|
||||
if (o1 === 0 && o2 === 0) {
|
||||
// Collinear: overlapping by more than a point.
|
||||
const key = a[0] !== b[0] ? 0 : 1;
|
||||
const lo = Math.max(Math.min(a[key], b[key]), Math.min(c[key], d[key]));
|
||||
const hi = Math.min(Math.max(a[key], b[key]), Math.max(c[key], d[key]));
|
||||
return hi > lo;
|
||||
}
|
||||
// One endpoint on the other segment: a crossing only when the rings pass
|
||||
// through each other there; a touch at a point is allowed.
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the rings of one part ([outer, ...holes], open, integer
|
||||
* coordinates) are simple: no ring crosses or touches itself away from its
|
||||
* shared vertices, no two rings cross or overlap (they may touch at a point),
|
||||
* and no ring is flat.
|
||||
*/
|
||||
export function partIsValid(rings) {
|
||||
const segments = [];
|
||||
for (let r = 0; r < rings.length; r++) {
|
||||
const ring = rings[r];
|
||||
if (ring.length < 3) return false;
|
||||
let twice = 0;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++)
|
||||
twice += ring[j][0] * ring[i][1] - ring[i][0] * ring[j][1];
|
||||
if (twice === 0) return false;
|
||||
for (let i = 0; i < ring.length; i++) {
|
||||
const a = ring[i];
|
||||
const b = ring[(i + 1) % ring.length];
|
||||
segments.push({
|
||||
r,
|
||||
i,
|
||||
n: ring.length,
|
||||
a,
|
||||
b,
|
||||
minX: Math.min(a[0], b[0]),
|
||||
maxX: Math.max(a[0], b[0]),
|
||||
});
|
||||
}
|
||||
}
|
||||
segments.sort((s, t) => s.minX - t.minX);
|
||||
for (let x = 0; x < segments.length; x++) {
|
||||
const s = segments[x];
|
||||
for (let y = x + 1; y < segments.length; y++) {
|
||||
const t = segments[y];
|
||||
if (t.minX > s.maxX) break;
|
||||
if (s.r === t.r) {
|
||||
const gap = Math.abs(s.i - t.i);
|
||||
// Neighbours share exactly one vertex; that is not a crossing — unless
|
||||
// they fold back over each other.
|
||||
if (gap === 1 || gap === s.n - 1) {
|
||||
if (s.n > 3 && collinearOverlap(s, t)) return false;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (segmentsCross(s, t)) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function collinearOverlap(s, t) {
|
||||
const shared =
|
||||
s.a === t.a || s.a === t.b ? s.a : s.b === t.a || s.b === t.b ? s.b : null;
|
||||
if (!shared) return false;
|
||||
const p = s.a === shared ? s.b : s.a;
|
||||
const q = t.a === shared ? t.b : t.a;
|
||||
if (orient(shared[0], shared[1], p[0], p[1], q[0], q[1]) !== 0) return false;
|
||||
// Collinear and pointing the same way from the shared vertex: a spike.
|
||||
return (
|
||||
(p[0] - shared[0]) * (q[0] - shared[0]) +
|
||||
(p[1] - shared[1]) * (q[1] - shared[1]) >
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
function toIntegers(polygons, decimals) {
|
||||
const f = 10 ** decimals;
|
||||
return polygons.map((poly) =>
|
||||
poly.map((ring) =>
|
||||
ring.map(([lon, lat]) => [Math.round(lon * f), Math.round(lat * f)]),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function polygonsValid(polygons, decimals) {
|
||||
return toIntegers(polygons, decimals).every(partIsValid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify a place like `simplifyUnit`, then halve the tolerance until every
|
||||
* part is valid; then one more decimal; then the source rings at two more.
|
||||
* @returns {{polygons: Array, decimals: number, repaired: boolean, valid: boolean}}
|
||||
*/
|
||||
export function simplifyPlace(source, params = PARAMS) {
|
||||
let tolerance = unitTolerance(source, params);
|
||||
const attempts = [];
|
||||
for (let step = 0; step <= params.maxRepairSteps; step++) {
|
||||
attempts.push([tolerance, params.decimals]);
|
||||
tolerance /= 2;
|
||||
}
|
||||
attempts.push([params.minToleranceDeg / 5, params.decimals + 1]);
|
||||
attempts.push([0, params.decimals + 1]);
|
||||
attempts.push([0, params.decimals + 2]);
|
||||
let first = null;
|
||||
for (let i = 0; i < attempts.length; i++) {
|
||||
const [tol, decimals] = attempts[i];
|
||||
const polygons = simplifyPolygons(source, params, tol, decimals);
|
||||
if (!polygons.length) continue;
|
||||
first ||= { polygons, decimals };
|
||||
if (polygonsValid(polygons, decimals))
|
||||
return { polygons, decimals, repaired: i > 0, valid: true };
|
||||
}
|
||||
return first
|
||||
? { ...first, repaired: false, valid: false }
|
||||
: {
|
||||
polygons: [],
|
||||
decimals: params.decimals,
|
||||
repaired: false,
|
||||
valid: false,
|
||||
};
|
||||
}
|
||||
|
||||
// ── names ────────────────────────────────────────────────────────────────
|
||||
|
||||
const GOVERNMENT =
|
||||
/\s+(?:metropolitan government|unified government|consolidated government|metro government|city and county|urban county)\b.*$/i;
|
||||
|
||||
/**
|
||||
* Everyday names for records whose Census name is formal: "Nashville-Davidson
|
||||
* metropolitan government (balance)" is also "Nashville"; "Urban Honolulu"
|
||||
* is "Honolulu".
|
||||
*/
|
||||
export function placeAliases(name) {
|
||||
const out = new Set();
|
||||
let base = String(name).replace(/\s*\(balance\)\s*$/i, '');
|
||||
if (base !== name) out.add(base);
|
||||
const stripped = base.replace(GOVERNMENT, '');
|
||||
if (stripped !== base) {
|
||||
out.add(stripped);
|
||||
base = stripped;
|
||||
}
|
||||
if (base !== name && /[-/]/.test(base)) out.add(base.split(/[-/]/)[0].trim());
|
||||
const urban = /^Urban (.+)$/.exec(name);
|
||||
if (urban) out.add(urban[1]);
|
||||
out.delete(name);
|
||||
return [...out].filter(Boolean).sort();
|
||||
}
|
||||
|
||||
// ── pack ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function bboxOf(polygons) {
|
||||
let w = Infinity;
|
||||
let s = Infinity;
|
||||
let e = -Infinity;
|
||||
let n = -Infinity;
|
||||
for (const poly of polygons)
|
||||
for (const [lon, lat] of poly[0]) {
|
||||
if (lon < w) w = lon;
|
||||
if (lon > e) e = lon;
|
||||
if (lat < s) s = lat;
|
||||
if (lat > n) n = lat;
|
||||
}
|
||||
return [w, s, e, n];
|
||||
}
|
||||
|
||||
function union(boxes) {
|
||||
return [
|
||||
Math.min(...boxes.map((b) => b[0])),
|
||||
Math.min(...boxes.map((b) => b[1])),
|
||||
Math.max(...boxes.map((b) => b[2])),
|
||||
Math.max(...boxes.map((b) => b[3])),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The box of a state's places. Alaska crosses the antimeridian: its box is
|
||||
* taken with western longitudes shifted past 180 when that is narrower.
|
||||
*/
|
||||
function unionBox(boxes) {
|
||||
const plain = union(boxes);
|
||||
const shifted = union(
|
||||
boxes.map(([w, s, e, n]) =>
|
||||
e < 0 ? [w + 360, s, e + 360, n] : [w, s, e, n],
|
||||
),
|
||||
);
|
||||
return shifted[2] - shifted[0] < plain[2] - plain[0] ? shifted : plain;
|
||||
}
|
||||
|
||||
async function build() {
|
||||
const params = PARAMS;
|
||||
const source = await fetchPinned('places', SOURCE);
|
||||
const files = unzip(source.bytes);
|
||||
const base = path.basename(new URL(SOURCE.url).pathname, '.zip');
|
||||
const shapes = readShp(files.get(`${base}.shp`));
|
||||
const records = readDbf(files.get(`${base}.dbf`));
|
||||
if (shapes.length !== records.length)
|
||||
throw new Error('places: shp/dbf record counts differ');
|
||||
const states = new Map();
|
||||
const stats = { records: records.length, repaired: [], invalid: [] };
|
||||
records.forEach((record, index) => {
|
||||
const unit = simplifyPlace(shpPolygons(shapes[index]), params);
|
||||
if (!unit.polygons.length) throw new Error(`no geometry: ${record.GEOID}`);
|
||||
if (unit.repaired) stats.repaired.push(record.GEOID);
|
||||
if (!unit.valid) stats.invalid.push(record.GEOID);
|
||||
const alt = placeAliases(record.NAME);
|
||||
const lsad = record.NAMELSAD.startsWith(record.NAME)
|
||||
? record.NAMELSAD.slice(record.NAME.length).trim()
|
||||
: '';
|
||||
let state = states.get(record.STUSPS);
|
||||
if (!state) {
|
||||
state = {
|
||||
st: record.STUSPS,
|
||||
fips: record.STATEFP,
|
||||
name: record.STATE_NAME,
|
||||
features: [],
|
||||
boxes: [],
|
||||
};
|
||||
states.set(record.STUSPS, state);
|
||||
}
|
||||
state.boxes.push(bboxOf(unit.polygons));
|
||||
state.features.push({
|
||||
geoid: record.GEOID,
|
||||
name: record.NAME,
|
||||
...(alt.length ? { alt } : {}),
|
||||
full: record.NAMELSAD,
|
||||
...(lsad ? { lsad } : {}),
|
||||
...(CDP_LSAD.has(record.LSAD) ? { cdp: true } : {}),
|
||||
...(unit.decimals !== params.decimals ? { d: unit.decimals } : {}),
|
||||
polygons: encodeUnit(unit),
|
||||
});
|
||||
});
|
||||
if (stats.invalid.length)
|
||||
throw new Error(`places still invalid: ${stats.invalid.join(', ')}`);
|
||||
|
||||
await mkdir(OUT, { recursive: true });
|
||||
for (const file of await readdir(OUT))
|
||||
if (/^[A-Z]{2}\.json$/.test(file)) await rm(path.join(OUT, file));
|
||||
const list = [...states.values()].sort((a, b) => a.st.localeCompare(b.st));
|
||||
let bytes = 0;
|
||||
for (const state of list) {
|
||||
state.features.sort((a, b) => a.geoid.localeCompare(b.geoid));
|
||||
const text =
|
||||
JSON.stringify({
|
||||
st: state.st,
|
||||
state: state.name,
|
||||
fips: state.fips,
|
||||
features: state.features,
|
||||
}) + '\n';
|
||||
bytes += Buffer.byteLength(text);
|
||||
await writeFile(path.join(OUT, `${state.st}.json`), text);
|
||||
}
|
||||
const round = (v) => Math.round(v * 1e4) / 1e4;
|
||||
const index = {
|
||||
meta: {
|
||||
title: 'US Census Bureau cartographic boundary places',
|
||||
source: base,
|
||||
url: SOURCE.url,
|
||||
sha256: source.digest,
|
||||
license: 'Public domain (U.S. Government work, 17 U.S.C. § 105)',
|
||||
schema:
|
||||
'states[]: st (USPS), fips, name, count, bbox [w, s, e, n] (e > 180 across the antimeridian). <st>.json: st, state, fips, features[]: geoid, name (NAME), alt?[] (everyday names for formal records), full (NAMELSAD), lsad? (city, town, CDP, …), cdp? (census designated place), d? (decimals when not meta.decimals), label? [lon, lat] (verified filled anchor in degrees when the outer centroid is unfilled), polygons[][ring] (first ring outer, rest holes; each ring open and encoded as integers in 10^-d degrees, first vertex absolute then [dLon, dLat] deltas)',
|
||||
decimals: params.decimals,
|
||||
curation: params,
|
||||
script: 'scripts/build-census-places.mjs',
|
||||
},
|
||||
states: list.map((s) => ({
|
||||
st: s.st,
|
||||
fips: s.fips,
|
||||
name: s.name,
|
||||
count: s.features.length,
|
||||
bbox: unionBox(s.boxes).map(round),
|
||||
})),
|
||||
};
|
||||
await writeFile(
|
||||
path.join(OUT, 'index.json'),
|
||||
JSON.stringify(index, null, 1) + '\n',
|
||||
);
|
||||
await writeFile(path.join(OUT, 'files.js'), filesModule(list));
|
||||
const count = list.reduce((n, s) => n + s.features.length, 0);
|
||||
console.log(
|
||||
`us_census_places: ${count} places in ${list.length} files, ${bytes} bytes; ` +
|
||||
`${stats.repaired.length} re-simplified to stay valid`,
|
||||
);
|
||||
const args = process.argv.slice(2);
|
||||
const python = args.includes('--python')
|
||||
? args[args.indexOf('--python') + 1]
|
||||
: 'python3';
|
||||
execFileSync(
|
||||
python,
|
||||
[path.join(ROOT, 'scripts/pack_geometry.py'), 'repair', 'census'],
|
||||
{ stdio: 'inherit', env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' } },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The file list as literal URLs, so a bundler emits every state file as an
|
||||
* asset and the loader can fetch one by its code.
|
||||
*/
|
||||
function filesModule(list) {
|
||||
const lines = [
|
||||
'// Generated by scripts/build-census-places.mjs; do not edit.',
|
||||
"export const INDEX_URL = () => new URL('./index.json', import.meta.url);",
|
||||
'export const STATE_URLS = {',
|
||||
...list.map(
|
||||
(s) => ` ${s.st}: () => new URL('./${s.st}.json', import.meta.url),`,
|
||||
),
|
||||
'};',
|
||||
];
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
const isMain =
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
if (isMain) await build();
|
||||
@@ -0,0 +1,434 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maintainer-only Who's On First neighborhood pack build (Shapely 2.1).
|
||||
|
||||
Reads the pinned Who's On First neighbourhood, macrohood and microhood
|
||||
archives and the WOF sources registry, keeps current polygon records whose
|
||||
geometry source matches the reviewed open-licence allowlist, simplifies them
|
||||
to about 15 m, then applies pinned source/ID exclusions after partitioning, and
|
||||
writes src/data/local_data/wof_neighborhoods/:
|
||||
|
||||
index.json format, licences per source, tile list with boxes
|
||||
<tile>.json records whose box centre falls in that quadtree tile
|
||||
files.js their URLs, for bundlers
|
||||
ATTRIBUTION.md per-source credits
|
||||
|
||||
Usage:
|
||||
python scripts/build-wof-neighborhoods.py --cache /tmp/wof-neighborhoods [--verify]
|
||||
|
||||
Requires shapely==2.1.2 on GEOS 3.13.1 (checked). --verify rebuilds in memory
|
||||
and fails unless every shipped file matches byte for byte.
|
||||
|
||||
Downloads go to --cache and are checked against the pinned SHA-256. The
|
||||
neighbourhood archive is about 400 MB.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
from shapely import make_valid
|
||||
from shapely.affinity import scale
|
||||
from shapely.geometry import MultiPolygon, shape
|
||||
|
||||
import pack_geometry
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
PACK = ROOT / 'src/data/local_data/wof_neighborhoods'
|
||||
# The legacy distribution is frozen (last published 2025-10-14) and only
|
||||
# offers `-latest` names; the SHA-256 pins make a changed file fail loudly.
|
||||
DIST = 'https://data.geocode.earth/wof/dist/legacy'
|
||||
ARCHIVES = {
|
||||
'neighbourhood': {
|
||||
'url': DIST + '/whosonfirst-data-neighbourhood-latest.tar.bz2',
|
||||
'sha256': '8e5e413aacf2244e4e4167fbd9be860bc407ae6de8b733e47f13abe6d3d10455',
|
||||
},
|
||||
'macrohood': {
|
||||
'url': DIST + '/whosonfirst-data-macrohood-latest.tar.bz2',
|
||||
'sha256': 'f5fbfabcb8229b83242dbd38530c27e61867e8b3b097e3393d9c841962a99679',
|
||||
},
|
||||
'microhood': {
|
||||
'url': DIST + '/whosonfirst-data-microhood-latest.tar.bz2',
|
||||
'sha256': 'e8197fa576e8831be8f091ac7e219d29224d89404ab0c56d377339e42a8f29ac',
|
||||
},
|
||||
}
|
||||
# The registry at a fixed commit.
|
||||
SOURCES = {
|
||||
'url': 'https://raw.githubusercontent.com/whosonfirst/whosonfirst-sources/'
|
||||
'e17af153c144fe6e08aa8db2607b4bf8aecd448d/data/sources-spec-latest.json',
|
||||
'sha256': 'd8a5dd873d26eee11583de4438185c046dec33848d5bde27047ca4bff44d9c39',
|
||||
}
|
||||
TOLERANCE_M = 15
|
||||
PRECISION = 5
|
||||
TILE_MAX_BYTES = 200_000
|
||||
TILE_MAX_DEPTH = 16
|
||||
LICENSE_POLICY = json.loads((ROOT / 'scripts/wof-open-licenses.json').read_text())
|
||||
ALLOWED_LICENSES = frozenset(LICENSE_POLICY['licenses'])
|
||||
REVIEWED_SOURCES = LICENSE_POLICY.get('reviewedSources', {})
|
||||
RECORD_EXCLUSIONS = LICENSE_POLICY['recordExclusions']
|
||||
FIELDS = ['id', 'name', 'country', 'type', 'bbox', 'aliases', 'source', 'rings', 'label']
|
||||
|
||||
|
||||
def fetch(cache, name, url, sha256):
|
||||
path = cache / name
|
||||
if not path.exists():
|
||||
print('Downloading', url, flush=True)
|
||||
# Plain urllib requests are refused by the WOF host; curl is not.
|
||||
subprocess.run(['curl', '--fail', '-L', '-o', str(path), url], check=True)
|
||||
digest = hashlib.sha256()
|
||||
with open(path, 'rb') as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b''):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != sha256:
|
||||
raise ValueError(f'{name}: SHA-256 {digest.hexdigest()} is not the pinned {sha256}')
|
||||
return path
|
||||
|
||||
|
||||
# ── geometry ────────────────────────────────────────────────────────────
|
||||
|
||||
def simplify(geometry):
|
||||
"""Simplify at TOLERANCE_M in local metres, snap to the pack grid, keep it
|
||||
valid, and order parts largest first. Returns (rings, label) where the
|
||||
label is a point inside the largest part, or ([], None)."""
|
||||
geometry = MultiPolygon(pack_geometry.polygon_parts(make_valid(geometry) if not geometry.is_valid else geometry))
|
||||
if geometry.is_empty:
|
||||
return [], None
|
||||
lat = geometry.centroid.y
|
||||
sx = 111320 * max(0.001, math.cos(math.radians(lat)))
|
||||
sy = 110574
|
||||
metric = scale(geometry, xfact=sx, yfact=sy, origin=(0, 0))
|
||||
metric = metric.simplify(TOLERANCE_M, preserve_topology=True)
|
||||
geometry = scale(metric, xfact=1 / sx, yfact=1 / sy, origin=(0, 0))
|
||||
# To whole pack units (10^-PRECISION degrees), where GEOS is exact.
|
||||
units = scale(geometry, xfact=10 ** PRECISION, yfact=10 ** PRECISION, origin=(0, 0))
|
||||
parts = pack_geometry.ordered_parts(pack_geometry.settle(units))
|
||||
if not parts:
|
||||
return [], None
|
||||
rings = [
|
||||
[pack_geometry.encode_polyline(pack_geometry.open_ring(p.exterior.coords))]
|
||||
+ [pack_geometry.encode_polyline(pack_geometry.open_ring(r.coords)) for r in p.interiors]
|
||||
for p in parts
|
||||
]
|
||||
# Check what ships: the geometry decoded from the encoded strings.
|
||||
decoded = [[pack_geometry.decode_polyline(r) for r in poly] for poly in rings]
|
||||
why = pack_geometry.problems(decoded)
|
||||
if why:
|
||||
raise ValueError(f'encoded geometry invalid: {why}')
|
||||
point = parts[0].representative_point()
|
||||
point = shape({'type': 'Point', 'coordinates': [round(point.x) / 10 ** PRECISION, round(point.y) / 10 ** PRECISION]})
|
||||
return rings, [point.x, point.y]
|
||||
|
||||
|
||||
# ── records ─────────────────────────────────────────────────────────────
|
||||
|
||||
def read_archive(path, records):
|
||||
with tarfile.open(path, 'r|bz2') as archive:
|
||||
for member in archive:
|
||||
if not member.name.endswith('.geojson') or '-alt-' in member.name:
|
||||
continue
|
||||
feature = json.load(archive.extractfile(member))
|
||||
if feature['geometry']['type'] not in ('Polygon', 'MultiPolygon'):
|
||||
records.pop(feature['properties']['wof:id'], None)
|
||||
continue
|
||||
records[feature['properties']['wof:id']] = feature
|
||||
|
||||
|
||||
def keep(properties):
|
||||
deprecated = properties.get('edtf:deprecated', 'uuuu') not in ('uuuu', '', None)
|
||||
superseded = bool(properties.get('wof:superseded_by', []))
|
||||
return properties.get('mz:is_current', -1) != 0 and not deprecated and not superseded
|
||||
|
||||
|
||||
def source_table(spec):
|
||||
lookup = {}
|
||||
for source in spec.values():
|
||||
for key in (source.get('name'), source.get('prefix')):
|
||||
if key:
|
||||
lookup[key] = source
|
||||
return lookup
|
||||
|
||||
|
||||
def verify_source_reviews(lookup):
|
||||
"""Pin exceptional source decisions to the exact reviewed registry evidence."""
|
||||
registry_commit = SOURCES['url'].split('/')[-3]
|
||||
for key, review in REVIEWED_SOURCES.items():
|
||||
source = lookup.get(key)
|
||||
if not source:
|
||||
raise ValueError(f'{key}: reviewed source is absent from the pinned registry')
|
||||
if review.get('registryCommit') != registry_commit:
|
||||
raise ValueError(f'{key}: review does not match the pinned registry commit')
|
||||
if review.get('registrySha256') != SOURCES['sha256']:
|
||||
raise ValueError(f'{key}: review does not match the pinned registry SHA-256')
|
||||
if review.get('rawLicenseType') != source.get('license_type'):
|
||||
raise ValueError(f'{key}: reviewed raw licence no longer matches the registry')
|
||||
text = source.get('license_text') or ''
|
||||
if review.get('licenseTextSha256') != hashlib.sha256(text.encode()).hexdigest():
|
||||
raise ValueError(f'{key}: reviewed licence text no longer matches the registry')
|
||||
if review['rawLicenseType'] not in ALLOWED_LICENSES:
|
||||
raise ValueError(f'{key}: reviewed raw licence is not admitted by policy')
|
||||
|
||||
|
||||
def licence_allowed(value):
|
||||
"""Only exact raw registry strings reviewed in the policy are accepted."""
|
||||
return isinstance(value, str) and value in ALLOWED_LICENSES
|
||||
|
||||
|
||||
def source_allowed(key, source):
|
||||
return (key != 'unknown' and bool(source) and
|
||||
licence_allowed(source.get('license_type')))
|
||||
|
||||
|
||||
def build_records(archives, lookup):
|
||||
raw = {}
|
||||
for kind in ('neighbourhood', 'macrohood', 'microhood'):
|
||||
read_archive(archives[kind], raw)
|
||||
print(kind, len(raw), flush=True)
|
||||
features = []
|
||||
excluded = collections.Counter()
|
||||
for wof_id in sorted(raw):
|
||||
properties = raw[wof_id]['properties']
|
||||
if not keep(properties):
|
||||
continue
|
||||
key = properties.get('src:geom', 'unknown')
|
||||
if not source_allowed(key, lookup.get(key)):
|
||||
excluded[key] += 1
|
||||
continue
|
||||
name = properties['wof:name']
|
||||
aliases = sorted({
|
||||
alias
|
||||
for k, values in properties.items()
|
||||
if k.startswith('name:') and isinstance(values, list)
|
||||
for alias in values
|
||||
if isinstance(alias, str) and alias != name
|
||||
})
|
||||
if not name.strip() and not aliases:
|
||||
continue # nothing to ask for it by
|
||||
geometry = shape(raw[wof_id]['geometry'])
|
||||
if geometry.is_empty:
|
||||
continue
|
||||
rings, label = simplify(geometry)
|
||||
if not rings:
|
||||
continue
|
||||
country = properties.get('iso:country', properties.get('wof:country'))
|
||||
features.append([
|
||||
wof_id, name, country, properties['wof:placetype'],
|
||||
[round(v, PRECISION) for v in geometry.bounds], aliases, key, rings, label,
|
||||
])
|
||||
return features, excluded
|
||||
|
||||
|
||||
# ── tiles ───────────────────────────────────────────────────────────────
|
||||
|
||||
def compact(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
||||
|
||||
|
||||
def tiles(features, key='', box=(-180.0, -90.0, 180.0, 90.0)):
|
||||
size = sum(len(compact(f).encode()) for f in features)
|
||||
if size <= TILE_MAX_BYTES or len(key) >= TILE_MAX_DEPTH:
|
||||
yield key or 'world', features
|
||||
return
|
||||
west, south, east, north = box
|
||||
mx, my = (west + east) / 2, (south + north) / 2
|
||||
quads = [[], [], [], []]
|
||||
for f in features:
|
||||
x = (f[4][0] + f[4][2]) / 2
|
||||
y = (f[4][1] + f[4][3]) / 2
|
||||
quads[(1 if x >= mx else 0) + (2 if y >= my else 0)].append(f)
|
||||
boxes = [(west, south, mx, my), (mx, south, east, my), (west, my, mx, north), (mx, my, east, north)]
|
||||
for i, quad in enumerate(quads):
|
||||
if quad:
|
||||
yield from tiles(quad, key + str(i), boxes[i])
|
||||
|
||||
|
||||
def union_box(features):
|
||||
return [
|
||||
min(f[4][0] for f in features), min(f[4][1] for f in features),
|
||||
max(f[4][2] for f in features), max(f[4][3] for f in features),
|
||||
]
|
||||
|
||||
|
||||
def exclude_records(partition):
|
||||
"""Filter approved source+IDs AFTER the original byte-sized partition."""
|
||||
if RECORD_EXCLUSIONS['archives'] != {k: a['sha256'] for k, a in ARCHIVES.items()}:
|
||||
raise ValueError('record exclusions do not match the pinned archives')
|
||||
if RECORD_EXCLUSIONS['registrySha256'] != SOURCES['sha256']:
|
||||
raise ValueError('record exclusions do not match the pinned registry')
|
||||
expected = {}
|
||||
for source, rule in RECORD_EXCLUSIONS['sources'].items():
|
||||
if rule['count'] != len(rule['ids']) or len(set(rule['ids'])) != len(rule['ids']):
|
||||
raise ValueError(f'{source}: invalid exclusion membership')
|
||||
for record_id in rule['ids']:
|
||||
if record_id in expected:
|
||||
raise ValueError(f'{record_id}: duplicate exclusion')
|
||||
expected[record_id] = source
|
||||
seen = set()
|
||||
kept = []
|
||||
for key, members in partition:
|
||||
retained = []
|
||||
for row in members:
|
||||
record_id, source = row[0], row[6]
|
||||
if record_id in expected:
|
||||
if source != expected[record_id] or record_id in seen:
|
||||
raise ValueError(f'{record_id}: exclusion source or membership changed')
|
||||
seen.add(record_id)
|
||||
elif source in RECORD_EXCLUSIONS['sources']:
|
||||
raise ValueError(f'{record_id}: unreviewed exclusion-source member')
|
||||
else:
|
||||
retained.append(row)
|
||||
if not retained:
|
||||
raise ValueError(f'{key}: exclusion would change the tile layout')
|
||||
kept.append((key, retained))
|
||||
if seen != set(expected):
|
||||
raise ValueError(f'missing excluded records: {sorted(set(expected) - seen)}')
|
||||
return kept
|
||||
|
||||
|
||||
# ── credits ─────────────────────────────────────────────────────────────
|
||||
|
||||
def licence_label(key, source):
|
||||
licence = source['license_type']
|
||||
if key == 'karmashapes':
|
||||
return 'CC BY 4.0'
|
||||
return {
|
||||
'CC-BY': 'CC BY', 'Attribution 3.0 Unported': 'CC BY 3.0', 'Public domain': 'Public Domain',
|
||||
'Open Government License - British Columbia, v2.0': 'Open Government Licence - British Columbia, v2.0',
|
||||
}.get(licence, licence)
|
||||
|
||||
|
||||
def exclusion_note():
|
||||
groups = ', '.join(f'{rule["count"]} `{key}` records' for key, rule in RECORD_EXCLUSIONS['sources'].items())
|
||||
lead = f'This derived pack excludes {groups}. ' if groups else ''
|
||||
return f'{lead}{RECORD_EXCLUSIONS["reason"]} Original upstream notices remain unchanged.'
|
||||
|
||||
|
||||
def attribution(counts, lookup):
|
||||
lines = [
|
||||
'# Neighborhood boundary credits',
|
||||
'',
|
||||
"Neighborhood boundaries from [Who's On First](https://whosonfirst.org/docs/licenses/) "
|
||||
'and contributors, used under the licences below. Modified: boundaries simplified to about 15 m, repaired and re-encoded.',
|
||||
'',
|
||||
'| Source | Records | Licence | Terms |',
|
||||
'|---|---:|---|---|',
|
||||
]
|
||||
for key, count in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])):
|
||||
source = lookup[key]
|
||||
name = source.get('fullname') or key
|
||||
terms = source.get('license') or ''
|
||||
link = f'[terms]({terms})' if terms.startswith('http') else terms
|
||||
lines.append(f'| {name} (`{key}`) | {count:,} | {licence_label(key, source)} | {link} |')
|
||||
via = [(key, lookup[key]) for key in sorted(counts) if lookup[key].get('src:via')]
|
||||
if via:
|
||||
lines += ['', '## Upstream sources', '']
|
||||
for key, source in via:
|
||||
lines += [f'### {source.get("fullname") or key}', '']
|
||||
for v in source['src:via']:
|
||||
note = f' ({v["source_note"]})' if v.get('source_note') else ''
|
||||
context = f'{v["context"]}: ' if v.get('context') else ''
|
||||
lines.append(f'- {context}{v.get("source_name", "")}{note} {v.get("source_link", "")}'.rstrip())
|
||||
lines.append('')
|
||||
while lines[-1] == '':
|
||||
lines.pop()
|
||||
lines += ['', '## Source-specific exclusions', '', exclusion_note()]
|
||||
return '\n'.join(lines).rstrip() + '\n'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--cache', type=Path, required=True)
|
||||
parser.add_argument('--verify', action='store_true', help='rebuild in memory and compare with the shipped pack')
|
||||
args = parser.parse_args()
|
||||
pack_geometry.require_pinned_tooling()
|
||||
args.cache.mkdir(parents=True, exist_ok=True)
|
||||
archives = {
|
||||
kind: fetch(args.cache, Path(a['url']).name, a['url'], a['sha256'])
|
||||
for kind, a in ARCHIVES.items()
|
||||
}
|
||||
spec = json.loads(fetch(args.cache, 'sources-spec-latest.json', SOURCES['url'], SOURCES['sha256']).read_text())
|
||||
lookup = source_table(spec)
|
||||
verify_source_reviews(lookup)
|
||||
features, excluded = build_records(archives, lookup)
|
||||
if len({f[0] for f in features}) != len(features):
|
||||
raise ValueError('duplicate WOF id')
|
||||
|
||||
out = {}
|
||||
tile_list = []
|
||||
# Partition the full pinned input first: early filtering can collapse tiles.
|
||||
partition = exclude_records(list(tiles(features)))
|
||||
features = [row for _, members in partition for row in members]
|
||||
for key, members in partition:
|
||||
members.sort(key=lambda f: f[0])
|
||||
out[f'{key}.json'] = compact({'tile': key, 'features': members}) + '\n'
|
||||
tile_list.append({'key': key, 'count': len(members), 'bbox': union_box(members)})
|
||||
counts = collections.Counter(f[6] for f in features)
|
||||
index = {
|
||||
'meta': {
|
||||
'title': "Who's On First neighbourhood, macrohood and microhood polygons",
|
||||
'archives': ARCHIVES,
|
||||
'sourcesRegistry': SOURCES,
|
||||
'recordExclusions': {key: rule['count'] for key, rule in RECORD_EXCLUSIONS['sources'].items()},
|
||||
'recordExclusionReason': RECORD_EXCLUSIONS['reason'],
|
||||
'license': 'Per source; see sources and ATTRIBUTION.md',
|
||||
'format': 'neighborhood-polylines-v1',
|
||||
'fields': FIELDS,
|
||||
'precision': PRECISION,
|
||||
'coordinateOrder': 'lon,lat',
|
||||
'toleranceM': TOLERANCE_M,
|
||||
'schema': (
|
||||
'tiles[]: key, count, bbox [w, s, e, n] of its records. <key>.json: tile, features[] as '
|
||||
'fields: WOF id, name, ISO country, placetype, bbox, aliases[] (other-language names), '
|
||||
'geometry source key, rings (polygons, largest first, each [outer, ...holes]; each ring '
|
||||
'an encoded polyline string: zigzag varint deltas of lon then lat at 10^-precision '
|
||||
'degrees), label [lon, lat] (a point inside the largest polygon)'
|
||||
),
|
||||
'sources': {
|
||||
key: {
|
||||
'name': lookup[key].get('fullname') or key,
|
||||
'license': licence_label(key, lookup[key]),
|
||||
'terms': lookup[key].get('license') or '',
|
||||
'records': counts[key],
|
||||
}
|
||||
for key in sorted(counts)
|
||||
},
|
||||
'script': 'scripts/build-wof-neighborhoods.py',
|
||||
},
|
||||
'tiles': tile_list,
|
||||
}
|
||||
out['index.json'] = json.dumps(index, ensure_ascii=False, indent=1) + '\n'
|
||||
out['ATTRIBUTION.md'] = attribution(counts, lookup)
|
||||
# Literal URLs, so a bundler emits every tile as an asset.
|
||||
out['files.js'] = '\n'.join([
|
||||
'// Generated by scripts/build-wof-neighborhoods.py; do not edit.',
|
||||
"export const INDEX_URL = () => new URL('./index.json', import.meta.url);",
|
||||
'export const TILE_URLS = {',
|
||||
*[f" t{t['key']}: () => new URL('./{t['key']}.json', import.meta.url)," for t in tile_list],
|
||||
'};',
|
||||
]) + '\n'
|
||||
if args.verify:
|
||||
# Reproducibility: a rebuild must match what ships, byte for byte.
|
||||
shipped = {p.name for p in PACK.glob('*.json')} | {'ATTRIBUTION.md', 'files.js'}
|
||||
differ = sorted(
|
||||
name for name in shipped | set(out)
|
||||
if not (PACK / name).exists() or name not in out
|
||||
or (PACK / name).read_bytes() != out[name].encode('utf-8')
|
||||
)
|
||||
if differ:
|
||||
raise SystemExit(f'rebuild differs from the shipped pack: {differ[:10]}')
|
||||
print(f'wof_neighborhoods: rebuild matches all {len(out)} shipped files')
|
||||
return
|
||||
PACK.mkdir(parents=True, exist_ok=True)
|
||||
for old in PACK.glob('*.json'):
|
||||
old.unlink()
|
||||
for name, text in out.items():
|
||||
(PACK / name).write_text(text, encoding='utf-8')
|
||||
print(f'wof_neighborhoods: {len(features)} records in {len(tile_list)} tiles; '
|
||||
f'{sum(rule["count"] for rule in RECORD_EXCLUSIONS["sources"].values())} source-specific record exclusions; '
|
||||
f'excluded {sum(excluded.values())} with an unapproved licence: {dict(excluded)}', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1111,6 +1111,9 @@
|
||||
"src/voice/globeResetConfirmation.test.mjs",
|
||||
"src/hudDms.test.mjs",
|
||||
"src/hudSummaryGate.test.mjs",
|
||||
"src/hudSummaryLine.test.mjs",
|
||||
"src/testSupport/stubMgrs.mjs",
|
||||
"src/tools/queries/hud-caption-feed-state.test.mjs",
|
||||
"src/voice/hudPlaceLabels.test.mjs",
|
||||
"src/voice/panel/preferences.test.mjs",
|
||||
"src/voice/panel/preferencesView.test.mjs",
|
||||
@@ -1186,5 +1189,22 @@
|
||||
"src/firstRunCopy.js",
|
||||
"src/data/headingDegrees.test.mjs",
|
||||
"src/data/distanceFormat.test.mjs",
|
||||
"src/sources/adsbFields.test.mjs"
|
||||
"src/sources/adsbFields.test.mjs",
|
||||
"src/sources/phaseTiming.test.mjs",
|
||||
"scripts/build-census-places.mjs",
|
||||
"src/data/placeBoundaries.test.mjs",
|
||||
"src/data/boundaryLabels.test.mjs",
|
||||
"src/data/packGeometry.test.mjs",
|
||||
"src/annotations/placeOutline.test.mjs",
|
||||
"src/annotations/outlineIdentity.test.mjs",
|
||||
"src/annotations/outlineRungs.test.mjs",
|
||||
"src/annotations/outlineRungsFallback.test.mjs",
|
||||
"src/search/photonSetting.test.mjs",
|
||||
"src/sources/nominatimOutlines.test.mjs",
|
||||
"src/sources/openFreeMapOutlines.test.mjs",
|
||||
"src/sources/openFreeMapOutlinesEdges.test.mjs",
|
||||
"src/standalone/layerSources.test.mjs",
|
||||
"src/tooling/nominatimGuardLifecycle.test.mjs",
|
||||
"src/tooling/nominatimLockOwnership.test.mjs",
|
||||
"src/tooling/nominatimOutlineRoute.test.mjs"
|
||||
]
|
||||
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maintainer-only topology check and repair for the bundled outline packs.
|
||||
|
||||
python scripts/pack_geometry.py check census|wof every shipped feature,
|
||||
decoded as a whole MultiPolygon, is valid under GEOS, keeps at least
|
||||
three distinct vertices per ring, and lists its largest part first
|
||||
python scripts/pack_geometry.py repair census rewrite the Census place
|
||||
files so every feature passes `check` (run by build-census-places.mjs)
|
||||
|
||||
Geometry is checked and repaired in the packs' own integer units (10^-d
|
||||
degrees), where GEOS arithmetic is exact: a vertex that lies on another edge
|
||||
after rounding is seen as the touch it is.
|
||||
|
||||
Requires Shapely 2.1.2 on GEOS 3.13.1, pinned so repairs are reproducible.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import shapely
|
||||
from shapely import make_valid, set_precision
|
||||
from shapely.geometry import MultiPolygon, Polygon
|
||||
from shapely.geometry.polygon import orient
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DATA = ROOT / 'src/data/local_data'
|
||||
SHAPELY_VERSION = '2.1.2'
|
||||
GEOS_VERSION = (3, 13, 1)
|
||||
|
||||
|
||||
def require_pinned_tooling():
|
||||
if shapely.__version__ != SHAPELY_VERSION or shapely.geos_version != GEOS_VERSION:
|
||||
raise RuntimeError(
|
||||
f'Use shapely=={SHAPELY_VERSION} on GEOS {".".join(map(str, GEOS_VERSION))}; '
|
||||
f'found {shapely.__version__} on GEOS {shapely.geos_version}'
|
||||
)
|
||||
|
||||
|
||||
def compact(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
||||
|
||||
|
||||
# ── encodings ────────────────────────────────────────────────────────────
|
||||
|
||||
def decode_deltas(encoded):
|
||||
"""[x0, y0, dx1, dy1, …] → [(x, y), …] in pack units (Census/admin packs)."""
|
||||
x = y = 0
|
||||
ring = []
|
||||
for i in range(0, len(encoded) - 1, 2):
|
||||
x += encoded[i]
|
||||
y += encoded[i + 1]
|
||||
ring.append((x, y))
|
||||
return ring
|
||||
|
||||
|
||||
def encode_deltas(coords):
|
||||
out = []
|
||||
px = py = 0
|
||||
for fx, fy in coords:
|
||||
x, y = round(fx), round(fy)
|
||||
out += [x - px, y - py]
|
||||
px, py = x, y
|
||||
return out
|
||||
|
||||
|
||||
def decode_polyline(encoded):
|
||||
"""Zigzag varint deltas, 5-bit groups offset by 63, lon then lat; pack units."""
|
||||
values = []
|
||||
last = [0, 0]
|
||||
i = 0
|
||||
while i < len(encoded):
|
||||
n = shift = 0
|
||||
while True:
|
||||
b = ord(encoded[i]) - 63
|
||||
i += 1
|
||||
n |= (b & 31) << shift
|
||||
shift += 5
|
||||
if b < 32:
|
||||
break
|
||||
axis = len(values) % 2
|
||||
last[axis] += ~(n >> 1) if n & 1 else n >> 1
|
||||
values.append(last[axis])
|
||||
return list(zip(values[::2], values[1::2]))
|
||||
|
||||
|
||||
def encode_polyline(coords):
|
||||
out = []
|
||||
last = [0, 0]
|
||||
for x, y in coords:
|
||||
for i, value in enumerate([round(x), round(y)]):
|
||||
n = value - last[i]
|
||||
last[i] = value
|
||||
n = (n << 1) ^ (n >> 63)
|
||||
while n >= 32:
|
||||
out.append(chr((32 | (n & 31)) + 63))
|
||||
n >>= 5
|
||||
out.append(chr(n + 63))
|
||||
return ''.join(out)
|
||||
|
||||
|
||||
# ── geometry ─────────────────────────────────────────────────────────────
|
||||
|
||||
def polygon_parts(geometry):
|
||||
if geometry.geom_type == 'Polygon':
|
||||
return [] if geometry.is_empty else [geometry]
|
||||
if hasattr(geometry, 'geoms'):
|
||||
return [p for g in geometry.geoms for p in polygon_parts(g)]
|
||||
return []
|
||||
|
||||
|
||||
def open_ring(coords):
|
||||
coords = [tuple(c[:2]) for c in coords]
|
||||
return coords[:-1] if len(coords) > 1 and coords[0] == coords[-1] else coords
|
||||
|
||||
|
||||
def ordered_parts(geometry):
|
||||
"""Polygon parts oriented (outer counter-clockwise), largest area first."""
|
||||
parts = [orient(p, 1.0) for p in polygon_parts(geometry)]
|
||||
return sorted(parts, key=lambda p: (-p.area, p.bounds))
|
||||
|
||||
|
||||
def on_grid(geometry):
|
||||
"""Snap to whole pack units keeping the result valid; polygon parts only."""
|
||||
snapped = set_precision(geometry, 1, mode='valid_output')
|
||||
return MultiPolygon(polygon_parts(snapped))
|
||||
|
||||
|
||||
def settle(geometry):
|
||||
"""A valid MultiPolygon of `geometry` (in pack units) on whole units."""
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
geometry = on_grid(MultiPolygon(polygon_parts(geometry)))
|
||||
for _ in range(3):
|
||||
if geometry.is_valid:
|
||||
break
|
||||
geometry = on_grid(make_valid(geometry))
|
||||
return geometry
|
||||
|
||||
|
||||
def rings_ok(polygons):
|
||||
return all(len(set(ring)) >= 3 for poly in polygons for ring in poly)
|
||||
|
||||
|
||||
def problems(polygons):
|
||||
"""Why decoded `polygons` ([[outer, *holes], …]) fail the pack contract."""
|
||||
if not polygons or not rings_ok(polygons):
|
||||
return 'ring with fewer than three distinct vertices'
|
||||
try:
|
||||
geometry = MultiPolygon([Polygon(p[0], p[1:]) for p in polygons])
|
||||
except Exception as error: # noqa: BLE001 - malformed input is a finding
|
||||
return f'unbuildable: {error}'
|
||||
if not geometry.is_valid:
|
||||
return shapely.is_valid_reason(geometry)
|
||||
areas = [part.area for part in geometry.geoms]
|
||||
if max(areas) > areas[0]:
|
||||
return 'largest part is not first'
|
||||
return None
|
||||
|
||||
|
||||
# ── packs ────────────────────────────────────────────────────────────────
|
||||
|
||||
def census_features():
|
||||
folder = DATA / 'us_census_places'
|
||||
decimals = json.loads((folder / 'index.json').read_text())['meta']['decimals']
|
||||
for path in sorted(folder.glob('[A-Z][A-Z].json')):
|
||||
pack = json.loads(path.read_text(encoding='utf-8'))
|
||||
yield path, pack, decimals
|
||||
|
||||
|
||||
def census_polygons(feature):
|
||||
return [[decode_deltas(r) for r in poly] for poly in feature['polygons']]
|
||||
|
||||
|
||||
def census_degrees(feature, decimals):
|
||||
factor = 10 ** feature.get('d', decimals)
|
||||
return [[[(x / factor, y / factor) for x, y in ring]
|
||||
for ring in poly] for poly in census_polygons(feature)]
|
||||
|
||||
|
||||
def ring_contains(ring, lat, lon):
|
||||
"""Match adminBoundaries.pointInRing on final degree-decoded vertices."""
|
||||
inside = False
|
||||
for i in range(len(ring)):
|
||||
xi, yi = ring[i]
|
||||
xj, yj = ring[i - 1]
|
||||
if (yi > lat) != (yj > lat) and lon < (xj - xi) * (lat - yi) / (yj - yi) + xi:
|
||||
inside = not inside
|
||||
return inside
|
||||
|
||||
|
||||
def label_is_filled(polygons, label):
|
||||
if not isinstance(label, (list, tuple)) or len(label) != 2:
|
||||
return False
|
||||
lon, lat = label
|
||||
if not all(isinstance(n, (int, float)) and math.isfinite(n) for n in label):
|
||||
return False
|
||||
return any(ring_contains(poly[0], lat, lon)
|
||||
and not any(ring_contains(hole, lat, lon) for hole in poly[1:])
|
||||
for poly in polygons)
|
||||
|
||||
|
||||
def census_label(feature, polygons):
|
||||
"""The effective runtime label, including its area-weighted fallback."""
|
||||
if isinstance(feature.get('label'), list):
|
||||
return feature['label']
|
||||
ring = polygons[0][0]
|
||||
area = cx = cy = 0
|
||||
for i in range(len(ring)):
|
||||
x0, y0 = ring[i - 1]
|
||||
x1, y1 = ring[i]
|
||||
cross = x0 * y1 - x1 * y0
|
||||
area += cross
|
||||
cx += (x0 + x1) * cross
|
||||
cy += (y0 + y1) * cross
|
||||
return [cx / (3 * area), cy / (3 * area)] if area else list(ring[0])
|
||||
|
||||
|
||||
def wof_features():
|
||||
folder = DATA / 'wof_neighborhoods'
|
||||
index = json.loads((folder / 'index.json').read_text(encoding='utf-8'))
|
||||
for tile in index['tiles']:
|
||||
pack = json.loads((folder / f'{tile["key"]}.json').read_text(encoding='utf-8'))
|
||||
for feature in pack['features']:
|
||||
yield feature, index['meta']['precision']
|
||||
|
||||
|
||||
def check(pack):
|
||||
bad = []
|
||||
total = 0
|
||||
if pack == 'census':
|
||||
for _, data, decimals in census_features():
|
||||
for feature in data['features']:
|
||||
total += 1
|
||||
why = problems(census_polygons(feature))
|
||||
if not why:
|
||||
polygons = census_degrees(feature, decimals)
|
||||
if not label_is_filled(polygons, census_label(feature, polygons)):
|
||||
why = 'effective label is outside filled geometry'
|
||||
if why:
|
||||
bad.append((feature['geoid'], feature['name'], why))
|
||||
else:
|
||||
for feature, precision in wof_features():
|
||||
total += 1
|
||||
polygons = [[decode_polyline(r) for r in poly] for poly in feature[7]]
|
||||
why = problems(polygons)
|
||||
if why:
|
||||
bad.append((feature[0], feature[1], why))
|
||||
for row in bad[:20]:
|
||||
print('invalid', *row)
|
||||
print(f'{pack}: {total} features, {len(bad)} failing')
|
||||
return not bad
|
||||
|
||||
|
||||
def repair_census():
|
||||
repaired = 0
|
||||
labelled = 0
|
||||
for path, data, decimals in census_features():
|
||||
changed = False
|
||||
for feature in data['features']:
|
||||
polygons = census_polygons(feature)
|
||||
if problems(polygons):
|
||||
geometry = MultiPolygon([Polygon(p[0], p[1:]) for p in polygons if len(set(p[0])) >= 3])
|
||||
parts = ordered_parts(settle(geometry))
|
||||
feature['polygons'] = [
|
||||
[encode_deltas(open_ring(p.exterior.coords))]
|
||||
+ [encode_deltas(open_ring(r.coords)) for r in p.interiors]
|
||||
for p in parts
|
||||
]
|
||||
feature.pop('label', None)
|
||||
polygons = census_polygons(feature)
|
||||
why = problems(polygons)
|
||||
if why:
|
||||
raise ValueError(f'{feature["geoid"]} {feature["name"]}: still {why}')
|
||||
repaired += 1
|
||||
changed = True
|
||||
decoded = census_degrees(feature, decimals)
|
||||
if label_is_filled(decoded, census_label(feature, decoded)):
|
||||
continue
|
||||
factor = 10 ** feature.get('d', decimals)
|
||||
for poly in polygons:
|
||||
point = Polygon(poly[0], poly[1:]).representative_point()
|
||||
label = [point.x / factor, point.y / factor]
|
||||
if label_is_filled(decoded, label):
|
||||
feature['label'] = label
|
||||
labelled += 1
|
||||
changed = True
|
||||
break
|
||||
else:
|
||||
raise ValueError(f'{feature["geoid"]} {feature["name"]}: no verified filled label')
|
||||
if changed:
|
||||
path.write_text(compact(data) + '\n', encoding='utf-8')
|
||||
print(f'us_census_places: {repaired} topology repairs, {labelled} verified interior labels added')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('command', choices=['check', 'repair'])
|
||||
parser.add_argument('pack', choices=['census', 'wof'])
|
||||
args = parser.parse_args()
|
||||
require_pinned_tooling()
|
||||
if args.command == 'repair':
|
||||
if args.pack != 'census':
|
||||
parser.error('the WOF build repairs its own geometry')
|
||||
repair_census()
|
||||
sys.exit(0 if check('census') else 1)
|
||||
sys.exit(0 if check(args.pack) else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -467,7 +467,16 @@
|
||||
"src/data/local_data/natural_earth/regions.json",
|
||||
"src/data/naturalEarthRegions.js",
|
||||
"src/data/bundledJson.js",
|
||||
"src/data/retryableLoad.js"
|
||||
"src/data/retryableLoad.js",
|
||||
"server/providers/common/same-site.js",
|
||||
"server/providers/common/source-root.js",
|
||||
"server/providers/regional/nominatimGate.js",
|
||||
"server/providers/regional/nominatimCache.js",
|
||||
"server/providers/regional/outlineSelect.js",
|
||||
"src/localRequestGate.mjs",
|
||||
"src/data/adminBoundaries.js",
|
||||
"src/data/placeNames.js",
|
||||
"src/sources/featureGeometry.js"
|
||||
],
|
||||
"external": []
|
||||
},
|
||||
@@ -476,6 +485,7 @@
|
||||
"exports": ["./server/providers/openai"],
|
||||
"modules": [
|
||||
"src/i18n/marker.js",
|
||||
"src/i18n/locale.js",
|
||||
"src/hudLayoutPolicy.js",
|
||||
"server/providers/common/rate-limit.js",
|
||||
"server/providers/common/request.js",
|
||||
@@ -528,6 +538,7 @@
|
||||
"src/search/geospatial.js",
|
||||
"src/search/http.js",
|
||||
"src/search/defaults.js",
|
||||
"src/sources/nominatimStatus.js",
|
||||
"src/search/alpsGeocoder.js",
|
||||
"src/search/coordinateGeocoder.js",
|
||||
"src/search/coordinateParser.js",
|
||||
@@ -3165,7 +3176,15 @@
|
||||
"src/ui/kit/UiBadge.vue",
|
||||
"src/ui/stores/globeActions.js",
|
||||
"src/ui/stores/recording.js",
|
||||
"src/ui/i18n.js"
|
||||
"src/ui/i18n.js",
|
||||
"src/annotations/outlineRungs.js",
|
||||
"src/sources/openFreeMapOutlines.js",
|
||||
"src/sources/nominatimOutlines.js",
|
||||
"src/sources/nominatimStatus.js",
|
||||
"src/data/placeBoundaries.js",
|
||||
"src/data/placeNames.js",
|
||||
"src/data/local_data/us_census_places/files.js",
|
||||
"src/data/local_data/wof_neighborhoods/files.js"
|
||||
],
|
||||
"external": [
|
||||
"hls.js",
|
||||
@@ -3765,7 +3784,15 @@
|
||||
"src/layers/wind/inspectionMarker.js",
|
||||
"src/ui/stores/bhoteKoshiPanel.js",
|
||||
"src/hudReadouts.js",
|
||||
"src/ui/i18n.js"
|
||||
"src/ui/i18n.js",
|
||||
"src/annotations/outlineRungs.js",
|
||||
"src/sources/openFreeMapOutlines.js",
|
||||
"src/sources/nominatimOutlines.js",
|
||||
"src/sources/nominatimStatus.js",
|
||||
"src/data/placeBoundaries.js",
|
||||
"src/data/placeNames.js",
|
||||
"src/data/local_data/us_census_places/files.js",
|
||||
"src/data/local_data/wof_neighborhoods/files.js"
|
||||
],
|
||||
"external": [
|
||||
"hls.js",
|
||||
@@ -3838,7 +3865,21 @@
|
||||
"src/sources/featureSource.js",
|
||||
"src/sources/capability.js",
|
||||
"src/data/cameraArrival.js",
|
||||
"src/data/cameraFraming.js"
|
||||
"src/data/cameraFraming.js",
|
||||
"src/sources/openFreeMap.js",
|
||||
"src/sources/vectorTiles.js",
|
||||
"src/sources/phaseTiming.js",
|
||||
"src/sources/militaryTileGeometry.js",
|
||||
"src/data/tomtomTiles.js",
|
||||
"src/data/installationNames.js",
|
||||
"src/annotations/outlineRungs.js",
|
||||
"src/sources/openFreeMapOutlines.js",
|
||||
"src/sources/nominatimOutlines.js",
|
||||
"src/sources/nominatimStatus.js",
|
||||
"src/data/placeBoundaries.js",
|
||||
"src/data/placeNames.js",
|
||||
"src/data/local_data/us_census_places/files.js",
|
||||
"src/data/local_data/wof_neighborhoods/files.js"
|
||||
],
|
||||
"external": [
|
||||
"cesium",
|
||||
|
||||
@@ -0,0 +1,853 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Traffic-warm OpenFreeMap outline acceptance.
|
||||
*
|
||||
* Each sample opens a fresh application page, loads the Texas Capitol z14 tile
|
||||
* through Street Traffic, then circles a building from that same tile. It
|
||||
* records application fetches separately from browser transfer bytes,
|
||||
* body-read/decode/extraction/install measures, and Long Tasks. A second ask
|
||||
* on the page proves the outline source's own consumer-cache path. A separate
|
||||
* page holds traffic's target response while the outline consumer subscribes,
|
||||
* proving that overlap still produces one application fetch and one decode.
|
||||
*
|
||||
* Acceptance records actual owned synchronous segments (16.7 ms each), full
|
||||
* eight-probe completion within 2 s, and separate inclusive latency. No sample may create a
|
||||
* >=50 ms Long Task, and neither the first nor second outline ask may fetch or
|
||||
* decode the traffic-warm tile again. The first explicit cold ask's target
|
||||
* projection must fit one frame; Traffic does no outline projection and the
|
||||
* warm ask reuses it. Traffic activation Long Tasks are reported separately.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const valueAfter = (flag, fallback) => {
|
||||
const index = args.indexOf(flag);
|
||||
return index >= 0 && args[index + 1] ? args[index + 1] : fallback;
|
||||
};
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.log(
|
||||
'Usage: node scripts/qa-openfreemap-same-tile.mjs [--url http://localhost:4173] [--runs 5] [--headful] [--out qa-shots/openfreemap-same-tile.json]',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const APP_URL = valueAfter('--url', 'http://localhost:4173');
|
||||
const RUNS = Number(valueAfter('--runs', '5'));
|
||||
const OUTPUT = path.resolve(
|
||||
valueAfter('--out', 'qa-shots/openfreemap-same-tile.json'),
|
||||
);
|
||||
const FRAME_BUDGET_MS = 16.7;
|
||||
const LONG_TASK_MS = 50;
|
||||
const TARGET = Object.freeze({
|
||||
latitude: 30.27472,
|
||||
longitude: -97.74035,
|
||||
z: 14,
|
||||
x: 3743,
|
||||
y: 6745,
|
||||
});
|
||||
const TARGET_PATH = `/${TARGET.z}/${TARGET.x}/${TARGET.y}.pbf`;
|
||||
|
||||
if (!Number.isInteger(RUNS) || RUNS < 3 || RUNS > 20)
|
||||
throw new Error('--runs must be an integer from 3 through 20');
|
||||
const targetUrl = new URL(APP_URL);
|
||||
if (!['http:', 'https:'].includes(targetUrl.protocol))
|
||||
throw new Error('--url must be HTTP(S)');
|
||||
|
||||
const percentile = (values, p) => {
|
||||
if (!values.length) return null;
|
||||
const sorted = values.slice().sort((a, b) => a - b);
|
||||
return sorted[Math.ceil(p * sorted.length) - 1];
|
||||
};
|
||||
const round = (value) =>
|
||||
value == null ? null : Number(Number(value).toFixed(3));
|
||||
const measures = (entries, name) =>
|
||||
entries.filter((entry) => entry.name === name).map((entry) => entry.ms);
|
||||
const sum = (values) => values.reduce((total, value) => total + value, 0);
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
// Numeric DEV evidence from the actual owners; waited frames are latency, not CPU.
|
||||
function workEvidence(entries, label) {
|
||||
const work = entries.filter(
|
||||
(entry) => entry.name === 'outlines:install-work',
|
||||
);
|
||||
assert.equal(work.length, 1, `${label}: one completed owned-work trace`);
|
||||
const trace = work[0].detail;
|
||||
assert.equal(
|
||||
trace?.status,
|
||||
'installed',
|
||||
`${label}: processed install required`,
|
||||
);
|
||||
assert.equal(trace.samplerStatus, 'installed');
|
||||
assert.equal(trace.rendererStatus, 'installed');
|
||||
assert.equal(trace.dropped, 0, `${label}: no lost work intervals`);
|
||||
assert.equal(trace.probes, 8, `${label}: all original probes required`);
|
||||
assert.ok(Number.isFinite(trace.timeOrigin));
|
||||
assert.ok(
|
||||
Number.isFinite(trace.samplerStart) && Number.isFinite(trace.completedAt),
|
||||
);
|
||||
assert.ok(
|
||||
Number.isFinite(trace.callerStart) &&
|
||||
trace.callerStart <= trace.samplerStart,
|
||||
);
|
||||
const phases = {
|
||||
'engine-schedule': 1,
|
||||
'ground-step': 8,
|
||||
'hybrid-complete': 1,
|
||||
'engine-process': 1,
|
||||
'engine-finalize': 1,
|
||||
'engine-settle': 1,
|
||||
};
|
||||
assert.equal(
|
||||
trace.slices?.length,
|
||||
13,
|
||||
`${label}: complete disjoint owner intervals`,
|
||||
);
|
||||
for (const [phase, count] of Object.entries(phases))
|
||||
assert.equal(
|
||||
trace.slices.filter((slice) => slice.phase === phase).length,
|
||||
count,
|
||||
`${label}: ${phase}`,
|
||||
);
|
||||
let end = -Infinity;
|
||||
const durations = trace.slices.map((slice) => {
|
||||
assert.ok(
|
||||
Number.isFinite(slice.start) &&
|
||||
Number.isFinite(slice.end) &&
|
||||
slice.end >= slice.start,
|
||||
);
|
||||
assert.ok(
|
||||
slice.start >= end,
|
||||
`${label}: no nested or double-counted owner intervals`,
|
||||
);
|
||||
end = slice.end;
|
||||
const duration = slice.end - slice.start;
|
||||
assert.ok(
|
||||
duration <= FRAME_BUDGET_MS,
|
||||
`${label}: ${slice.phase} work slice ${duration} exceeds ${FRAME_BUDGET_MS} ms`,
|
||||
);
|
||||
return duration;
|
||||
});
|
||||
const completionMs = trace.completedAt - trace.samplerStart;
|
||||
assert.ok(
|
||||
completionMs >= 0 && completionMs <= 2000,
|
||||
`${label}: processed completion ${completionMs} ms exceeds 2000 ms`,
|
||||
);
|
||||
// This sum describes owned tail work, never the complete browser task/frame.
|
||||
const tailStart = trace.slices
|
||||
.filter((slice) => slice.phase === 'ground-step')
|
||||
.at(-1).start;
|
||||
return {
|
||||
timeOrigin: trace.timeOrigin,
|
||||
slices: trace.slices,
|
||||
maxSliceMs: Math.max(...durations),
|
||||
ownedTailSumMs: trace.slices
|
||||
.filter((slice) => slice.start >= tailStart)
|
||||
.reduce((total, slice) => total + slice.end - slice.start, 0),
|
||||
completionMs,
|
||||
callerCompletionMs: trace.completedAt - trace.callerStart,
|
||||
probes: trace.probes,
|
||||
dropped: trace.dropped,
|
||||
};
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(OUTPUT), { recursive: true });
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: args.includes('--headful') ? false : 'new',
|
||||
protocolTimeout: 240_000,
|
||||
...(process.env.PUPPETEER_EXECUTABLE_PATH
|
||||
? { executablePath: process.env.PUPPETEER_EXECUTABLE_PATH }
|
||||
: {}),
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--use-gl=angle',
|
||||
...(args.includes('--headful')
|
||||
? []
|
||||
: ['--use-angle=swiftshader', '--enable-unsafe-swiftshader']),
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--window-size=1440,1000',
|
||||
],
|
||||
});
|
||||
|
||||
const report = {
|
||||
appUrl: APP_URL,
|
||||
target: TARGET,
|
||||
targetPath: TARGET_PATH,
|
||||
acceptance: {
|
||||
maxOwnedSliceMs: FRAME_BUDGET_MS,
|
||||
samplerCompletionMaxMs: 2000,
|
||||
legacyInclusive: 'reported-not-gated',
|
||||
maxLongTaskMs: LONG_TASK_MS,
|
||||
coldProjectionMaxMs: FRAME_BUDGET_MS,
|
||||
coldApplicationFetches: 0,
|
||||
coldOutlineTransferBytes: 0,
|
||||
coldDecodes: 0,
|
||||
warmApplicationFetches: 0,
|
||||
warmDecodes: 0,
|
||||
overlapApplicationFetches: 1,
|
||||
overlapDecodes: 1,
|
||||
},
|
||||
renderer: null,
|
||||
samples: [],
|
||||
errors: [],
|
||||
};
|
||||
|
||||
function relevantMeasures(raw) {
|
||||
return raw
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.name.startsWith('roads:') || entry.name.startsWith('outlines:'),
|
||||
)
|
||||
.map((entry) => ({
|
||||
name: entry.name,
|
||||
ms: entry.duration,
|
||||
at: entry.startTime,
|
||||
detail: entry.detail || {},
|
||||
}));
|
||||
}
|
||||
|
||||
async function beginOutline(page, label) {
|
||||
await page.evaluate(
|
||||
({ label, target }) => {
|
||||
performance.clearMeasures();
|
||||
window.__qaOutlineLongTasks.length = 0;
|
||||
window.__qaOutlineStarted = performance.now();
|
||||
window.__qaOutlinePromise = Promise.resolve(
|
||||
window.__gevVoiceCommands.runner('annotate_map', {
|
||||
annotations: [
|
||||
{
|
||||
type: 'area',
|
||||
target: 'this building',
|
||||
label,
|
||||
entityKind: 'building',
|
||||
latitude: target.latitude,
|
||||
longitude: target.longitude,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
},
|
||||
{ label, target: TARGET },
|
||||
);
|
||||
}
|
||||
|
||||
async function settleOutline(page, label) {
|
||||
return page.evaluate(async (label) => {
|
||||
const result = await window.__qaOutlinePromise;
|
||||
let annotation = null;
|
||||
const deadline = performance.now() + 30_000;
|
||||
while (performance.now() < deadline) {
|
||||
annotation = (window.__gevAnnotations?.list?.() || []).find(
|
||||
(item) => item.label === label,
|
||||
);
|
||||
if (annotation && !annotation.pendingOutline) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
}
|
||||
if (!annotation || annotation.pendingOutline)
|
||||
throw new Error(`${label}: outline did not settle`);
|
||||
const { scene } = window.__godsEyeView.viewer;
|
||||
await new Promise((resolve) => {
|
||||
const timer = setTimeout(resolve, 1500);
|
||||
const remove = scene.postRender.addEventListener(() => {
|
||||
clearTimeout(timer);
|
||||
remove();
|
||||
resolve();
|
||||
});
|
||||
scene.requestRender();
|
||||
});
|
||||
await new Promise((resolve) =>
|
||||
requestAnimationFrame(() => requestAnimationFrame(resolve)),
|
||||
);
|
||||
return {
|
||||
commandToRenderMs: performance.now() - window.__qaOutlineStarted,
|
||||
result,
|
||||
annotation: {
|
||||
pending: Boolean(annotation.pendingOutline),
|
||||
unavailable: Boolean(annotation.outlineUnavailable),
|
||||
synthesized: Boolean(annotation.synthesized),
|
||||
footprintKind: annotation.footprintKind || null,
|
||||
ringPoints: annotation.ring?.length || 0,
|
||||
parts: annotation.polygons?.length || (annotation.ring ? 1 : 0),
|
||||
},
|
||||
measures: performance.getEntriesByType('measure').map((entry) => ({
|
||||
name: entry.name,
|
||||
duration: entry.duration,
|
||||
startTime: entry.startTime,
|
||||
detail: entry.detail,
|
||||
})),
|
||||
longTasks: window.__qaOutlineLongTasks.slice(),
|
||||
};
|
||||
}, label);
|
||||
}
|
||||
|
||||
async function driveOutline(page, label) {
|
||||
await beginOutline(page, label);
|
||||
return settleOutline(page, label);
|
||||
}
|
||||
|
||||
try {
|
||||
for (let run = 0; run <= RUNS; run += 1) {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1440, height: 1000 });
|
||||
let overlapMode = false;
|
||||
let overlapApplicationRequests = 0;
|
||||
let heldTargetRequest = null;
|
||||
let resolveHeldTarget;
|
||||
const heldTarget = new Promise((resolve) => {
|
||||
resolveHeldTarget = resolve;
|
||||
});
|
||||
if (run === 0) {
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => {
|
||||
let isTarget = false;
|
||||
try {
|
||||
isTarget = new URL(request.url()).pathname.endsWith(TARGET_PATH);
|
||||
} catch {
|
||||
// Invalid URLs are passed through for the browser to handle.
|
||||
}
|
||||
if (overlapMode && isTarget) {
|
||||
overlapApplicationRequests += 1;
|
||||
if (!heldTargetRequest) {
|
||||
heldTargetRequest = request;
|
||||
resolveHeldTarget();
|
||||
return;
|
||||
}
|
||||
}
|
||||
void request.continue();
|
||||
});
|
||||
}
|
||||
await page.evaluateOnNewDocument(() => {
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed');
|
||||
window.__qaOutlineLongTasks = [];
|
||||
try {
|
||||
new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries())
|
||||
window.__qaOutlineLongTasks.push({
|
||||
at: entry.startTime,
|
||||
ms: entry.duration,
|
||||
});
|
||||
}).observe({ type: 'longtask', buffered: false });
|
||||
window.__qaLongTaskSupported = true;
|
||||
} catch {
|
||||
window.__qaLongTaskSupported = false;
|
||||
}
|
||||
});
|
||||
|
||||
let phase = 'startup';
|
||||
const network = [];
|
||||
const pending = new Map();
|
||||
const client = await page.createCDPSession();
|
||||
await client.send('Network.enable');
|
||||
client.on('Network.requestWillBeSent', (event) => {
|
||||
let url;
|
||||
try {
|
||||
url = new URL(event.request.url);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
url.hostname !== 'tiles.openfreemap.org' ||
|
||||
!url.pathname.endsWith('.pbf')
|
||||
)
|
||||
return;
|
||||
const entry = {
|
||||
requestId: event.requestId,
|
||||
phase,
|
||||
url: url.origin + url.pathname,
|
||||
targetTile: url.pathname.endsWith(TARGET_PATH),
|
||||
cached: false,
|
||||
transferBytes: 0,
|
||||
responseBytes: 0,
|
||||
finished: false,
|
||||
};
|
||||
pending.set(event.requestId, entry);
|
||||
network.push(entry);
|
||||
});
|
||||
client.on('Network.requestServedFromCache', ({ requestId }) => {
|
||||
const entry = pending.get(requestId);
|
||||
if (entry) entry.cached = true;
|
||||
});
|
||||
client.on('Network.responseReceived', ({ requestId, response }) => {
|
||||
const entry = pending.get(requestId);
|
||||
if (!entry) return;
|
||||
entry.status = response.status;
|
||||
entry.cached ||= Boolean(
|
||||
response.fromDiskCache ||
|
||||
response.fromServiceWorker ||
|
||||
response.fromPrefetchCache,
|
||||
);
|
||||
});
|
||||
client.on('Network.dataReceived', ({ requestId, dataLength }) => {
|
||||
const entry = pending.get(requestId);
|
||||
if (entry) entry.responseBytes += dataLength;
|
||||
});
|
||||
client.on('Network.loadingFinished', ({ requestId, encodedDataLength }) => {
|
||||
const entry = pending.get(requestId);
|
||||
if (!entry) return;
|
||||
entry.transferBytes = entry.cached ? 0 : encodedDataLength;
|
||||
entry.finished = true;
|
||||
pending.delete(requestId);
|
||||
});
|
||||
client.on('Network.loadingFailed', ({ requestId, errorText }) => {
|
||||
const entry = pending.get(requestId);
|
||||
if (!entry) return;
|
||||
entry.failure = errorText;
|
||||
entry.finished = true;
|
||||
pending.delete(requestId);
|
||||
});
|
||||
|
||||
const pageErrors = [];
|
||||
page.on('pageerror', (error) => pageErrors.push(String(error.message)));
|
||||
const navigation = new URL(APP_URL);
|
||||
navigation.searchParams.set('welcome', '0');
|
||||
navigation.searchParams.set('trafficDebug', '1');
|
||||
navigation.searchParams.set('trafficRoads', 'osm');
|
||||
await page.goto(navigation.href, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 90_000,
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.viewer &&
|
||||
window.__godsEyeView?.dataManager &&
|
||||
window.__gevVoiceCommands?.runner &&
|
||||
window.__gevAnnotations &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120_000, polling: 250 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.styleManager.setDetection({ enabled: false }),
|
||||
);
|
||||
await page.evaluate(async (target) => {
|
||||
const { viewer, dataManager } = window.__godsEyeView;
|
||||
await dataManager.setEnabled('traffic', false);
|
||||
viewer.camera.cancelFlight();
|
||||
viewer.camera.setView({
|
||||
destination: window.__CESIUM__.Cartesian3.fromDegrees(
|
||||
target.longitude,
|
||||
target.latitude,
|
||||
700,
|
||||
),
|
||||
orientation: {
|
||||
heading: 0,
|
||||
pitch: (-65 * Math.PI) / 180,
|
||||
roll: 0,
|
||||
},
|
||||
});
|
||||
viewer.camera.moveEnd.raiseEvent();
|
||||
window.__gevAnnotations.clear();
|
||||
performance.clearMeasures();
|
||||
}, TARGET);
|
||||
await sleep(750);
|
||||
|
||||
if (run === 0) {
|
||||
phase = 'overlap';
|
||||
overlapMode = true;
|
||||
await page.evaluate(() => {
|
||||
window.__qaOutlineLongTasks.length = 0;
|
||||
performance.clearMeasures();
|
||||
});
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', true),
|
||||
);
|
||||
await Promise.race([
|
||||
heldTarget,
|
||||
sleep(30_000).then(() => {
|
||||
throw new Error('overlap phase did not intercept the target tile');
|
||||
}),
|
||||
]);
|
||||
await beginOutline(page, 'QA Capitol overlap');
|
||||
await sleep(100);
|
||||
await heldTargetRequest.continue();
|
||||
const overlap = await settleOutline(page, 'QA Capitol overlap');
|
||||
await sleep(750);
|
||||
const trafficStats = await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('traffic')
|
||||
.module.getStats(),
|
||||
);
|
||||
assert.equal(trafficStats.error, null, 'overlap traffic source error');
|
||||
overlap.measures = relevantMeasures(overlap.measures);
|
||||
overlap.work = workEvidence(overlap.measures, 'overlap');
|
||||
overlap.network = network.filter(
|
||||
(entry) => entry.phase === 'overlap' && entry.targetTile,
|
||||
);
|
||||
const overlapDecodes = overlap.measures
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.name === 'roads:decode' &&
|
||||
String(entry.detail?.key).endsWith(
|
||||
`:${TARGET.z}/${TARGET.x}/${TARGET.y}`,
|
||||
),
|
||||
)
|
||||
.map((entry) => entry.ms);
|
||||
assert.equal(
|
||||
overlapApplicationRequests,
|
||||
1,
|
||||
'overlapping traffic and outline consumers issue one application request',
|
||||
);
|
||||
assert.equal(
|
||||
overlap.network.length,
|
||||
1,
|
||||
'overlapping traffic and outline consumers emit one target network request',
|
||||
);
|
||||
assert.equal(
|
||||
overlapDecodes.length,
|
||||
1,
|
||||
'overlapping traffic and outline consumers decode once',
|
||||
);
|
||||
assert.equal(overlap.annotation.unavailable, false);
|
||||
assert.equal(overlap.annotation.footprintKind, 'building');
|
||||
report.overlap = {
|
||||
applicationFetches: overlapApplicationRequests,
|
||||
targetNetworkRequests: overlap.network.length,
|
||||
transferBytes: sum(overlap.network.map((entry) => entry.transferBytes)),
|
||||
decodes: overlapDecodes.length,
|
||||
decodeMs: round(sum(overlapDecodes)),
|
||||
longTasks: overlap.longTasks.length,
|
||||
longTaskEntries: overlap.longTasks,
|
||||
work: overlap.work,
|
||||
};
|
||||
await page.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
phase = 'traffic';
|
||||
await page.evaluate(() => {
|
||||
window.__qaOutlineLongTasks.length = 0;
|
||||
performance.clearMeasures();
|
||||
});
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', true),
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const stats = window.__godsEyeView.dataManager.layers
|
||||
.get('traffic')
|
||||
.module.getStats();
|
||||
return stats.count > 0 && !stats.loading && !stats.error;
|
||||
},
|
||||
{ timeout: 180_000, polling: 250 },
|
||||
);
|
||||
await sleep(1500);
|
||||
const trafficMeasures = relevantMeasures(
|
||||
await page.evaluate(() =>
|
||||
performance.getEntriesByType('measure').map((entry) => ({
|
||||
name: entry.name,
|
||||
duration: entry.duration,
|
||||
startTime: entry.startTime,
|
||||
detail: entry.detail,
|
||||
})),
|
||||
),
|
||||
);
|
||||
const trafficLongTasks = await page.evaluate(() =>
|
||||
window.__qaOutlineLongTasks.slice(),
|
||||
);
|
||||
const trafficTarget = network.filter(
|
||||
(entry) => entry.phase === 'traffic' && entry.targetTile,
|
||||
);
|
||||
const targetTile = `${TARGET.z}/${TARGET.x}/${TARGET.y}`;
|
||||
const targetProjectionMeasures = trafficMeasures.filter(
|
||||
(entry) =>
|
||||
entry.detail?.tile === targetTile &&
|
||||
['outlines:street-extract', 'outlines:polygon-extract'].includes(
|
||||
entry.name,
|
||||
),
|
||||
);
|
||||
const targetProjectionMs = sum(
|
||||
targetProjectionMeasures.map((entry) => entry.ms),
|
||||
);
|
||||
assert.ok(
|
||||
trafficTarget.length >= 1,
|
||||
`sample ${run}: traffic did not load ${TARGET_PATH}`,
|
||||
);
|
||||
assert.equal(
|
||||
trafficMeasures.filter((entry) =>
|
||||
['outlines:street-extract', 'outlines:polygon-extract'].includes(
|
||||
entry.name,
|
||||
),
|
||||
).length,
|
||||
0,
|
||||
`sample ${run}: traffic must leave outline projection to the explicit ask`,
|
||||
);
|
||||
|
||||
phase = 'outline-cold';
|
||||
const cold = await driveOutline(page, `QA Capitol cold ${run}`);
|
||||
await sleep(750);
|
||||
cold.measures = relevantMeasures(cold.measures);
|
||||
cold.work = workEvidence(cold.measures, 'cold');
|
||||
cold.network = network.filter(
|
||||
(entry) => entry.phase === 'outline-cold' && entry.targetTile,
|
||||
);
|
||||
cold.longTasks = cold.longTasks.map((entry) => ({
|
||||
at: round(entry.at),
|
||||
ms: round(entry.ms),
|
||||
}));
|
||||
|
||||
await page.evaluate(() => window.__gevAnnotations.clear());
|
||||
phase = 'outline-warm';
|
||||
const warm = await driveOutline(page, `QA Capitol warm ${run}`);
|
||||
await sleep(750);
|
||||
warm.measures = relevantMeasures(warm.measures);
|
||||
warm.work = workEvidence(warm.measures, 'warm');
|
||||
warm.network = network.filter(
|
||||
(entry) => entry.phase === 'outline-warm' && entry.targetTile,
|
||||
);
|
||||
warm.longTasks = warm.longTasks.map((entry) => ({
|
||||
at: round(entry.at),
|
||||
ms: round(entry.ms),
|
||||
}));
|
||||
|
||||
const supported = await page.evaluate(() => window.__qaLongTaskSupported);
|
||||
assert.equal(supported, true, `sample ${run}: Long Tasks unsupported`);
|
||||
assert.deepEqual(pageErrors, [], `sample ${run}: browser errors`);
|
||||
assert.equal(cold.annotation.unavailable, false);
|
||||
assert.equal(cold.annotation.synthesized, false);
|
||||
assert.equal(cold.annotation.footprintKind, 'building');
|
||||
assert.ok(cold.annotation.ringPoints >= 4);
|
||||
assert.equal(warm.annotation.footprintKind, 'building');
|
||||
assert.ok(warm.annotation.ringPoints >= 4);
|
||||
|
||||
const coldDecodes = measures(cold.measures, 'roads:decode');
|
||||
const coldInstalls = measures(cold.measures, 'outlines:install');
|
||||
const warmDecodes = measures(warm.measures, 'roads:decode');
|
||||
const coldProjectionMeasures = cold.measures.filter(
|
||||
(entry) =>
|
||||
entry.detail?.tile === targetTile &&
|
||||
['outlines:street-extract', 'outlines:polygon-extract'].includes(
|
||||
entry.name,
|
||||
),
|
||||
);
|
||||
cold.targetProjectionMs = sum(
|
||||
coldProjectionMeasures.map((entry) => entry.ms),
|
||||
);
|
||||
assert.equal(
|
||||
cold.network.length,
|
||||
0,
|
||||
'shared decoded tile performs no second application fetch',
|
||||
);
|
||||
assert.equal(sum(cold.network.map((entry) => entry.transferBytes)), 0);
|
||||
assert.equal(
|
||||
coldDecodes.length,
|
||||
0,
|
||||
'shared decoded tile is not parsed again',
|
||||
);
|
||||
assert.equal(
|
||||
coldProjectionMeasures.filter(
|
||||
(entry) => entry.name === 'outlines:street-extract',
|
||||
).length,
|
||||
1,
|
||||
'the first explicit cold ask owns target street extraction exactly once',
|
||||
);
|
||||
assert.equal(
|
||||
coldProjectionMeasures.filter(
|
||||
(entry) => entry.name === 'outlines:polygon-extract',
|
||||
).length,
|
||||
1,
|
||||
'the first explicit cold ask owns target polygon extraction exactly once',
|
||||
);
|
||||
assert.ok(
|
||||
cold.targetProjectionMs <= FRAME_BUDGET_MS,
|
||||
`sample ${run}: cold target projection ${cold.targetProjectionMs} ms exceeds ${FRAME_BUDGET_MS} ms`,
|
||||
);
|
||||
assert.equal(
|
||||
measures(warm.measures, 'outlines:street-extract').length,
|
||||
0,
|
||||
'warm reuse performs no street extraction',
|
||||
);
|
||||
assert.equal(
|
||||
measures(warm.measures, 'outlines:polygon-extract').length,
|
||||
0,
|
||||
'warm reuse performs no polygon extraction',
|
||||
);
|
||||
assert.ok(coldInstalls.length >= 1, 'cold outline installs geometry');
|
||||
assert.equal(warm.network.length, 0, 'decoded-cache hit performs no fetch');
|
||||
assert.equal(warmDecodes.length, 0, 'decoded-cache hit performs no decode');
|
||||
|
||||
report.renderer ||= await page.evaluate(() => {
|
||||
const gl = window.__godsEyeView.viewer.scene.context._gl;
|
||||
const debug = gl.getExtension('WEBGL_debug_renderer_info');
|
||||
return debug
|
||||
? gl.getParameter(debug.UNMASKED_RENDERER_WEBGL)
|
||||
: gl.getParameter(gl.RENDERER);
|
||||
});
|
||||
if (args.includes('--headful')) {
|
||||
assert.doesNotMatch(
|
||||
String(report.renderer),
|
||||
/swiftshader|software rasterizer|llvmpipe/i,
|
||||
`sample ${run}: headful evidence requires a hardware renderer`,
|
||||
);
|
||||
}
|
||||
report.samples.push({
|
||||
run,
|
||||
pageErrors,
|
||||
traffic: {
|
||||
applicationFetches: trafficTarget.length,
|
||||
transferBytes: sum(trafficTarget.map((entry) => entry.transferBytes)),
|
||||
decodes: measures(trafficMeasures, 'roads:decode').length,
|
||||
targetProjectionMs: round(targetProjectionMs),
|
||||
longTasks: trafficLongTasks.map((entry) => ({
|
||||
at: round(entry.at),
|
||||
ms: round(entry.ms),
|
||||
})),
|
||||
measures: trafficMeasures,
|
||||
network: trafficTarget,
|
||||
},
|
||||
cold: {
|
||||
...cold,
|
||||
applicationFetches: cold.network.length,
|
||||
transferBytes: sum(cold.network.map((entry) => entry.transferBytes)),
|
||||
decodeMs: round(sum(coldDecodes)),
|
||||
bodyReadMs: round(sum(measures(cold.measures, 'roads:tile-body-read'))),
|
||||
streetExtractMs: round(
|
||||
sum(measures(cold.measures, 'outlines:street-extract')),
|
||||
),
|
||||
polygonExtractMs: round(
|
||||
sum(measures(cold.measures, 'outlines:polygon-extract')),
|
||||
),
|
||||
installMs: round(sum(coldInstalls)),
|
||||
legacyInclusiveMs: round(
|
||||
sum(coldDecodes) +
|
||||
sum(measures(cold.measures, 'outlines:street-extract')) +
|
||||
sum(measures(cold.measures, 'outlines:polygon-extract')) +
|
||||
sum(coldInstalls),
|
||||
),
|
||||
},
|
||||
warm: {
|
||||
...warm,
|
||||
applicationFetches: warm.network.length,
|
||||
transferBytes: sum(warm.network.map((entry) => entry.transferBytes)),
|
||||
decodeMs: round(sum(warmDecodes)),
|
||||
installMs: round(sum(measures(warm.measures, 'outlines:install'))),
|
||||
},
|
||||
});
|
||||
await page.close();
|
||||
}
|
||||
|
||||
const coldInclusive = report.samples.map(
|
||||
(sample) => sample.cold.legacyInclusiveMs,
|
||||
);
|
||||
const outlineLongTasks = report.samples.flatMap((sample) => [
|
||||
...sample.cold.longTasks,
|
||||
...sample.warm.longTasks,
|
||||
]);
|
||||
const trafficLongTasks = report.samples.flatMap(
|
||||
(sample) => sample.traffic.longTasks,
|
||||
);
|
||||
report.summary = {
|
||||
samples: report.samples.length,
|
||||
overlapApplicationFetches: report.overlap?.applicationFetches ?? null,
|
||||
overlapDecodes: report.overlap?.decodes ?? null,
|
||||
legacyInclusiveP50Ms: round(percentile(coldInclusive, 0.5)),
|
||||
legacyInclusiveP95Ms: round(percentile(coldInclusive, 0.95)),
|
||||
legacyInclusiveMaxMs: round(Math.max(...coldInclusive)),
|
||||
coldCommandToRenderP95Ms: round(
|
||||
percentile(
|
||||
report.samples.map((sample) => sample.cold.commandToRenderMs),
|
||||
0.95,
|
||||
),
|
||||
),
|
||||
coldApplicationFetches: sum(
|
||||
report.samples.map((sample) => sample.cold.applicationFetches),
|
||||
),
|
||||
coldTransferBytes: sum(
|
||||
report.samples.map((sample) => sample.cold.transferBytes),
|
||||
),
|
||||
coldDecodes: sum(
|
||||
report.samples.map(
|
||||
(sample) => measures(sample.cold.measures, 'roads:decode').length,
|
||||
),
|
||||
),
|
||||
warmApplicationFetches: sum(
|
||||
report.samples.map((sample) => sample.warm.applicationFetches),
|
||||
),
|
||||
warmDecodes: sum(
|
||||
report.samples.map(
|
||||
(sample) => measures(sample.warm.measures, 'roads:decode').length,
|
||||
),
|
||||
),
|
||||
coldProjectionMaxMs: Math.max(
|
||||
...report.samples.map((sample) => sample.cold.targetProjectionMs),
|
||||
),
|
||||
trafficBaselineLongTasks: trafficLongTasks.length,
|
||||
trafficBaselineMaxLongTaskMs: round(
|
||||
trafficLongTasks.length
|
||||
? Math.max(...trafficLongTasks.map((entry) => entry.ms))
|
||||
: 0,
|
||||
),
|
||||
longTasks: outlineLongTasks.length,
|
||||
maxLongTaskMs: round(
|
||||
outlineLongTasks.length
|
||||
? Math.max(...outlineLongTasks.map((entry) => entry.ms))
|
||||
: 0,
|
||||
),
|
||||
};
|
||||
const coldSlices = report.samples.map(
|
||||
(sample) => sample.cold.work.maxSliceMs,
|
||||
);
|
||||
report.summary.coldOwnedSliceP95Ms = round(percentile(coldSlices, 0.95));
|
||||
report.summary.maxOwnedSliceMs = Math.max(
|
||||
report.overlap.work.maxSliceMs,
|
||||
...report.samples.flatMap((sample) => [
|
||||
sample.cold.work.maxSliceMs,
|
||||
sample.warm.work.maxSliceMs,
|
||||
]),
|
||||
);
|
||||
report.summary.maxSamplerCompletionMs = Math.max(
|
||||
report.overlap.work.completionMs,
|
||||
...report.samples.flatMap((sample) => [
|
||||
sample.cold.work.completionMs,
|
||||
sample.warm.work.completionMs,
|
||||
]),
|
||||
);
|
||||
assert.ok(report.summary.maxOwnedSliceMs <= FRAME_BUDGET_MS);
|
||||
assert.ok(report.summary.maxSamplerCompletionMs <= 2000);
|
||||
assert.ok(
|
||||
report.summary.coldProjectionMaxMs <= FRAME_BUDGET_MS,
|
||||
'cold outline projection exceeded one frame',
|
||||
);
|
||||
assert.equal(report.summary.longTasks, 0, 'outline ask created a Long Task');
|
||||
assert.equal(report.summary.coldTransferBytes, 0);
|
||||
assert.equal(report.summary.coldApplicationFetches, 0);
|
||||
assert.equal(report.summary.coldDecodes, 0);
|
||||
assert.equal(report.summary.warmApplicationFetches, 0);
|
||||
assert.equal(report.summary.warmDecodes, 0);
|
||||
assert.equal(report.summary.overlapApplicationFetches, 1);
|
||||
assert.equal(report.summary.overlapDecodes, 1);
|
||||
report.passed = true;
|
||||
console.table(
|
||||
report.samples.map((sample) => ({
|
||||
run: sample.run,
|
||||
coldProjectionMs: sample.cold.targetProjectionMs,
|
||||
appFetches: sample.cold.applicationFetches,
|
||||
transferBytes: sample.cold.transferBytes,
|
||||
bodyReadMs: sample.cold.bodyReadMs,
|
||||
decodeMs: sample.cold.decodeMs,
|
||||
streetMs: sample.cold.streetExtractMs,
|
||||
polygonMs: sample.cold.polygonExtractMs,
|
||||
installMs: sample.cold.installMs,
|
||||
legacyInclusiveMs: sample.cold.legacyInclusiveMs,
|
||||
maxOwnedSliceMs: sample.cold.work.maxSliceMs,
|
||||
renderMs: round(sample.cold.commandToRenderMs),
|
||||
longTasks: sample.cold.longTasks.length,
|
||||
})),
|
||||
);
|
||||
console.log('OpenFreeMap same-tile acceptance passed:', report.summary);
|
||||
} catch (error) {
|
||||
report.passed = false;
|
||||
report.failure = String(error?.stack || error);
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await fs.writeFile(OUTPUT, `${JSON.stringify(report, null, 2)}\n`);
|
||||
await browser.close();
|
||||
}
|
||||
+20
-3
@@ -3073,20 +3073,27 @@ async function main() {
|
||||
expectedSlot: targetIndex,
|
||||
previewPlayDelta: previewPlays - playsBefore,
|
||||
commitPlayDelta: window.__qaRadioPlayCalls.length - playsBefore,
|
||||
audioState: state.audioState,
|
||||
tuningStatic: state.tuningStatic,
|
||||
samples,
|
||||
spread: Math.max(...samples) - Math.min(...samples),
|
||||
};
|
||||
});
|
||||
check(
|
||||
'pointer preview never plays and release commits the nearest station once without a needle sweep',
|
||||
'from a stopped Radio, pointer release only selects the nearest station, silently and without a needle sweep',
|
||||
tunerDirectRelease.targetId
|
||||
&& tunerDirectRelease.selectedId === tunerDirectRelease.targetId
|
||||
&& tunerDirectRelease.sliderSlot === tunerDirectRelease.expectedSlot
|
||||
&& tunerDirectRelease.previewPlayDelta === 0
|
||||
&& tunerDirectRelease.commitPlayDelta === 1
|
||||
&& tunerDirectRelease.commitPlayDelta === 0
|
||||
&& tunerDirectRelease.audioState === 'stopped'
|
||||
&& !tunerDirectRelease.tuningStatic
|
||||
&& tunerDirectRelease.spread < 0.5,
|
||||
JSON.stringify(tunerDirectRelease),
|
||||
);
|
||||
// The release plays only if a stream was active when the drag began.
|
||||
await page.evaluate(() => window.__godsEyeView.dataManager.layers.get('radio').module.togglePlayback({ origin: 'user' }));
|
||||
await page.waitForFunction(() => window.__godsEyeView.dataManager.layers.get('radio').module.getUIState().audioState === 'playing');
|
||||
// Earlier cases scroll and resize the directory. Bring the physical drag
|
||||
// target back into its scroller before deriving viewport mouse coordinates.
|
||||
await page.$eval('#radio-tuner-slider', (slider) => slider.scrollIntoView({ block: 'center', inline: 'nearest' }));
|
||||
@@ -3494,6 +3501,9 @@ async function main() {
|
||||
});
|
||||
});
|
||||
await sleep(500);
|
||||
// Next left the Radio stopped; a release plays only from an active stream.
|
||||
await page.evaluate(() => window.__godsEyeView.dataManager.layers.get('radio').module.togglePlayback({ origin: 'user' }));
|
||||
await page.waitForFunction(() => window.__godsEyeView.dataManager.layers.get('radio').module.getUIState().audioState === 'playing');
|
||||
const failedTuner = await page.evaluate(async () => {
|
||||
const gev = window.__godsEyeView;
|
||||
const slider = document.getElementById('radio-tuner-slider');
|
||||
@@ -3692,7 +3702,14 @@ async function main() {
|
||||
radio.stopPlayback();
|
||||
Array.from({ length: gev.viewer.dataSources.length }, (_, index) => gev.viewer.dataSources.get(index))
|
||||
.find((item) => item.name === 'Radio stations').clustering.enabled = true;
|
||||
radio.selectStation('00000000-0000-4000-8000-000000000001', { autoplay: false, focus: false });
|
||||
// Next plays only while a stream is active, so start from a playing station.
|
||||
radio.selectStation('00000000-0000-4000-8000-000000000001', { autoplay: true, focus: false });
|
||||
});
|
||||
await page.waitForFunction(() => window.__godsEyeView.dataManager.layers.get('radio').module.getUIState().audioState === 'playing');
|
||||
await page.evaluate(() => {
|
||||
const gev = window.__godsEyeView;
|
||||
const camera = gev.viewer.camera;
|
||||
const radio = gev.dataManager.layers.get('radio').module;
|
||||
const pool = gev.styleManager._radioControls._radioTunerPool;
|
||||
const selectedIndex = pool.findIndex((station) => station.id === radio.getUIState().selected?.id);
|
||||
window.__qaRadioExpectedPrimary = pool[(selectedIndex + 1) % pool.length];
|
||||
|
||||
@@ -23,6 +23,9 @@
|
||||
* node scripts/qa-voice-routing.mjs # both layers, :4415
|
||||
* node scripts/qa-voice-routing.mjs --layer routing --budget 120
|
||||
* node scripts/qa-voice-routing.mjs --layer behavior --url http://localhost:4173
|
||||
* node scripts/qa-voice-routing.mjs --layer behavior --behavior outlines
|
||||
* node scripts/qa-voice-routing.mjs --layer behavior --behavior acceptance [--headful] [--shots qa-shots/outlines]
|
||||
* # operator-only: may spend the configured guarded Nominatim allowance
|
||||
* node scripts/qa-voice-routing.mjs --layer behavior --deixis-only --headful --shots qa-shots/point-and-ask
|
||||
* node scripts/qa-voice-routing.mjs --layer deixis --budget 12 # pointer_context routing (model turns)
|
||||
* node scripts/qa-voice-routing.mjs --layer deixis-live --url ... # in-app text turns (model turns)
|
||||
@@ -72,6 +75,7 @@ const LAYER = getOpt('--layer', 'all'); // routing | behavior | all | deixis | d
|
||||
const TURN_BUDGET = Number(getOpt('--budget', '120'));
|
||||
const PHRASES_PER_SESSION = Number(getOpt('--batch', '6'));
|
||||
const ONLY = getOpt('--only', null); // substring filter on phrase text
|
||||
const BEHAVIOR = getOpt('--behavior', 'all'); // all | outlines | acceptance
|
||||
const DEIXIS_ONLY = process.argv.includes('--deixis-only'); // behavior: point-and-ask block only
|
||||
const HEADFUL = process.argv.includes('--headful'); // real GPU, visible window
|
||||
const SHOT_DIR = getOpt('--shots', null); // point-and-ask evidence screenshots
|
||||
@@ -1325,7 +1329,10 @@ async function runBehaviorLayer() {
|
||||
// Overpass queries through the app and any direct request to an Overpass
|
||||
// host (the status probe is not a query).
|
||||
const overpassRequests = [];
|
||||
const packTimes = [];
|
||||
const requestStarted = new Map();
|
||||
page.on('request', (request) => {
|
||||
requestStarted.set(request, Date.now());
|
||||
const target = new URL(request.url());
|
||||
const viaApp =
|
||||
target.origin === new URL(APP_URL).origin &&
|
||||
@@ -1336,12 +1343,26 @@ async function runBehaviorLayer() {
|
||||
`${request.method()} ${target.origin}${target.pathname}`,
|
||||
);
|
||||
});
|
||||
page.on('requestfinished', (request) => {
|
||||
if (/us_census_places|wof_neighborhoods/.test(request.url()))
|
||||
packTimes.push({
|
||||
file: request.url().split('/').pop().split('?')[0],
|
||||
ms: Date.now() - (requestStarted.get(request) ?? Date.now()),
|
||||
});
|
||||
requestStarted.delete(request);
|
||||
});
|
||||
const resolverLog = [];
|
||||
page.on('console', (message) => {
|
||||
const text = message.text();
|
||||
if (text.startsWith('[Resolver]')) resolverLog.push(text);
|
||||
});
|
||||
|
||||
try {
|
||||
// The point-and-ask block skips the first-run launcher so its evidence
|
||||
// screenshots show the map; the full run keeps the historical URL.
|
||||
const behaviorUrl = new URL(APP_URL);
|
||||
if (DEIXIS_ONLY) behaviorUrl.searchParams.set('welcome', '0');
|
||||
if (DEIXIS_ONLY || BEHAVIOR === 'acceptance')
|
||||
behaviorUrl.searchParams.set('welcome', '0');
|
||||
await page.goto(behaviorUrl.href, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 60000,
|
||||
@@ -1384,6 +1405,22 @@ async function runBehaviorLayer() {
|
||||
});
|
||||
const settle = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
if (BEHAVIOR === 'outlines') {
|
||||
await runBundledOutlineScenarios({
|
||||
page,
|
||||
run,
|
||||
settle,
|
||||
overpassRequests,
|
||||
resolverLog,
|
||||
packTimes,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (BEHAVIOR === 'acceptance') {
|
||||
await runOutlineAcceptance({ page, run, settle, overpassRequests });
|
||||
return;
|
||||
}
|
||||
|
||||
// (1) fly_to_location lands near the target. Annotations follow IMMEDIATELY
|
||||
// while the camera is local — the resolver's proximity gate rejects far
|
||||
// targets BY DESIGN (annotating Austin from over the Alps must fail).
|
||||
@@ -2295,6 +2332,15 @@ async function runBehaviorLayer() {
|
||||
`ok=${gated?.ok} error=${String(gated?.error || '').slice(0, 90)} active=${cockpitAfterGate}`,
|
||||
);
|
||||
|
||||
await runBundledOutlineScenarios({
|
||||
page,
|
||||
run,
|
||||
settle,
|
||||
overpassRequests,
|
||||
resolverLog,
|
||||
packTimes,
|
||||
});
|
||||
|
||||
const shotDir = path.join(ROOT, 'qa-shots');
|
||||
fs.mkdirSync(shotDir, { recursive: true });
|
||||
await page.screenshot({
|
||||
@@ -2318,6 +2364,259 @@ async function runBehaviorLayer() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Area asks that the bundled packs answer with no boundary lookup: a US
|
||||
* Census place, the San Francisco DataSF pack, and Who's On First
|
||||
* neighborhoods. Each teleports the camera over the place, draws the area,
|
||||
* and checks the outline, the pack that answered, and that no request went
|
||||
* to Overpass.
|
||||
*/
|
||||
const BUNDLED_OUTLINES = [
|
||||
{ target: 'Austin', lat: 30.2672, lon: -97.7431, pack: 'us-census' },
|
||||
{ target: 'the Mission District', lat: 37.7599, lon: -122.4148, pack: 'datasf', entityKind: 'district' },
|
||||
{ target: 'Notting Hill', lat: 51.5118, lon: -0.2046, pack: 'wof' },
|
||||
{ target: 'Williamsburg', lat: 40.7081, lon: -73.9571, pack: 'wof' },
|
||||
{ target: 'Le Marais', lat: 48.8575, lon: 2.358, pack: 'wof' },
|
||||
];
|
||||
|
||||
async function runBundledOutlineScenarios({
|
||||
page,
|
||||
run,
|
||||
settle,
|
||||
overpassRequests,
|
||||
resolverLog,
|
||||
packTimes,
|
||||
}) {
|
||||
const timings = [];
|
||||
for (const scenario of BUNDLED_OUTLINES) {
|
||||
const label = `behavior: "outline ${scenario.target}" draws the bundled ${scenario.pack} outline`;
|
||||
await page.evaluate(({ lat, lon }) => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
viewer.camera.cancelFlight();
|
||||
viewer.camera.setView({
|
||||
destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({
|
||||
longitude: (lon * Math.PI) / 180,
|
||||
latitude: (lat * Math.PI) / 180,
|
||||
height: 9000,
|
||||
}),
|
||||
});
|
||||
}, scenario);
|
||||
await run('clear_annotations', {});
|
||||
await settle(1500);
|
||||
const overpassBefore = overpassRequests.length;
|
||||
const logBefore = resolverLog.length;
|
||||
const packsBefore = packTimes.length;
|
||||
const started = Date.now();
|
||||
const r = await run('annotate_map', {
|
||||
annotations: [{
|
||||
type: 'area',
|
||||
target: scenario.target,
|
||||
label: scenario.target,
|
||||
...(scenario.entityKind ? { entityKind: scenario.entityKind } : {}),
|
||||
}],
|
||||
});
|
||||
let ringPoints = 0;
|
||||
for (let i = 0; i < 40 && ringPoints < 4; i += 1) {
|
||||
ringPoints = await page.evaluate((target) => {
|
||||
const items = window.__gevAnnotations?.list?.() || [];
|
||||
const item = items.find((it) => it.label === target);
|
||||
return Array.isArray(item?.ring) ? item.ring.length : 0;
|
||||
}, scenario.target);
|
||||
if (ringPoints < 4) await settle(250);
|
||||
}
|
||||
const ms = Date.now() - started;
|
||||
const line = resolverLog.slice(logBefore).find((l) => l.includes('bundled')) || '';
|
||||
const pack = /\((us-census|datasf|wof|natural-earth)[,)]/.exec(line)?.[1] || 'none';
|
||||
const packFiles = packTimes.slice(packsBefore);
|
||||
const overpass = overpassRequests.slice(overpassBefore);
|
||||
timings.push({ target: scenario.target, ms, pack, overpass: overpass.length });
|
||||
report((r?.drawn ?? 0) >= 1 && ringPoints >= 4 && pack === scenario.pack && overpass.length === 0,
|
||||
label,
|
||||
`drawn=${r?.drawn} ring=${ringPoints} pack=${pack} outlineMs=${ms} overpass=${overpass.length} ` +
|
||||
`packFiles=${packFiles.map((f) => `${f.file}:${f.ms}ms`).join(',') || 'cached'}`);
|
||||
}
|
||||
await run('clear_annotations', {});
|
||||
report(
|
||||
overpassRequests.length === 0,
|
||||
'behavior: no request reached Overpass during the run',
|
||||
`overpass=${overpassRequests.length} ${overpassRequests.slice(0, 2).join(' ')}`,
|
||||
);
|
||||
console.log(` bundled outline timings: ${JSON.stringify(timings)}`);
|
||||
console.log(` pack files loaded: ${packTimes.map((f) => `${f.file}:${f.ms}ms`).join(' ')}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Outline acceptance across six countries: each ask teleports the camera
|
||||
* over the place, draws the area once cold and once warm (after clearing),
|
||||
* and records the rung that answered, the parts drawn, the latency, and how
|
||||
* many public Nominatim requests the server spent (its daily count in
|
||||
* .gev-cache/nominatim/state.json). Nothing may reach Overpass.
|
||||
*/
|
||||
const OUTLINE_ASKS = [
|
||||
// group, target, camera lat/lon/height(m), extra annotation args, shot name
|
||||
['country', 'Switzerland', 46.8, 8.2, 1_200_000],
|
||||
['country', 'Iran', 32.4, 53.7, 3_500_000],
|
||||
['state', 'Texas', 31.0, -99.0, 2_000_000],
|
||||
['state', 'Bavaria', 48.9, 11.4, 900_000],
|
||||
['city', 'Austin', 30.2672, -97.7431, 60_000, {}, 'austin-city'],
|
||||
['city', 'Paris', 48.8566, 2.3522, 40_000],
|
||||
['city', 'Tokyo', 35.6762, 139.6503, 90_000],
|
||||
['city', 'São Paulo', -23.5505, -46.6333, 90_000],
|
||||
['city', 'Mumbai', 19.076, 72.8777, 60_000],
|
||||
['city', 'Lagos', 6.5244, 3.3792, 60_000],
|
||||
['neighborhood', 'the Mission District', 37.7599, -122.4148, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'the Marina', 37.8037, -122.4368, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'Notting Hill', 51.5118, -0.2046, 8000, { entityKind: 'district' }, 'notting-hill'],
|
||||
['neighborhood', 'Williamsburg', 40.7081, -73.9571, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'SoHo', 40.7233, -74.003, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'Le Marais', 48.8575, 2.358, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'Shibuya', 35.6595, 139.7005, 8000, { entityKind: 'district' }],
|
||||
['landmark', 'the Texas Capitol grounds', 30.2747, -97.7404, 3000, { entityKind: 'compound' }],
|
||||
['landmark', 'Central Park', 40.7829, -73.9654, 6000, {}, 'central-park'],
|
||||
['landmark', 'the Eiffel Tower', 48.8584, 2.2945, 3000],
|
||||
['landmark', 'Stanford', 37.4275, -122.1697, 8000],
|
||||
['landmark', 'Stanford University', 37.4275, -122.1697, 8000],
|
||||
['landmark', 'Lady Bird Lake', 30.2638, -97.7526, 6000],
|
||||
['street', 'Congress Avenue', 30.2672, -97.7425, 3000, { entityKind: 'street' }, 'congress-avenue'],
|
||||
['street', 'Lombard Street', 37.8021, -122.4187, 3000, { entityKind: 'street' }],
|
||||
['building', 'this building', 30.27472, -97.74035, 1200,
|
||||
{ entityKind: 'building', latitude: 30.27472, longitude: -97.74035 }, 'building'],
|
||||
].map(([group, target, lat, lon, height, extra = {}, shot = null]) => ({ group, target, lat, lon, height, extra, shot }));
|
||||
|
||||
const NOMINATIM_STATE = path.join(ROOT, '.gev-cache', 'nominatim', 'state.json');
|
||||
function nominatimCount() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(NOMINATIM_STATE, 'utf8')).count || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function runOutlineAcceptance({
|
||||
page,
|
||||
run,
|
||||
settle,
|
||||
overpassRequests,
|
||||
}) {
|
||||
const trace = [];
|
||||
page.on('console', (msg) => {
|
||||
const text = msg.text();
|
||||
if (/^\[(Outline|Resolver)\]/.test(text)) trace.push(text);
|
||||
});
|
||||
const nominatimStart = nominatimCount();
|
||||
const overpassStart = overpassRequests.length;
|
||||
if (SHOT_DIR) fs.mkdirSync(path.resolve(ROOT, SHOT_DIR), { recursive: true });
|
||||
|
||||
const teleport = (ask) => page.evaluate(({ lat, lon, height }) => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
viewer.camera.cancelFlight();
|
||||
viewer.camera.setView({
|
||||
destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({
|
||||
longitude: (lon * Math.PI) / 180,
|
||||
latitude: (lat * Math.PI) / 180,
|
||||
height,
|
||||
}),
|
||||
});
|
||||
}, ask);
|
||||
|
||||
const rungOf = (lines, state) => {
|
||||
for (const line of lines) {
|
||||
const m = /→ (openfreemap|nominatim)\b/.exec(line) ||
|
||||
/bundled → ([a-z-]+)/.exec(line) ||
|
||||
/\((us-census|datasf|wof|natural-earth),/.exec(line);
|
||||
if (m) return m[1];
|
||||
}
|
||||
if (state?.synthesized) return 'approximate';
|
||||
return state?.ring >= 4 ? 'other' : 'point';
|
||||
};
|
||||
|
||||
const once = async (ask) => {
|
||||
await run('clear_annotations', {});
|
||||
await teleport(ask);
|
||||
await settle(1200);
|
||||
trace.length = 0;
|
||||
const before = nominatimCount();
|
||||
const label = `QA ${ask.target}`;
|
||||
const started = Date.now();
|
||||
const r = await run('annotate_map', {
|
||||
annotations: [{ type: 'area', target: ask.target, label, ...ask.extra }],
|
||||
});
|
||||
let state = null;
|
||||
for (let i = 0; i < 120; i += 1) {
|
||||
state = await page.evaluate((l) => {
|
||||
const it = (window.__gevAnnotations?.list?.() || []).find((a) => a.label === l);
|
||||
if (!it) return null;
|
||||
return {
|
||||
ring: Array.isArray(it.ring) ? it.ring.length : 0,
|
||||
parts: Array.isArray(it.polygons) && it.polygons.length ? it.polygons.length : (it.ring ? 1 : 0),
|
||||
synthesized: Boolean(it.synthesized),
|
||||
unavailable: Boolean(it.outlineUnavailable),
|
||||
pending: Boolean(it.pendingOutline),
|
||||
};
|
||||
}, label);
|
||||
if (state && !state.pending) break;
|
||||
if (!state && r?.drawn === 0 && i > 8) break;
|
||||
await settle(250);
|
||||
}
|
||||
const ms = Date.now() - started;
|
||||
return {
|
||||
ms,
|
||||
drawn: r?.drawn ?? 0,
|
||||
state,
|
||||
rung: rungOf(trace, state),
|
||||
nominatim: nominatimCount() - before,
|
||||
error: r?.error || r?.results?.find?.((x) => x?.error)?.error || null,
|
||||
};
|
||||
};
|
||||
|
||||
const table = [];
|
||||
// --only "Paris|Stanford" reruns a few rows.
|
||||
const only = ONLY ? ONLY.split('|') : null;
|
||||
for (const ask of OUTLINE_ASKS.filter((a) => !only || only.some((o) => a.target.includes(o)))) {
|
||||
const cold = await once(ask);
|
||||
if (SHOT_DIR && ask.shot) {
|
||||
await settle(HEADFUL ? 6000 : 2500);
|
||||
await page.screenshot({ path: path.resolve(ROOT, SHOT_DIR, `${ask.shot}.png`) });
|
||||
}
|
||||
const warm = await once(ask);
|
||||
const rendered = Boolean(cold.state?.ring >= 4) && !cold.state?.synthesized;
|
||||
const row = {
|
||||
group: ask.group,
|
||||
ask: ask.target,
|
||||
rung: cold.rung,
|
||||
rendered,
|
||||
approximate: Boolean(cold.state?.synthesized),
|
||||
parts: cold.state?.parts ?? 0,
|
||||
coldMs: cold.ms,
|
||||
warmMs: warm.ms,
|
||||
warmRung: warm.rung,
|
||||
nominatim: cold.nominatim + warm.nominatim,
|
||||
};
|
||||
table.push(row);
|
||||
const detail = `rung=${row.rung} parts=${row.parts} cold=${row.coldMs}ms warm=${row.warmMs}ms ` +
|
||||
`nominatim=${row.nominatim}${row.approximate ? ' approximate' : ''}` +
|
||||
(rendered ? '' : ` trace=${trace.slice(-2).join(' | ').slice(0, 300)}`);
|
||||
if (rendered) report(true, `outlines: ${ask.group} "${ask.target}"`, detail);
|
||||
else skipped(`outlines: ${ask.group} "${ask.target}" (no exact outline)`, detail);
|
||||
}
|
||||
await run('clear_annotations', {});
|
||||
|
||||
const overpass = overpassRequests.slice(overpassStart);
|
||||
report(overpass.length === 0, 'outlines: no request reached Overpass during the run',
|
||||
`overpass=${overpass.length} ${overpass.slice(0, 2).join(' ')}`);
|
||||
const used = nominatimCount() - nominatimStart;
|
||||
report(used <= 30, 'outlines: public Nominatim requests within the run budget (30)', `used=${used}`);
|
||||
|
||||
console.log('\n | Group | Ask | Rung | Outline | Parts | Cold ms | Warm ms | Nominatim |');
|
||||
console.log(' |---|---|---|---|---|---|---|---|');
|
||||
for (const row of table) {
|
||||
const outline = row.rendered ? 'yes' : row.approximate ? 'approximate' : 'no (point)';
|
||||
console.log(` | ${row.group} | ${row.ask} | ${row.rung} | ${outline} | ${row.parts} | ${row.coldMs} | ${row.warmMs} | ${row.nominatim} |`);
|
||||
}
|
||||
if (SHOT_DIR)
|
||||
fs.writeFileSync(path.resolve(ROOT, SHOT_DIR, 'outline-table.json'), JSON.stringify(table, null, 2));
|
||||
}
|
||||
|
||||
// ── Main ────────────────────────────────────────────────────
|
||||
console.log('\nVoice-surface QA harness');
|
||||
console.log(` App URL : ${APP_URL}`);
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"reviewedSources": {
|
||||
"zs": {
|
||||
"name": "Zetashapes",
|
||||
"registryCommit": "e17af153c144fe6e08aa8db2607b4bf8aecd448d",
|
||||
"registrySha256": "d8a5dd873d26eee11583de4438185c046dec33848d5bde27047ca4bff44d9c39",
|
||||
"rawLicenseType": "Public domain",
|
||||
"licenseTextSha256": "91757f96f74f6dfb4a7206486b78586da0d039e9611e224fc793c7167dcfe615",
|
||||
"decision": "Approved from the pinned WOF registry text: no added restrictions beyond public-domain US TIGER/Line data and Flickr-derived data; retain both upstream credits."
|
||||
}
|
||||
},
|
||||
"licenses": [
|
||||
"Attribution 3.0 Unported",
|
||||
"CC BY",
|
||||
"CC BY 2.0",
|
||||
"CC BY 3.0",
|
||||
"CC BY 3.0 US",
|
||||
"CC BY 4.0",
|
||||
"CC-BY",
|
||||
"CC0",
|
||||
"CC0 1.0 Universal",
|
||||
"ODC-By, v1.0",
|
||||
"ODC-By v1.0",
|
||||
"OGL - Canada",
|
||||
"OGL - City of New Westminster",
|
||||
"OGL - City of Surrey",
|
||||
"Open Government Licence v3.0",
|
||||
"Open Government Licence - British Columbia, v2.0",
|
||||
"Open Government Licence - Canada",
|
||||
"Open Government Licence - City of Calgary",
|
||||
"Open Government Licence - City of Regina",
|
||||
"Open Government Licence - Edmonton",
|
||||
"Open Government Licence - Ontario, v1.0",
|
||||
"Open Government License - British Columbia, v2.0",
|
||||
"Open Government License - Winnipeg",
|
||||
"PDDL 1.0",
|
||||
"Public domain",
|
||||
"Public Domain",
|
||||
"Singapore Open Data Licence v1"
|
||||
],
|
||||
"recordExclusions": {
|
||||
"reason": "Sources whose pinned registry licence is only an \"equivalent\" designation (for example \"CC BY (equivalent)\" or \"CC0 (equivalent)\"), including can-ons, can-saskodp, can-gatsudd and esp-aytomad, are not admitted: their redistribution terms have not been established for this derived pack.",
|
||||
"archives": {
|
||||
"neighbourhood": "8e5e413aacf2244e4e4167fbd9be860bc407ae6de8b733e47f13abe6d3d10455",
|
||||
"macrohood": "f5fbfabcb8229b83242dbd38530c27e61867e8b3b097e3393d9c841962a99679",
|
||||
"microhood": "e8197fa576e8831be8f091ac7e219d29224d89404ab0c56d377339e42a8f29ac"
|
||||
},
|
||||
"registrySha256": "d8a5dd873d26eee11583de4438185c046dec33848d5bde27047ca4bff44d9c39",
|
||||
"sources": {}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user