mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 10:53:59 +00:00
feat(outlines): voice outlines without Overpass — bundled US places and neighborhoods, guarded Nominatim, OpenFreeMap streets and buildings
This commit is contained in:
Bilawal Sidhu committed
1 parent
e7707d9a0f
commit
9559e7a282
333 files changed
+13203
-271
No files matched your search
@@ -111,8 +111,8 @@ const cacheDir = path.resolve(
|
||||
);
|
||||
const only = args.includes('--only') ? args[args.indexOf('--only') + 1] : null;
|
||||
|
||||
async function fetchPinned(name) {
|
||||
const { url, sha256 } = SOURCES[name];
|
||||
export async function fetchPinned(name, source = SOURCES[name]) {
|
||||
const { url, sha256 } = source;
|
||||
const file = path.join(cacheDir, path.basename(new URL(url).pathname));
|
||||
let bytes = await readFile(file).catch(() => null);
|
||||
if (!bytes) {
|
||||
@@ -137,7 +137,7 @@ const EARTH_RADIUS_KM = 6371;
|
||||
const toRad = (d) => (d * Math.PI) / 180;
|
||||
|
||||
/** Spherical ring area (km²); matches src/data/naturalEarthRegions.js. */
|
||||
function ringAreaKm2(ring) {
|
||||
export function ringAreaKm2(ring) {
|
||||
const n = ring.length;
|
||||
if (n < 3) return 0;
|
||||
let sum = 0;
|
||||
@@ -151,7 +151,7 @@ function ringAreaKm2(ring) {
|
||||
}
|
||||
|
||||
/** Signed planar (shoelace) area: positive counter-clockwise in lon/lat. */
|
||||
function signedArea(ring) {
|
||||
export function signedArea(ring) {
|
||||
let sum = 0;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++)
|
||||
sum += ring[j][0] * ring[i][1] - ring[i][0] * ring[j][1];
|
||||
@@ -209,15 +209,16 @@ function douglasPeucker(points, tolerance) {
|
||||
* Simplify a closed ring to an OPEN rounded ring. The ring is split at its
|
||||
* farthest vertex from the start so Douglas-Peucker has two anchored halves
|
||||
* (a closed ring has no chord to measure against). Returns null when fewer
|
||||
* than four distinct vertices survive.
|
||||
* than `minVertices` distinct vertices survive (four by default; a pack that
|
||||
* keeps triangles passes three).
|
||||
*/
|
||||
function simplifyRing(ring, tolerance, decimals) {
|
||||
function simplifyRing(ring, tolerance, decimals, minVertices = 4) {
|
||||
let open = ring.slice();
|
||||
const first = open[0];
|
||||
const last = open[open.length - 1];
|
||||
if (open.length > 1 && first[0] === last[0] && first[1] === last[1])
|
||||
open = open.slice(0, -1);
|
||||
if (open.length < 4) return null;
|
||||
if (open.length < minVertices) return null;
|
||||
let far = 1;
|
||||
let farDist = -1;
|
||||
for (let i = 1; i < open.length; i++) {
|
||||
@@ -247,7 +248,7 @@ function simplifyRing(ring, tolerance, decimals) {
|
||||
out[0][1] === out[out.length - 1][1]
|
||||
)
|
||||
out.pop();
|
||||
return out.length >= 4 ? out : null;
|
||||
return out.length >= minVertices ? out : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -255,16 +256,17 @@ function simplifyRing(ring, tolerance, decimals) {
|
||||
* area floors (the largest part always stays), orient outers
|
||||
* counter-clockwise and holes clockwise, sort parts by area descending.
|
||||
*/
|
||||
function simplifyPolygons(polygons, params, tolerance, decimals) {
|
||||
export function simplifyPolygons(polygons, params, tolerance, decimals) {
|
||||
const minVertices = params.minRingVertices ?? 4;
|
||||
const parts = [];
|
||||
for (const poly of polygons) {
|
||||
const outer = simplifyRing(poly[0], tolerance, decimals);
|
||||
const outer = simplifyRing(poly[0], tolerance, decimals, minVertices);
|
||||
if (!outer) continue;
|
||||
const area = ringAreaKm2(outer);
|
||||
if (signedArea(outer) < 0) outer.reverse();
|
||||
const holes = [];
|
||||
for (const hole of poly.slice(1)) {
|
||||
const ring = simplifyRing(hole, tolerance, decimals);
|
||||
const ring = simplifyRing(hole, tolerance, decimals, minVertices);
|
||||
if (!ring || ringAreaKm2(ring) < params.minHoleKm2) continue;
|
||||
if (signedArea(ring) > 0) ring.reverse();
|
||||
holes.push(ring);
|
||||
@@ -287,18 +289,22 @@ function simplifyPolygons(polygons, params, tolerance, decimals) {
|
||||
* and a fifth of the minimum tolerance, or failing that unsimplified at two
|
||||
* more decimals; the caller records `d`.
|
||||
*/
|
||||
function simplifyUnit(polygons, params) {
|
||||
export function unitTolerance(polygons, params) {
|
||||
const sourceArea = polygons.reduce(
|
||||
(sum, poly) => sum + ringAreaKm2(poly[0]),
|
||||
0,
|
||||
);
|
||||
const tolerance = Math.min(
|
||||
return Math.min(
|
||||
params.maxToleranceDeg,
|
||||
Math.max(
|
||||
params.minToleranceDeg,
|
||||
(params.toleranceFactor * Math.sqrt(sourceArea)) / 111.32,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function simplifyUnit(polygons, params) {
|
||||
const tolerance = unitTolerance(polygons, params);
|
||||
const coarse = simplifyPolygons(polygons, params, tolerance, params.decimals);
|
||||
if (coarse.length) return { polygons: coarse, decimals: params.decimals };
|
||||
const fine = simplifyPolygons(
|
||||
@@ -334,7 +340,7 @@ function encodeRing(ring, decimals) {
|
||||
return out;
|
||||
}
|
||||
|
||||
function encodeUnit({ polygons, decimals }) {
|
||||
export function encodeUnit({ polygons, decimals }) {
|
||||
return polygons.map((poly) => poly.map((ring) => encodeRing(ring, decimals)));
|
||||
}
|
||||
|
||||
@@ -362,7 +368,7 @@ function bboxOf(polygons) {
|
||||
// ── zip / shapefile / dbf ───────────────────────────────────────────────
|
||||
|
||||
/** Read named entries from a zip archive (stored or deflated). */
|
||||
function unzip(buffer) {
|
||||
export function unzip(buffer) {
|
||||
let eocd = buffer.length - 22;
|
||||
while (eocd >= 0 && buffer.readUInt32LE(eocd) !== 0x06054b50) eocd--;
|
||||
if (eocd < 0) throw new Error('zip: end of central directory not found');
|
||||
@@ -394,7 +400,7 @@ function unzip(buffer) {
|
||||
}
|
||||
|
||||
/** Polygon records (shape type 5) as arrays of closed [lon, lat] rings. */
|
||||
function readShp(buffer) {
|
||||
export function readShp(buffer) {
|
||||
const shapes = [];
|
||||
let offset = 100;
|
||||
while (offset < buffer.length) {
|
||||
@@ -431,7 +437,7 @@ function readShp(buffer) {
|
||||
}
|
||||
|
||||
/** dBASE records as plain objects (character and numeric fields). */
|
||||
function readDbf(buffer) {
|
||||
export function readDbf(buffer) {
|
||||
const count = buffer.readUInt32LE(4);
|
||||
const headerLength = buffer.readUInt16LE(8);
|
||||
const recordLength = buffer.readUInt16LE(10);
|
||||
@@ -458,7 +464,7 @@ function readDbf(buffer) {
|
||||
}
|
||||
|
||||
/** Group shapefile rings into polygons: clockwise outers, holes assigned by containment. */
|
||||
function shpPolygons(rings) {
|
||||
export function shpPolygons(rings) {
|
||||
const outers = [];
|
||||
const holes = [];
|
||||
for (const ring of rings) {
|
||||
@@ -475,7 +481,7 @@ function shpPolygons(rings) {
|
||||
return polygons;
|
||||
}
|
||||
|
||||
function pointInRing(ring, x, y) {
|
||||
export function pointInRing(ring, x, y) {
|
||||
let inside = false;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++) {
|
||||
const [xi, yi] = ring[i];
|
||||
@@ -742,18 +748,24 @@ async function writePack(relative, pack) {
|
||||
);
|
||||
}
|
||||
|
||||
if (!only || only === 'countries')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/countries.json',
|
||||
await buildCountries(),
|
||||
);
|
||||
if (!only || only === 'admin1')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/states_provinces.json',
|
||||
await buildAdmin1(),
|
||||
);
|
||||
if (!only || only === 'counties')
|
||||
await writePack(
|
||||
'src/data/local_data/us_census_counties/counties.json',
|
||||
await buildCounties(),
|
||||
);
|
||||
const isMain =
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
|
||||
if (isMain) {
|
||||
if (!only || only === 'countries')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/countries.json',
|
||||
await buildCountries(),
|
||||
);
|
||||
if (!only || only === 'admin1')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/states_provinces.json',
|
||||
await buildAdmin1(),
|
||||
);
|
||||
if (!only || only === 'counties')
|
||||
await writePack(
|
||||
'src/data/local_data/us_census_counties/counties.json',
|
||||
await buildCounties(),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the bundled US places pack read by `src/data/placeBoundaries.js`:
|
||||
*
|
||||
* src/data/local_data/us_census_places/index.json state list, boxes, meta
|
||||
* src/data/local_data/us_census_places/<ST>.json one file per state
|
||||
* src/data/local_data/us_census_places/files.js their URLs, for bundlers
|
||||
*
|
||||
* Source: US Census Bureau 2025 cartographic boundary places
|
||||
* (cb_2025_us_place_500k, public domain), pinned by URL and SHA-256, so a
|
||||
* rerun reproduces the pack byte for byte. The download is cached outside the
|
||||
* repository (default: the system temp directory).
|
||||
*
|
||||
* Geometry uses the county pack's simplification and encoding
|
||||
* (scripts/build-admin-packs.mjs) with two differences: triangles are kept,
|
||||
* and a unit whose simplified rings cross themselves or each other is
|
||||
* simplified again at half the tolerance (finer precision last). Then
|
||||
* `scripts/pack_geometry.py repair census` checks every feature as a whole
|
||||
* MultiPolygon under GEOS and repairs the ones that are still invalid (rings
|
||||
* touching themselves, overlapping parts), so it needs Python with
|
||||
* shapely==2.1.2 on GEOS 3.13.1.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/build-census-places.mjs [--cache <dir>] [--python <exe>]
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
PARAMS as ADMIN_PARAMS,
|
||||
encodeUnit,
|
||||
fetchPinned,
|
||||
readDbf,
|
||||
readShp,
|
||||
shpPolygons,
|
||||
simplifyPolygons,
|
||||
unitTolerance,
|
||||
unzip,
|
||||
} from './build-admin-packs.mjs';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const OUT = path.join(ROOT, 'src/data/local_data/us_census_places');
|
||||
|
||||
export const SOURCE = Object.freeze({
|
||||
url: 'https://www2.census.gov/geo/tiger/GENZ2025/shp/cb_2025_us_place_500k.zip',
|
||||
sha256: 'ce0e4019ecd4123d03d53aaa936eed0459b82e3e14b89a3dcd4d5e8b3308627d',
|
||||
});
|
||||
|
||||
/** County-pack parameters; triangles are valid rings here. */
|
||||
export const PARAMS = Object.freeze({
|
||||
...ADMIN_PARAMS.counties,
|
||||
minRingVertices: 3,
|
||||
maxRepairSteps: 8,
|
||||
});
|
||||
|
||||
/** Census LSAD codes for census designated places (unincorporated). */
|
||||
const CDP_LSAD = new Set(['55', '57', '62']);
|
||||
|
||||
// ── validity ─────────────────────────────────────────────────────────────
|
||||
|
||||
function orient(ax, ay, bx, by, cx, cy) {
|
||||
const v = (bx - ax) * (cy - ay) - (by - ay) * (cx - ax);
|
||||
return v > 0 ? 1 : v < 0 ? -1 : 0;
|
||||
}
|
||||
|
||||
function onSegment(ax, ay, bx, by, px, py) {
|
||||
return (
|
||||
Math.min(ax, bx) <= px &&
|
||||
px <= Math.max(ax, bx) &&
|
||||
Math.min(ay, by) <= py &&
|
||||
py <= Math.max(ay, by)
|
||||
);
|
||||
}
|
||||
|
||||
/** Whether segments ab and cd share any point (touching counts). */
|
||||
export function segmentsIntersect(a, b, c, d) {
|
||||
const o1 = orient(a[0], a[1], b[0], b[1], c[0], c[1]);
|
||||
const o2 = orient(a[0], a[1], b[0], b[1], d[0], d[1]);
|
||||
const o3 = orient(c[0], c[1], d[0], d[1], a[0], a[1]);
|
||||
const o4 = orient(c[0], c[1], d[0], d[1], b[0], b[1]);
|
||||
if (o1 !== o2 && o3 !== o4) return true;
|
||||
if (o1 === 0 && onSegment(a[0], a[1], b[0], b[1], c[0], c[1])) return true;
|
||||
if (o2 === 0 && onSegment(a[0], a[1], b[0], b[1], d[0], d[1])) return true;
|
||||
if (o3 === 0 && onSegment(c[0], c[1], d[0], d[1], a[0], a[1])) return true;
|
||||
if (o4 === 0 && onSegment(c[0], c[1], d[0], d[1], b[0], b[1])) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether segments of two different rings cross or overlap. Rings may touch
|
||||
* at a point (a hole meeting its outer ring at a vertex is valid).
|
||||
*/
|
||||
function segmentsCross(s, t) {
|
||||
const [a, b, c, d] = [s.a, s.b, t.a, t.b];
|
||||
const o1 = orient(a[0], a[1], b[0], b[1], c[0], c[1]);
|
||||
const o2 = orient(a[0], a[1], b[0], b[1], d[0], d[1]);
|
||||
const o3 = orient(c[0], c[1], d[0], d[1], a[0], a[1]);
|
||||
const o4 = orient(c[0], c[1], d[0], d[1], b[0], b[1]);
|
||||
if (o1 && o2 && o3 && o4) return o1 !== o2 && o3 !== o4;
|
||||
if (o1 === 0 && o2 === 0) {
|
||||
// Collinear: overlapping by more than a point.
|
||||
const key = a[0] !== b[0] ? 0 : 1;
|
||||
const lo = Math.max(Math.min(a[key], b[key]), Math.min(c[key], d[key]));
|
||||
const hi = Math.min(Math.max(a[key], b[key]), Math.max(c[key], d[key]));
|
||||
return hi > lo;
|
||||
}
|
||||
// One endpoint on the other segment: a crossing only when the rings pass
|
||||
// through each other there; a touch at a point is allowed.
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the rings of one part ([outer, ...holes], open, integer
|
||||
* coordinates) are simple: no ring crosses or touches itself away from its
|
||||
* shared vertices, no two rings cross or overlap (they may touch at a point),
|
||||
* and no ring is flat.
|
||||
*/
|
||||
export function partIsValid(rings) {
|
||||
const segments = [];
|
||||
for (let r = 0; r < rings.length; r++) {
|
||||
const ring = rings[r];
|
||||
if (ring.length < 3) return false;
|
||||
let twice = 0;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++)
|
||||
twice += ring[j][0] * ring[i][1] - ring[i][0] * ring[j][1];
|
||||
if (twice === 0) return false;
|
||||
for (let i = 0; i < ring.length; i++) {
|
||||
const a = ring[i];
|
||||
const b = ring[(i + 1) % ring.length];
|
||||
segments.push({
|
||||
r,
|
||||
i,
|
||||
n: ring.length,
|
||||
a,
|
||||
b,
|
||||
minX: Math.min(a[0], b[0]),
|
||||
maxX: Math.max(a[0], b[0]),
|
||||
});
|
||||
}
|
||||
}
|
||||
segments.sort((s, t) => s.minX - t.minX);
|
||||
for (let x = 0; x < segments.length; x++) {
|
||||
const s = segments[x];
|
||||
for (let y = x + 1; y < segments.length; y++) {
|
||||
const t = segments[y];
|
||||
if (t.minX > s.maxX) break;
|
||||
if (s.r === t.r) {
|
||||
const gap = Math.abs(s.i - t.i);
|
||||
// Neighbours share exactly one vertex; that is not a crossing — unless
|
||||
// they fold back over each other.
|
||||
if (gap === 1 || gap === s.n - 1) {
|
||||
if (s.n > 3 && collinearOverlap(s, t)) return false;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (segmentsCross(s, t)) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function collinearOverlap(s, t) {
|
||||
const shared =
|
||||
s.a === t.a || s.a === t.b ? s.a : s.b === t.a || s.b === t.b ? s.b : null;
|
||||
if (!shared) return false;
|
||||
const p = s.a === shared ? s.b : s.a;
|
||||
const q = t.a === shared ? t.b : t.a;
|
||||
if (orient(shared[0], shared[1], p[0], p[1], q[0], q[1]) !== 0) return false;
|
||||
// Collinear and pointing the same way from the shared vertex: a spike.
|
||||
return (
|
||||
(p[0] - shared[0]) * (q[0] - shared[0]) +
|
||||
(p[1] - shared[1]) * (q[1] - shared[1]) >
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
function toIntegers(polygons, decimals) {
|
||||
const f = 10 ** decimals;
|
||||
return polygons.map((poly) =>
|
||||
poly.map((ring) =>
|
||||
ring.map(([lon, lat]) => [Math.round(lon * f), Math.round(lat * f)]),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function polygonsValid(polygons, decimals) {
|
||||
return toIntegers(polygons, decimals).every(partIsValid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify a place like `simplifyUnit`, then halve the tolerance until every
|
||||
* part is valid; then one more decimal; then the source rings at two more.
|
||||
* @returns {{polygons: Array, decimals: number, repaired: boolean, valid: boolean}}
|
||||
*/
|
||||
export function simplifyPlace(source, params = PARAMS) {
|
||||
let tolerance = unitTolerance(source, params);
|
||||
const attempts = [];
|
||||
for (let step = 0; step <= params.maxRepairSteps; step++) {
|
||||
attempts.push([tolerance, params.decimals]);
|
||||
tolerance /= 2;
|
||||
}
|
||||
attempts.push([params.minToleranceDeg / 5, params.decimals + 1]);
|
||||
attempts.push([0, params.decimals + 1]);
|
||||
attempts.push([0, params.decimals + 2]);
|
||||
let first = null;
|
||||
for (let i = 0; i < attempts.length; i++) {
|
||||
const [tol, decimals] = attempts[i];
|
||||
const polygons = simplifyPolygons(source, params, tol, decimals);
|
||||
if (!polygons.length) continue;
|
||||
first ||= { polygons, decimals };
|
||||
if (polygonsValid(polygons, decimals))
|
||||
return { polygons, decimals, repaired: i > 0, valid: true };
|
||||
}
|
||||
return first
|
||||
? { ...first, repaired: false, valid: false }
|
||||
: {
|
||||
polygons: [],
|
||||
decimals: params.decimals,
|
||||
repaired: false,
|
||||
valid: false,
|
||||
};
|
||||
}
|
||||
|
||||
// ── names ────────────────────────────────────────────────────────────────
|
||||
|
||||
const GOVERNMENT =
|
||||
/\s+(?:metropolitan government|unified government|consolidated government|metro government|city and county|urban county)\b.*$/i;
|
||||
|
||||
/**
|
||||
* Everyday names for records whose Census name is formal: "Nashville-Davidson
|
||||
* metropolitan government (balance)" is also "Nashville"; "Urban Honolulu"
|
||||
* is "Honolulu".
|
||||
*/
|
||||
export function placeAliases(name) {
|
||||
const out = new Set();
|
||||
let base = String(name).replace(/\s*\(balance\)\s*$/i, '');
|
||||
if (base !== name) out.add(base);
|
||||
const stripped = base.replace(GOVERNMENT, '');
|
||||
if (stripped !== base) {
|
||||
out.add(stripped);
|
||||
base = stripped;
|
||||
}
|
||||
if (base !== name && /[-/]/.test(base)) out.add(base.split(/[-/]/)[0].trim());
|
||||
const urban = /^Urban (.+)$/.exec(name);
|
||||
if (urban) out.add(urban[1]);
|
||||
out.delete(name);
|
||||
return [...out].filter(Boolean).sort();
|
||||
}
|
||||
|
||||
// ── pack ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function bboxOf(polygons) {
|
||||
let w = Infinity;
|
||||
let s = Infinity;
|
||||
let e = -Infinity;
|
||||
let n = -Infinity;
|
||||
for (const poly of polygons)
|
||||
for (const [lon, lat] of poly[0]) {
|
||||
if (lon < w) w = lon;
|
||||
if (lon > e) e = lon;
|
||||
if (lat < s) s = lat;
|
||||
if (lat > n) n = lat;
|
||||
}
|
||||
return [w, s, e, n];
|
||||
}
|
||||
|
||||
function union(boxes) {
|
||||
return [
|
||||
Math.min(...boxes.map((b) => b[0])),
|
||||
Math.min(...boxes.map((b) => b[1])),
|
||||
Math.max(...boxes.map((b) => b[2])),
|
||||
Math.max(...boxes.map((b) => b[3])),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The box of a state's places. Alaska crosses the antimeridian: its box is
|
||||
* taken with western longitudes shifted past 180 when that is narrower.
|
||||
*/
|
||||
function unionBox(boxes) {
|
||||
const plain = union(boxes);
|
||||
const shifted = union(
|
||||
boxes.map(([w, s, e, n]) =>
|
||||
e < 0 ? [w + 360, s, e + 360, n] : [w, s, e, n],
|
||||
),
|
||||
);
|
||||
return shifted[2] - shifted[0] < plain[2] - plain[0] ? shifted : plain;
|
||||
}
|
||||
|
||||
async function build() {
|
||||
const params = PARAMS;
|
||||
const source = await fetchPinned('places', SOURCE);
|
||||
const files = unzip(source.bytes);
|
||||
const base = path.basename(new URL(SOURCE.url).pathname, '.zip');
|
||||
const shapes = readShp(files.get(`${base}.shp`));
|
||||
const records = readDbf(files.get(`${base}.dbf`));
|
||||
if (shapes.length !== records.length)
|
||||
throw new Error('places: shp/dbf record counts differ');
|
||||
const states = new Map();
|
||||
const stats = { records: records.length, repaired: [], invalid: [] };
|
||||
records.forEach((record, index) => {
|
||||
const unit = simplifyPlace(shpPolygons(shapes[index]), params);
|
||||
if (!unit.polygons.length) throw new Error(`no geometry: ${record.GEOID}`);
|
||||
if (unit.repaired) stats.repaired.push(record.GEOID);
|
||||
if (!unit.valid) stats.invalid.push(record.GEOID);
|
||||
const alt = placeAliases(record.NAME);
|
||||
const lsad = record.NAMELSAD.startsWith(record.NAME)
|
||||
? record.NAMELSAD.slice(record.NAME.length).trim()
|
||||
: '';
|
||||
let state = states.get(record.STUSPS);
|
||||
if (!state) {
|
||||
state = {
|
||||
st: record.STUSPS,
|
||||
fips: record.STATEFP,
|
||||
name: record.STATE_NAME,
|
||||
features: [],
|
||||
boxes: [],
|
||||
};
|
||||
states.set(record.STUSPS, state);
|
||||
}
|
||||
state.boxes.push(bboxOf(unit.polygons));
|
||||
state.features.push({
|
||||
geoid: record.GEOID,
|
||||
name: record.NAME,
|
||||
...(alt.length ? { alt } : {}),
|
||||
full: record.NAMELSAD,
|
||||
...(lsad ? { lsad } : {}),
|
||||
...(CDP_LSAD.has(record.LSAD) ? { cdp: true } : {}),
|
||||
...(unit.decimals !== params.decimals ? { d: unit.decimals } : {}),
|
||||
polygons: encodeUnit(unit),
|
||||
});
|
||||
});
|
||||
if (stats.invalid.length)
|
||||
throw new Error(`places still invalid: ${stats.invalid.join(', ')}`);
|
||||
|
||||
await mkdir(OUT, { recursive: true });
|
||||
for (const file of await readdir(OUT))
|
||||
if (/^[A-Z]{2}\.json$/.test(file)) await rm(path.join(OUT, file));
|
||||
const list = [...states.values()].sort((a, b) => a.st.localeCompare(b.st));
|
||||
let bytes = 0;
|
||||
for (const state of list) {
|
||||
state.features.sort((a, b) => a.geoid.localeCompare(b.geoid));
|
||||
const text =
|
||||
JSON.stringify({
|
||||
st: state.st,
|
||||
state: state.name,
|
||||
fips: state.fips,
|
||||
features: state.features,
|
||||
}) + '\n';
|
||||
bytes += Buffer.byteLength(text);
|
||||
await writeFile(path.join(OUT, `${state.st}.json`), text);
|
||||
}
|
||||
const round = (v) => Math.round(v * 1e4) / 1e4;
|
||||
const index = {
|
||||
meta: {
|
||||
title: 'US Census Bureau cartographic boundary places',
|
||||
source: base,
|
||||
url: SOURCE.url,
|
||||
sha256: source.digest,
|
||||
license: 'Public domain (U.S. Government work, 17 U.S.C. § 105)',
|
||||
schema:
|
||||
'states[]: st (USPS), fips, name, count, bbox [w, s, e, n] (e > 180 across the antimeridian). <st>.json: st, state, fips, features[]: geoid, name (NAME), alt?[] (everyday names for formal records), full (NAMELSAD), lsad? (city, town, CDP, …), cdp? (census designated place), d? (decimals when not meta.decimals), polygons[][ring] (first ring outer, rest holes; each ring open and encoded as integers in 10^-d degrees, first vertex absolute then [dLon, dLat] deltas)',
|
||||
decimals: params.decimals,
|
||||
curation: params,
|
||||
script: 'scripts/build-census-places.mjs',
|
||||
},
|
||||
states: list.map((s) => ({
|
||||
st: s.st,
|
||||
fips: s.fips,
|
||||
name: s.name,
|
||||
count: s.features.length,
|
||||
bbox: unionBox(s.boxes).map(round),
|
||||
})),
|
||||
};
|
||||
await writeFile(
|
||||
path.join(OUT, 'index.json'),
|
||||
JSON.stringify(index, null, 1) + '\n',
|
||||
);
|
||||
await writeFile(path.join(OUT, 'files.js'), filesModule(list));
|
||||
const count = list.reduce((n, s) => n + s.features.length, 0);
|
||||
console.log(
|
||||
`us_census_places: ${count} places in ${list.length} files, ${bytes} bytes; ` +
|
||||
`${stats.repaired.length} re-simplified to stay valid`,
|
||||
);
|
||||
const args = process.argv.slice(2);
|
||||
const python = args.includes('--python')
|
||||
? args[args.indexOf('--python') + 1]
|
||||
: 'python3';
|
||||
execFileSync(
|
||||
python,
|
||||
[path.join(ROOT, 'scripts/pack_geometry.py'), 'repair', 'census'],
|
||||
{ stdio: 'inherit', env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' } },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The file list as literal URLs, so a bundler emits every state file as an
|
||||
* asset and the loader can fetch one by its code.
|
||||
*/
|
||||
function filesModule(list) {
|
||||
const lines = [
|
||||
'// Generated by scripts/build-census-places.mjs; do not edit.',
|
||||
"export const INDEX_URL = new URL('./index.json', import.meta.url);",
|
||||
'export const STATE_URLS = {',
|
||||
...list.map(
|
||||
(s) => ` ${s.st}: new URL('./${s.st}.json', import.meta.url),`,
|
||||
),
|
||||
'};',
|
||||
];
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
const isMain =
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
if (isMain) await build();
|
||||
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maintainer-only Who's On First neighborhood pack build (Shapely 2.1).
|
||||
|
||||
Reads the pinned Who's On First neighbourhood, macrohood and microhood
|
||||
archives and the WOF sources registry, keeps current polygon records whose
|
||||
geometry source has a stated licence, simplifies them to about 15 m, and
|
||||
writes src/data/local_data/wof_neighborhoods/:
|
||||
|
||||
index.json format, licences per source, tile list with boxes
|
||||
<tile>.json records whose box centre falls in that quadtree tile
|
||||
files.js their URLs, for bundlers
|
||||
ATTRIBUTION.md per-source credits
|
||||
|
||||
Usage:
|
||||
python scripts/build-wof-neighborhoods.py --cache /tmp/wof-neighborhoods [--verify]
|
||||
|
||||
Requires shapely==2.1.2 on GEOS 3.13.1 (checked). --verify rebuilds in memory
|
||||
and fails unless every shipped file matches byte for byte.
|
||||
|
||||
Downloads go to --cache and are checked against the pinned SHA-256. The
|
||||
neighbourhood archive is about 400 MB.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
from shapely import make_valid
|
||||
from shapely.affinity import scale
|
||||
from shapely.geometry import MultiPolygon, shape
|
||||
|
||||
import pack_geometry
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
PACK = ROOT / 'src/data/local_data/wof_neighborhoods'
|
||||
# The legacy distribution is frozen (last published 2025-10-14) and only
|
||||
# offers `-latest` names; the SHA-256 pins make a changed file fail loudly.
|
||||
DIST = 'https://data.geocode.earth/wof/dist/legacy'
|
||||
ARCHIVES = {
|
||||
'neighbourhood': {
|
||||
'url': DIST + '/whosonfirst-data-neighbourhood-latest.tar.bz2',
|
||||
'sha256': '8e5e413aacf2244e4e4167fbd9be860bc407ae6de8b733e47f13abe6d3d10455',
|
||||
},
|
||||
'macrohood': {
|
||||
'url': DIST + '/whosonfirst-data-macrohood-latest.tar.bz2',
|
||||
'sha256': 'f5fbfabcb8229b83242dbd38530c27e61867e8b3b097e3393d9c841962a99679',
|
||||
},
|
||||
'microhood': {
|
||||
'url': DIST + '/whosonfirst-data-microhood-latest.tar.bz2',
|
||||
'sha256': 'e8197fa576e8831be8f091ac7e219d29224d89404ab0c56d377339e42a8f29ac',
|
||||
},
|
||||
}
|
||||
# The registry at a fixed commit.
|
||||
SOURCES = {
|
||||
'url': 'https://raw.githubusercontent.com/whosonfirst/whosonfirst-sources/'
|
||||
'e17af153c144fe6e08aa8db2607b4bf8aecd448d/data/sources-spec-latest.json',
|
||||
'sha256': 'd8a5dd873d26eee11583de4438185c046dec33848d5bde27047ca4bff44d9c39',
|
||||
}
|
||||
TOLERANCE_M = 15
|
||||
PRECISION = 5
|
||||
TILE_MAX_BYTES = 200_000
|
||||
TILE_MAX_DEPTH = 16
|
||||
# A geometry source is kept only when the registry states its licence.
|
||||
UNSTATED = re.compile(r'assumed|unknown|unresolved|n/a', re.I)
|
||||
FIELDS = ['id', 'name', 'country', 'type', 'bbox', 'aliases', 'source', 'rings', 'label']
|
||||
|
||||
|
||||
def fetch(cache, name, url, sha256):
|
||||
path = cache / name
|
||||
if not path.exists():
|
||||
print('Downloading', url, flush=True)
|
||||
# Plain urllib requests are refused by the WOF host; curl is not.
|
||||
subprocess.run(['curl', '--fail', '-L', '-o', str(path), url], check=True)
|
||||
digest = hashlib.sha256()
|
||||
with open(path, 'rb') as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b''):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != sha256:
|
||||
raise ValueError(f'{name}: SHA-256 {digest.hexdigest()} is not the pinned {sha256}')
|
||||
return path
|
||||
|
||||
|
||||
# ── geometry ────────────────────────────────────────────────────────────
|
||||
|
||||
def simplify(geometry):
|
||||
"""Simplify at TOLERANCE_M in local metres, snap to the pack grid, keep it
|
||||
valid, and order parts largest first. Returns (rings, label) where the
|
||||
label is a point inside the largest part, or ([], None)."""
|
||||
geometry = MultiPolygon(pack_geometry.polygon_parts(make_valid(geometry) if not geometry.is_valid else geometry))
|
||||
if geometry.is_empty:
|
||||
return [], None
|
||||
lat = geometry.centroid.y
|
||||
sx = 111320 * max(0.001, math.cos(math.radians(lat)))
|
||||
sy = 110574
|
||||
metric = scale(geometry, xfact=sx, yfact=sy, origin=(0, 0))
|
||||
metric = metric.simplify(TOLERANCE_M, preserve_topology=True)
|
||||
geometry = scale(metric, xfact=1 / sx, yfact=1 / sy, origin=(0, 0))
|
||||
# To whole pack units (10^-PRECISION degrees), where GEOS is exact.
|
||||
units = scale(geometry, xfact=10 ** PRECISION, yfact=10 ** PRECISION, origin=(0, 0))
|
||||
parts = pack_geometry.ordered_parts(pack_geometry.settle(units))
|
||||
if not parts:
|
||||
return [], None
|
||||
rings = [
|
||||
[pack_geometry.encode_polyline(pack_geometry.open_ring(p.exterior.coords))]
|
||||
+ [pack_geometry.encode_polyline(pack_geometry.open_ring(r.coords)) for r in p.interiors]
|
||||
for p in parts
|
||||
]
|
||||
# Check what ships: the geometry decoded from the encoded strings.
|
||||
decoded = [[pack_geometry.decode_polyline(r) for r in poly] for poly in rings]
|
||||
why = pack_geometry.problems(decoded)
|
||||
if why:
|
||||
raise ValueError(f'encoded geometry invalid: {why}')
|
||||
point = parts[0].representative_point()
|
||||
point = shape({'type': 'Point', 'coordinates': [round(point.x) / 10 ** PRECISION, round(point.y) / 10 ** PRECISION]})
|
||||
return rings, [point.x, point.y]
|
||||
|
||||
|
||||
# ── records ─────────────────────────────────────────────────────────────
|
||||
|
||||
def read_archive(path, records):
|
||||
with tarfile.open(path, 'r|bz2') as archive:
|
||||
for member in archive:
|
||||
if not member.name.endswith('.geojson') or '-alt-' in member.name:
|
||||
continue
|
||||
feature = json.load(archive.extractfile(member))
|
||||
if feature['geometry']['type'] not in ('Polygon', 'MultiPolygon'):
|
||||
records.pop(feature['properties']['wof:id'], None)
|
||||
continue
|
||||
records[feature['properties']['wof:id']] = feature
|
||||
|
||||
|
||||
def keep(properties):
|
||||
deprecated = properties.get('edtf:deprecated', 'uuuu') not in ('uuuu', '', None)
|
||||
superseded = bool(properties.get('wof:superseded_by', []))
|
||||
return properties.get('mz:is_current', -1) != 0 and not deprecated and not superseded
|
||||
|
||||
|
||||
def source_table(spec):
|
||||
lookup = {}
|
||||
for source in spec.values():
|
||||
for key in (source.get('name'), source.get('prefix')):
|
||||
if key:
|
||||
lookup[key] = source
|
||||
return lookup
|
||||
|
||||
|
||||
def stated(key, source):
|
||||
licence = (source or {}).get('license_type') or ''
|
||||
return key != 'unknown' and bool(source) and bool(licence) and not UNSTATED.search(licence)
|
||||
|
||||
|
||||
def build_records(archives, lookup):
|
||||
raw = {}
|
||||
for kind in ('neighbourhood', 'macrohood', 'microhood'):
|
||||
read_archive(archives[kind], raw)
|
||||
print(kind, len(raw), flush=True)
|
||||
features = []
|
||||
excluded = collections.Counter()
|
||||
for wof_id in sorted(raw):
|
||||
properties = raw[wof_id]['properties']
|
||||
if not keep(properties):
|
||||
continue
|
||||
key = properties.get('src:geom', 'unknown')
|
||||
if not stated(key, lookup.get(key)):
|
||||
excluded[key] += 1
|
||||
continue
|
||||
name = properties['wof:name']
|
||||
aliases = sorted({
|
||||
alias
|
||||
for k, values in properties.items()
|
||||
if k.startswith('name:') and isinstance(values, list)
|
||||
for alias in values
|
||||
if isinstance(alias, str) and alias != name
|
||||
})
|
||||
if not name.strip() and not aliases:
|
||||
continue # nothing to ask for it by
|
||||
geometry = shape(raw[wof_id]['geometry'])
|
||||
if geometry.is_empty:
|
||||
continue
|
||||
rings, label = simplify(geometry)
|
||||
if not rings:
|
||||
continue
|
||||
country = properties.get('iso:country', properties.get('wof:country'))
|
||||
features.append([
|
||||
wof_id, name, country, properties['wof:placetype'],
|
||||
[round(v, PRECISION) for v in geometry.bounds], aliases, key, rings, label,
|
||||
])
|
||||
return features, excluded
|
||||
|
||||
|
||||
# ── tiles ───────────────────────────────────────────────────────────────
|
||||
|
||||
def compact(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
||||
|
||||
|
||||
def tiles(features, key='', box=(-180.0, -90.0, 180.0, 90.0)):
|
||||
size = sum(len(compact(f).encode()) for f in features)
|
||||
if size <= TILE_MAX_BYTES or len(key) >= TILE_MAX_DEPTH:
|
||||
yield key or 'world', features
|
||||
return
|
||||
west, south, east, north = box
|
||||
mx, my = (west + east) / 2, (south + north) / 2
|
||||
quads = [[], [], [], []]
|
||||
for f in features:
|
||||
x = (f[4][0] + f[4][2]) / 2
|
||||
y = (f[4][1] + f[4][3]) / 2
|
||||
quads[(1 if x >= mx else 0) + (2 if y >= my else 0)].append(f)
|
||||
boxes = [(west, south, mx, my), (mx, south, east, my), (west, my, mx, north), (mx, my, east, north)]
|
||||
for i, quad in enumerate(quads):
|
||||
if quad:
|
||||
yield from tiles(quad, key + str(i), boxes[i])
|
||||
|
||||
|
||||
def union_box(features):
|
||||
return [
|
||||
min(f[4][0] for f in features), min(f[4][1] for f in features),
|
||||
max(f[4][2] for f in features), max(f[4][3] for f in features),
|
||||
]
|
||||
|
||||
|
||||
# ── credits ─────────────────────────────────────────────────────────────
|
||||
|
||||
def licence_label(key, source):
|
||||
licence = source['license_type']
|
||||
if key == 'karmashapes':
|
||||
return 'CC BY 4.0'
|
||||
return {
|
||||
'CC-BY': 'CC BY', 'Attribution 3.0 Unported': 'CC BY 3.0', 'Public domain': 'Public Domain',
|
||||
'Open Government License - British Columbia, v2.0': 'Open Government Licence - British Columbia, v2.0',
|
||||
}.get(licence, licence)
|
||||
|
||||
|
||||
def attribution(counts, lookup):
|
||||
lines = [
|
||||
'# Neighborhood boundary credits',
|
||||
'',
|
||||
"Neighborhood boundaries from [Who's On First](https://whosonfirst.org/docs/licenses/) "
|
||||
'and contributors, used under the licences below. Modified: boundaries simplified to about 15 m, repaired and re-encoded.',
|
||||
'',
|
||||
'| Source | Records | Licence | Terms |',
|
||||
'|---|---:|---|---|',
|
||||
]
|
||||
for key, count in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])):
|
||||
source = lookup[key]
|
||||
name = source.get('fullname') or key
|
||||
terms = source.get('license') or ''
|
||||
link = f'[terms]({terms})' if terms.startswith('http') else terms
|
||||
lines.append(f'| {name} (`{key}`) | {count:,} | {licence_label(key, source)} | {link} |')
|
||||
via = [(key, lookup[key]) for key in sorted(counts) if lookup[key].get('src:via')]
|
||||
if via:
|
||||
lines += ['', '## Upstream sources', '']
|
||||
for key, source in via:
|
||||
lines += [f'### {source.get("fullname") or key}', '']
|
||||
for v in source['src:via']:
|
||||
note = f' ({v["source_note"]})' if v.get('source_note') else ''
|
||||
context = f'{v["context"]}: ' if v.get('context') else ''
|
||||
lines.append(f'- {context}{v.get("source_name", "")}{note} {v.get("source_link", "")}'.rstrip())
|
||||
lines.append('')
|
||||
return '\n'.join(lines).rstrip() + '\n'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--cache', type=Path, required=True)
|
||||
parser.add_argument('--verify', action='store_true', help='rebuild in memory and compare with the shipped pack')
|
||||
args = parser.parse_args()
|
||||
pack_geometry.require_pinned_tooling()
|
||||
args.cache.mkdir(parents=True, exist_ok=True)
|
||||
archives = {
|
||||
kind: fetch(args.cache, Path(a['url']).name, a['url'], a['sha256'])
|
||||
for kind, a in ARCHIVES.items()
|
||||
}
|
||||
spec = json.loads(fetch(args.cache, 'sources-spec-latest.json', SOURCES['url'], SOURCES['sha256']).read_text())
|
||||
lookup = source_table(spec)
|
||||
features, excluded = build_records(archives, lookup)
|
||||
if len({f[0] for f in features}) != len(features):
|
||||
raise ValueError('duplicate WOF id')
|
||||
|
||||
out = {}
|
||||
tile_list = []
|
||||
for key, members in tiles(features):
|
||||
members.sort(key=lambda f: f[0])
|
||||
out[f'{key}.json'] = compact({'tile': key, 'features': members}) + '\n'
|
||||
tile_list.append({'key': key, 'count': len(members), 'bbox': union_box(members)})
|
||||
counts = collections.Counter(f[6] for f in features)
|
||||
index = {
|
||||
'meta': {
|
||||
'title': "Who's On First neighbourhood, macrohood and microhood polygons",
|
||||
'archives': ARCHIVES,
|
||||
'sourcesRegistry': SOURCES,
|
||||
'license': 'Per source; see sources and ATTRIBUTION.md',
|
||||
'format': 'neighborhood-polylines-v1',
|
||||
'fields': FIELDS,
|
||||
'precision': PRECISION,
|
||||
'coordinateOrder': 'lon,lat',
|
||||
'toleranceM': TOLERANCE_M,
|
||||
'schema': (
|
||||
'tiles[]: key, count, bbox [w, s, e, n] of its records. <key>.json: tile, features[] as '
|
||||
'fields: WOF id, name, ISO country, placetype, bbox, aliases[] (other-language names), '
|
||||
'geometry source key, rings (polygons, largest first, each [outer, ...holes]; each ring '
|
||||
'an encoded polyline string: zigzag varint deltas of lon then lat at 10^-precision '
|
||||
'degrees), label [lon, lat] (a point inside the largest polygon)'
|
||||
),
|
||||
'sources': {
|
||||
key: {
|
||||
'name': lookup[key].get('fullname') or key,
|
||||
'license': licence_label(key, lookup[key]),
|
||||
'terms': lookup[key].get('license') or '',
|
||||
'records': counts[key],
|
||||
}
|
||||
for key in sorted(counts)
|
||||
},
|
||||
'script': 'scripts/build-wof-neighborhoods.py',
|
||||
},
|
||||
'tiles': tile_list,
|
||||
}
|
||||
out['index.json'] = json.dumps(index, ensure_ascii=False, indent=1) + '\n'
|
||||
out['ATTRIBUTION.md'] = attribution(counts, lookup)
|
||||
# Literal URLs, so a bundler emits every tile as an asset.
|
||||
out['files.js'] = '\n'.join([
|
||||
'// Generated by scripts/build-wof-neighborhoods.py; do not edit.',
|
||||
"export const INDEX_URL = new URL('./index.json', import.meta.url);",
|
||||
'export const TILE_URLS = {',
|
||||
*[f" t{t['key']}: new URL('./{t['key']}.json', import.meta.url)," for t in tile_list],
|
||||
'};',
|
||||
]) + '\n'
|
||||
if args.verify:
|
||||
# Reproducibility: a rebuild must match what ships, byte for byte.
|
||||
shipped = {p.name for p in PACK.glob('*.json')} | {'ATTRIBUTION.md', 'files.js'}
|
||||
differ = sorted(
|
||||
name for name in shipped | set(out)
|
||||
if not (PACK / name).exists() or name not in out
|
||||
or (PACK / name).read_text(encoding='utf-8') != out[name]
|
||||
)
|
||||
if differ:
|
||||
raise SystemExit(f'rebuild differs from the shipped pack: {differ[:10]}')
|
||||
print(f'wof_neighborhoods: rebuild matches all {len(out)} shipped files')
|
||||
return
|
||||
PACK.mkdir(parents=True, exist_ok=True)
|
||||
for old in PACK.glob('*.json'):
|
||||
old.unlink()
|
||||
for name, text in out.items():
|
||||
(PACK / name).write_text(text, encoding='utf-8')
|
||||
print(f'wof_neighborhoods: {len(features)} records in {len(tile_list)} tiles; '
|
||||
f'excluded {sum(excluded.values())} with an unstated licence: {dict(excluded)}', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -30,6 +30,11 @@
|
||||
"src/annotations/regionRing.test.mjs",
|
||||
"src/data/adminBoundaries.test.mjs",
|
||||
"scripts/build-admin-packs.mjs",
|
||||
"scripts/build-census-places.mjs",
|
||||
"src/data/placeBoundaries.test.mjs",
|
||||
"src/data/boundaryLabels.test.mjs",
|
||||
"src/data/packGeometry.test.mjs",
|
||||
"src/annotations/placeOutline.test.mjs",
|
||||
"scripts/qa-admin-outlines.mjs",
|
||||
"src/data/inputOwnership.test.mjs",
|
||||
"docs/CODE-BOUNDARIES.md",
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maintainer-only topology check and repair for the bundled outline packs.
|
||||
|
||||
python scripts/pack_geometry.py check census|wof every shipped feature,
|
||||
decoded as a whole MultiPolygon, is valid under GEOS, keeps at least
|
||||
three distinct vertices per ring, and lists its largest part first
|
||||
python scripts/pack_geometry.py repair census rewrite the Census place
|
||||
files so every feature passes `check` (run by build-census-places.mjs)
|
||||
|
||||
Geometry is checked and repaired in the packs' own integer units (10^-d
|
||||
degrees), where GEOS arithmetic is exact: a vertex that lies on another edge
|
||||
after rounding is seen as the touch it is.
|
||||
|
||||
Requires Shapely 2.1.2 on GEOS 3.13.1, pinned so repairs are reproducible.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import shapely
|
||||
from shapely import make_valid, set_precision
|
||||
from shapely.geometry import MultiPolygon, Polygon
|
||||
from shapely.geometry.polygon import orient
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DATA = ROOT / 'src/data/local_data'
|
||||
SHAPELY_VERSION = '2.1.2'
|
||||
GEOS_VERSION = (3, 13, 1)
|
||||
|
||||
|
||||
def require_pinned_tooling():
|
||||
if shapely.__version__ != SHAPELY_VERSION or shapely.geos_version != GEOS_VERSION:
|
||||
raise RuntimeError(
|
||||
f'Use shapely=={SHAPELY_VERSION} on GEOS {".".join(map(str, GEOS_VERSION))}; '
|
||||
f'found {shapely.__version__} on GEOS {shapely.geos_version}'
|
||||
)
|
||||
|
||||
|
||||
def compact(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
||||
|
||||
|
||||
# ── encodings ────────────────────────────────────────────────────────────
|
||||
|
||||
def decode_deltas(encoded):
|
||||
"""[x0, y0, dx1, dy1, …] → [(x, y), …] in pack units (Census/admin packs)."""
|
||||
x = y = 0
|
||||
ring = []
|
||||
for i in range(0, len(encoded) - 1, 2):
|
||||
x += encoded[i]
|
||||
y += encoded[i + 1]
|
||||
ring.append((x, y))
|
||||
return ring
|
||||
|
||||
|
||||
def encode_deltas(coords):
|
||||
out = []
|
||||
px = py = 0
|
||||
for fx, fy in coords:
|
||||
x, y = round(fx), round(fy)
|
||||
out += [x - px, y - py]
|
||||
px, py = x, y
|
||||
return out
|
||||
|
||||
|
||||
def decode_polyline(encoded):
|
||||
"""Zigzag varint deltas, 5-bit groups offset by 63, lon then lat; pack units."""
|
||||
values = []
|
||||
last = [0, 0]
|
||||
i = 0
|
||||
while i < len(encoded):
|
||||
n = shift = 0
|
||||
while True:
|
||||
b = ord(encoded[i]) - 63
|
||||
i += 1
|
||||
n |= (b & 31) << shift
|
||||
shift += 5
|
||||
if b < 32:
|
||||
break
|
||||
axis = len(values) % 2
|
||||
last[axis] += ~(n >> 1) if n & 1 else n >> 1
|
||||
values.append(last[axis])
|
||||
return list(zip(values[::2], values[1::2]))
|
||||
|
||||
|
||||
def encode_polyline(coords):
|
||||
out = []
|
||||
last = [0, 0]
|
||||
for x, y in coords:
|
||||
for i, value in enumerate([round(x), round(y)]):
|
||||
n = value - last[i]
|
||||
last[i] = value
|
||||
n = (n << 1) ^ (n >> 63)
|
||||
while n >= 32:
|
||||
out.append(chr((32 | (n & 31)) + 63))
|
||||
n >>= 5
|
||||
out.append(chr(n + 63))
|
||||
return ''.join(out)
|
||||
|
||||
|
||||
# ── geometry ─────────────────────────────────────────────────────────────
|
||||
|
||||
def polygon_parts(geometry):
|
||||
if geometry.geom_type == 'Polygon':
|
||||
return [] if geometry.is_empty else [geometry]
|
||||
if hasattr(geometry, 'geoms'):
|
||||
return [p for g in geometry.geoms for p in polygon_parts(g)]
|
||||
return []
|
||||
|
||||
|
||||
def open_ring(coords):
|
||||
coords = [tuple(c[:2]) for c in coords]
|
||||
return coords[:-1] if len(coords) > 1 and coords[0] == coords[-1] else coords
|
||||
|
||||
|
||||
def ordered_parts(geometry):
|
||||
"""Polygon parts oriented (outer counter-clockwise), largest area first."""
|
||||
parts = [orient(p, 1.0) for p in polygon_parts(geometry)]
|
||||
return sorted(parts, key=lambda p: (-p.area, p.bounds))
|
||||
|
||||
|
||||
def on_grid(geometry):
|
||||
"""Snap to whole pack units keeping the result valid; polygon parts only."""
|
||||
snapped = set_precision(geometry, 1, mode='valid_output')
|
||||
return MultiPolygon(polygon_parts(snapped))
|
||||
|
||||
|
||||
def settle(geometry):
|
||||
"""A valid MultiPolygon of `geometry` (in pack units) on whole units."""
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
geometry = on_grid(MultiPolygon(polygon_parts(geometry)))
|
||||
for _ in range(3):
|
||||
if geometry.is_valid:
|
||||
break
|
||||
geometry = on_grid(make_valid(geometry))
|
||||
return geometry
|
||||
|
||||
|
||||
def rings_ok(polygons):
|
||||
return all(len(set(ring)) >= 3 for poly in polygons for ring in poly)
|
||||
|
||||
|
||||
def problems(polygons):
|
||||
"""Why decoded `polygons` ([[outer, *holes], …]) fail the pack contract."""
|
||||
if not polygons or not rings_ok(polygons):
|
||||
return 'ring with fewer than three distinct vertices'
|
||||
try:
|
||||
geometry = MultiPolygon([Polygon(p[0], p[1:]) for p in polygons])
|
||||
except Exception as error: # noqa: BLE001 - malformed input is a finding
|
||||
return f'unbuildable: {error}'
|
||||
if not geometry.is_valid:
|
||||
return shapely.is_valid_reason(geometry)
|
||||
areas = [part.area for part in geometry.geoms]
|
||||
if max(areas) > areas[0]:
|
||||
return 'largest part is not first'
|
||||
return None
|
||||
|
||||
|
||||
# ── packs ────────────────────────────────────────────────────────────────
|
||||
|
||||
def census_features():
|
||||
folder = DATA / 'us_census_places'
|
||||
decimals = json.loads((folder / 'index.json').read_text())['meta']['decimals']
|
||||
for path in sorted(folder.glob('[A-Z][A-Z].json')):
|
||||
pack = json.loads(path.read_text(encoding='utf-8'))
|
||||
yield path, pack, decimals
|
||||
|
||||
|
||||
def census_polygons(feature):
|
||||
return [[decode_deltas(r) for r in poly] for poly in feature['polygons']]
|
||||
|
||||
|
||||
def wof_features():
|
||||
folder = DATA / 'wof_neighborhoods'
|
||||
index = json.loads((folder / 'index.json').read_text(encoding='utf-8'))
|
||||
for tile in index['tiles']:
|
||||
pack = json.loads((folder / f'{tile["key"]}.json').read_text(encoding='utf-8'))
|
||||
for feature in pack['features']:
|
||||
yield feature, index['meta']['precision']
|
||||
|
||||
|
||||
def check(pack):
|
||||
bad = []
|
||||
total = 0
|
||||
if pack == 'census':
|
||||
for _, data, decimals in census_features():
|
||||
for feature in data['features']:
|
||||
total += 1
|
||||
why = problems(census_polygons(feature))
|
||||
if why:
|
||||
bad.append((feature['geoid'], feature['name'], why))
|
||||
else:
|
||||
for feature, precision in wof_features():
|
||||
total += 1
|
||||
polygons = [[decode_polyline(r) for r in poly] for poly in feature[7]]
|
||||
why = problems(polygons)
|
||||
if why:
|
||||
bad.append((feature[0], feature[1], why))
|
||||
for row in bad[:20]:
|
||||
print('invalid', *row)
|
||||
print(f'{pack}: {total} features, {len(bad)} failing')
|
||||
return not bad
|
||||
|
||||
|
||||
def repair_census():
|
||||
repaired = 0
|
||||
for path, data, decimals in census_features():
|
||||
changed = False
|
||||
for feature in data['features']:
|
||||
polygons = census_polygons(feature)
|
||||
if not problems(polygons):
|
||||
continue
|
||||
geometry = MultiPolygon([Polygon(p[0], p[1:]) for p in polygons if len(set(p[0])) >= 3])
|
||||
parts = ordered_parts(settle(geometry))
|
||||
feature['polygons'] = [
|
||||
[encode_deltas(open_ring(p.exterior.coords))]
|
||||
+ [encode_deltas(open_ring(r.coords)) for r in p.interiors]
|
||||
for p in parts
|
||||
]
|
||||
why = problems(census_polygons(feature))
|
||||
if why:
|
||||
raise ValueError(f'{feature["geoid"]} {feature["name"]}: still {why}')
|
||||
repaired += 1
|
||||
changed = True
|
||||
if changed:
|
||||
path.write_text(compact(data) + '\n', encoding='utf-8')
|
||||
print(f'us_census_places: {repaired} features repaired to valid GEOS topology')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('command', choices=['check', 'repair'])
|
||||
parser.add_argument('pack', choices=['census', 'wof'])
|
||||
args = parser.parse_args()
|
||||
require_pinned_tooling()
|
||||
if args.command == 'repair':
|
||||
if args.pack != 'census':
|
||||
parser.error('the WOF build repairs its own geometry')
|
||||
repair_census()
|
||||
sys.exit(0 if check('census') else 1)
|
||||
sys.exit(0 if check(args.pack) else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -244,6 +244,9 @@
|
||||
"server/providers/regional/http.js",
|
||||
"server/providers/regional/news.js",
|
||||
"server/providers/regional/place.js",
|
||||
"server/providers/regional/nominatimGate.js",
|
||||
"server/providers/regional/nominatimCache.js",
|
||||
"server/providers/regional/outlineSelect.js",
|
||||
"server/providers/regional/query.js",
|
||||
"server/providers/regional/weather-effects.js",
|
||||
"server/providers/regional/weather.js",
|
||||
@@ -1328,6 +1331,9 @@
|
||||
"src/annotations/hybridAnnotationRenderer.js",
|
||||
"src/annotations/index.js",
|
||||
"src/annotations/resolver.js",
|
||||
"src/annotations/outlineRungs.js",
|
||||
"src/sources/openFreeMapOutlines.js",
|
||||
"src/sources/nominatimOutlines.js",
|
||||
"src/annotations/screenAnnotationRenderer.js",
|
||||
"src/annotations/worldAnnotationRenderer.js",
|
||||
"src/app/catalog.js",
|
||||
@@ -1454,6 +1460,9 @@
|
||||
"src/data/motionModel.js",
|
||||
"src/data/naturalEarthRegions.js",
|
||||
"src/data/adminBoundaries.js",
|
||||
"src/data/placeBoundaries.js",
|
||||
"src/data/local_data/us_census_places/files.js",
|
||||
"src/data/local_data/wof_neighborhoods/files.js",
|
||||
"src/data/local_data/natural_earth/states_provinces.json",
|
||||
"src/data/local_data/natural_earth/countries.json",
|
||||
"src/data/local_data/us_census_counties/counties.json",
|
||||
@@ -2059,6 +2068,9 @@
|
||||
"src/data/bhoteKoshiShotPaths.js",
|
||||
"src/annotations/annotationResolver.js",
|
||||
"src/annotations/resolver.js",
|
||||
"src/annotations/outlineRungs.js",
|
||||
"src/sources/openFreeMapOutlines.js",
|
||||
"src/sources/nominatimOutlines.js",
|
||||
"src/app/catalog.js",
|
||||
"src/app/constructCatalog.js",
|
||||
"src/layers/wind/index.js",
|
||||
@@ -2134,6 +2146,9 @@
|
||||
"src/data/motionModel.js",
|
||||
"src/data/naturalEarthRegions.js",
|
||||
"src/data/adminBoundaries.js",
|
||||
"src/data/placeBoundaries.js",
|
||||
"src/data/local_data/us_census_places/files.js",
|
||||
"src/data/local_data/wof_neighborhoods/files.js",
|
||||
"src/data/local_data/natural_earth/states_provinces.json",
|
||||
"src/data/local_data/natural_earth/countries.json",
|
||||
"src/data/local_data/us_census_counties/counties.json",
|
||||
@@ -2527,8 +2542,16 @@
|
||||
"application-operations": {
|
||||
"exports": ["./application/operations"],
|
||||
"modules": [
|
||||
"src/sources/openFreeMap.js",
|
||||
"src/sources/vectorTiles.js",
|
||||
"src/sources/phaseTiming.js",
|
||||
"src/sources/militaryTileGeometry.js",
|
||||
"src/data/tomtomTiles.js",
|
||||
"src/annotations/annotationResolver.js",
|
||||
"src/annotations/resolver.js",
|
||||
"src/annotations/outlineRungs.js",
|
||||
"src/sources/openFreeMapOutlines.js",
|
||||
"src/sources/nominatimOutlines.js",
|
||||
"src/app/operations.js",
|
||||
"src/sources/sourceSlot.js",
|
||||
"src/app/surfaceServices.js",
|
||||
@@ -2539,6 +2562,9 @@
|
||||
"src/data/local_data/neighborhoods/san-francisco.json",
|
||||
"src/data/naturalEarthRegions.js",
|
||||
"src/data/adminBoundaries.js",
|
||||
"src/data/placeBoundaries.js",
|
||||
"src/data/local_data/us_census_places/files.js",
|
||||
"src/data/local_data/wof_neighborhoods/files.js",
|
||||
"src/data/local_data/natural_earth/states_provinces.json",
|
||||
"src/data/local_data/natural_earth/countries.json",
|
||||
"src/data/local_data/us_census_counties/counties.json",
|
||||
|
||||
@@ -23,6 +23,9 @@
|
||||
* node scripts/qa-voice-routing.mjs # both layers, :4415
|
||||
* node scripts/qa-voice-routing.mjs --layer routing --budget 120
|
||||
* node scripts/qa-voice-routing.mjs --layer behavior --url http://localhost:4173
|
||||
* node scripts/qa-voice-routing.mjs --layer behavior --behavior outlines # bundled outlines only
|
||||
* node scripts/qa-voice-routing.mjs --layer behavior --outlines [--headful] [--shots qa-shots/pr2] [--only "Paris|Stanford"]
|
||||
* # six-country outline table: rung, parts, cold/warm latency, Nominatim use
|
||||
*
|
||||
* House gotchas honored: camera.cancelFlight() before every teleport; puppeteer
|
||||
* suites must run sequentially with other harnesses (SwiftShader saturation);
|
||||
@@ -57,6 +60,11 @@ const LAYER = getOpt('--layer', 'all'); // routing | behavior | all
|
||||
const TURN_BUDGET = Number(getOpt('--budget', '120'));
|
||||
const PHRASES_PER_SESSION = Number(getOpt('--batch', '6'));
|
||||
const ONLY = getOpt('--only', null); // substring filter on phrase text
|
||||
const BEHAVIOR = process.argv.includes('--outlines')
|
||||
? 'acceptance'
|
||||
: getOpt('--behavior', 'all'); // all | outlines | acceptance
|
||||
const HEADFUL = process.argv.includes('--headful');
|
||||
const SHOTS_DIR = getOpt('--shots', null);
|
||||
|
||||
// ── Reporting ───────────────────────────────────────────────
|
||||
let pass = 0, fail = 0, skip = 0;
|
||||
@@ -374,13 +382,12 @@ async function runBehaviorLayer() {
|
||||
}
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
headless: HEADFUL ? false : 'new',
|
||||
...(CHROME_EXECUTABLE ? { executablePath: CHROME_EXECUTABLE } : {}),
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--use-gl=angle',
|
||||
'--use-angle=swiftshader',
|
||||
...(HEADFUL ? [] : ['--use-gl=angle', '--use-angle=swiftshader']),
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-web-security',
|
||||
'--disable-background-timer-throttling',
|
||||
@@ -392,9 +399,32 @@ async function runBehaviorLayer() {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1500, height: 950 });
|
||||
page.on('pageerror', (e) => console.log(` [page error] ${String(e).slice(0, 140)}`));
|
||||
// Network and resolver logs for the bundled-outline scenarios.
|
||||
const requests = [];
|
||||
const packTimes = [];
|
||||
const started = new Map();
|
||||
page.on('request', (req) => {
|
||||
requests.push(req.url());
|
||||
started.set(req, Date.now());
|
||||
});
|
||||
page.on('requestfinished', (req) => {
|
||||
if (/us_census_places|wof_neighborhoods/.test(req.url()))
|
||||
packTimes.push({
|
||||
file: req.url().split('/').pop().split('?')[0],
|
||||
ms: Date.now() - (started.get(req) ?? Date.now()),
|
||||
});
|
||||
});
|
||||
const resolverLog = [];
|
||||
page.on('console', (msg) => {
|
||||
const text = msg.text();
|
||||
if (text.startsWith('[Resolver]')) resolverLog.push(text);
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
// The outline table skips the first-run chooser, which covers the map.
|
||||
const startUrl = new URL(APP_URL);
|
||||
if (BEHAVIOR === 'acceptance') startUrl.searchParams.set('welcome', '0');
|
||||
await page.goto(startUrl.href, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForFunction(
|
||||
() => window.__godsEyeView?.viewer && window.__gevVoiceCommands?.runner && window.__gevAnnotations,
|
||||
{ timeout: 120000, polling: 250 },
|
||||
@@ -414,6 +444,15 @@ async function runBehaviorLayer() {
|
||||
});
|
||||
const settle = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
if (BEHAVIOR === 'outlines') {
|
||||
await runBundledOutlineScenarios({ page, run, settle, requests, resolverLog, packTimes });
|
||||
return;
|
||||
}
|
||||
if (BEHAVIOR === 'acceptance') {
|
||||
await runOutlineAcceptance({ page, run, settle, requests });
|
||||
return;
|
||||
}
|
||||
|
||||
// (1) fly_to_location lands near the target. Annotations follow IMMEDIATELY
|
||||
// while the camera is local — the resolver's proximity gate rejects far
|
||||
// targets BY DESIGN (annotating Austin from over the Alps must fail).
|
||||
@@ -700,6 +739,80 @@ async function runBehaviorLayer() {
|
||||
'behavior: far-target annotate is proximity-rejected by design',
|
||||
`drawn=${r?.drawn} failed=${r?.failed}`);
|
||||
|
||||
// (6c) Overpass-free outlines: bundled → OpenFreeMap tiles → guarded
|
||||
// Nominatim. Each case flies there, asks once, waits for the progressive
|
||||
// upgrade, and records which rung answered ([Outline]/[Resolver] trace).
|
||||
// No browser request may reach an Overpass host.
|
||||
const outlineTrace = [];
|
||||
const overpassHits = [];
|
||||
const onConsole = (msg) => {
|
||||
const text = msg.text();
|
||||
if (/^\[(Outline|Resolver)\]/.test(text)) outlineTrace.push(text);
|
||||
};
|
||||
const onRequest = (req) => {
|
||||
try {
|
||||
if (/overpass/i.test(new URL(req.url()).hostname)) overpassHits.push(req.url());
|
||||
} catch { /* data: and blob: URLs */ }
|
||||
};
|
||||
page.on('console', onConsole);
|
||||
page.on('request', onRequest);
|
||||
const outlineCases = [
|
||||
{ name: 'outline Paris', fly: 'Paris, France', ann: { type: 'area', target: 'Paris', label: 'Paris' } },
|
||||
// A world city outside the bundled packs exercises the Nominatim city rung.
|
||||
{ name: 'outline Toulouse', fly: 'Toulouse, France', ann: { type: 'area', target: 'Toulouse', label: 'Toulouse' } },
|
||||
{ name: 'outline the Texas Capitol grounds', fly: 'Texas State Capitol, Austin', ann: { type: 'area', target: 'the Texas Capitol grounds', label: 'Capitol grounds', entityKind: 'compound' } },
|
||||
{ name: 'outline Central Park', fly: 'Central Park, New York', ann: { type: 'area', target: 'Central Park', label: 'Central Park' } },
|
||||
{ name: 'highlight Congress Avenue', fly: 'Texas State Capitol, Austin', ann: { type: 'area', target: 'Congress Avenue, Austin', label: 'Congress Avenue', entityKind: 'street' } },
|
||||
// A bare highlight (no footprint flag, no entityKind) of a street name.
|
||||
{ name: 'highlight Congress Avenue (bare highlight)', fly: 'Texas State Capitol, Austin', ann: { type: 'highlight', target: 'Congress Avenue, Austin', label: 'Congress highlight' } },
|
||||
{ name: 'circle this building', fly: 'Texas State Capitol, Austin', ann: { type: 'area', target: 'this building', label: 'This building', entityKind: 'building', latitude: 30.27472, longitude: -97.74035 } },
|
||||
];
|
||||
const outlineTable = [];
|
||||
for (const c of outlineCases) {
|
||||
// Marks are left in place: later cases (fly_route) use the drawn route.
|
||||
await run('fly_to_location', { query: c.fly });
|
||||
// Wait for the flight to finish: the resolver biases by the view.
|
||||
let before = await camState();
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
await settle(1000);
|
||||
const now = await camState();
|
||||
const still = Math.abs(now.lat - before.lat) + Math.abs(now.lon - before.lon) < 1e-4 &&
|
||||
Math.abs(now.altKm - before.altKm) < 0.01;
|
||||
before = now;
|
||||
if (still) break;
|
||||
}
|
||||
outlineTrace.length = 0;
|
||||
r = await run('annotate_map', { annotations: [c.ann] });
|
||||
let state = null;
|
||||
for (let i = 0; i < 16; i += 1) {
|
||||
await settle(1500);
|
||||
state = await page.evaluate((label) => {
|
||||
const it = (window.__gevAnnotations?.list?.() || []).find((a) => a.label === label);
|
||||
if (!it) return null;
|
||||
return {
|
||||
ring: Array.isArray(it.ring) ? it.ring.length : 0,
|
||||
parts: Array.isArray(it.polygons) ? it.polygons.length : 0,
|
||||
synthesized: Boolean(it.synthesized),
|
||||
unavailable: Boolean(it.outlineUnavailable),
|
||||
kind: it.footprintKind || null,
|
||||
};
|
||||
}, c.ann.label);
|
||||
if (state?.ring >= 4 || state?.unavailable) break;
|
||||
}
|
||||
const rung =
|
||||
outlineTrace.map((line) => line.match(/→ (openfreemap|nominatim)/)?.[1]).find(Boolean) ||
|
||||
(outlineTrace.some((line) => /FINAL source=bundled|bundled/.test(line)) ? 'bundled' : 'none');
|
||||
const rendered = Boolean(state?.ring >= 4) && !state?.synthesized;
|
||||
outlineTable.push({ ask: c.name, rung, rendered, ...state });
|
||||
report((r?.drawn ?? 0) >= 1 && rendered, `behavior: "${c.name}" draws an outline`,
|
||||
`rung=${rung} ${JSON.stringify(state)}${rendered ? '' : ` trace=${outlineTrace.slice(-3).join(' | ').slice(0, 400)}`}`);
|
||||
}
|
||||
page.off('console', onConsole);
|
||||
page.off('request', onRequest);
|
||||
console.log(' outline rungs:', JSON.stringify(outlineTable));
|
||||
report(overpassHits.length === 0, 'behavior: outline asks send no request to an Overpass host',
|
||||
overpassHits.slice(0, 3).join(' '));
|
||||
|
||||
// Camera-verb scenarios: shed the heavy layers first — headless
|
||||
// SwiftShader drops to ~1 fps with fires+vessels loaded and every
|
||||
// motion assert starves (environment, not product).
|
||||
@@ -1000,6 +1113,8 @@ async function runBehaviorLayer() {
|
||||
'behavior: cockpit entry with Contacts off is refused, not half-entered',
|
||||
`ok=${gated?.ok} error=${String(gated?.error || '').slice(0, 90)} active=${cockpitAfterGate}`);
|
||||
|
||||
await runBundledOutlineScenarios({ page, run, settle, requests, resolverLog, packTimes });
|
||||
|
||||
const shotDir = path.join(ROOT, 'qa-shots');
|
||||
fs.mkdirSync(shotDir, { recursive: true });
|
||||
await page.screenshot({ path: path.join(shotDir, 'voice-behavior-final.png') });
|
||||
@@ -1010,6 +1125,259 @@ async function runBehaviorLayer() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Area asks that the bundled packs answer with no boundary lookup: a US
|
||||
* Census place, the San Francisco DataSF pack, and Who's On First
|
||||
* neighborhoods. Each teleports the camera over the place, draws the area,
|
||||
* and checks the outline, the pack that answered, and that no request went
|
||||
* to Overpass.
|
||||
*/
|
||||
const BUNDLED_OUTLINES = [
|
||||
{ target: 'Austin', lat: 30.2672, lon: -97.7431, pack: 'us-census' },
|
||||
{ target: 'the Mission District', lat: 37.7599, lon: -122.4148, pack: 'datasf', entityKind: 'district' },
|
||||
{ target: 'Notting Hill', lat: 51.5118, lon: -0.2046, pack: 'wof' },
|
||||
{ target: 'Williamsburg', lat: 40.7081, lon: -73.9571, pack: 'wof' },
|
||||
{ target: 'Le Marais', lat: 48.8575, lon: 2.358, pack: 'wof' },
|
||||
];
|
||||
|
||||
/**
|
||||
* A request to an Overpass host or a query to the app's Overpass proxy. The
|
||||
* proxy's /status probe answers locally (is an operator server configured?).
|
||||
*/
|
||||
function isOverpassRequest(url) {
|
||||
try {
|
||||
const { hostname, pathname } = new URL(url);
|
||||
return hostname.includes('overpass') ||
|
||||
(pathname.startsWith('/api/overpass') && pathname !== '/api/overpass/status');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runBundledOutlineScenarios({ page, run, settle, requests, resolverLog, packTimes }) {
|
||||
const timings = [];
|
||||
for (const scenario of BUNDLED_OUTLINES) {
|
||||
const label = `behavior: "outline ${scenario.target}" draws the bundled ${scenario.pack} outline`;
|
||||
await page.evaluate(({ lat, lon }) => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
viewer.camera.cancelFlight();
|
||||
viewer.camera.setView({
|
||||
destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({
|
||||
longitude: (lon * Math.PI) / 180,
|
||||
latitude: (lat * Math.PI) / 180,
|
||||
height: 9000,
|
||||
}),
|
||||
});
|
||||
}, scenario);
|
||||
await run('clear_annotations', {});
|
||||
await settle(1500);
|
||||
const requestsBefore = requests.length;
|
||||
const logBefore = resolverLog.length;
|
||||
const packsBefore = packTimes.length;
|
||||
const started = Date.now();
|
||||
const r = await run('annotate_map', {
|
||||
annotations: [{
|
||||
type: 'area',
|
||||
target: scenario.target,
|
||||
label: scenario.target,
|
||||
...(scenario.entityKind ? { entityKind: scenario.entityKind } : {}),
|
||||
}],
|
||||
});
|
||||
let ringPoints = 0;
|
||||
for (let i = 0; i < 40 && ringPoints < 4; i += 1) {
|
||||
ringPoints = await page.evaluate((target) => {
|
||||
const items = window.__gevAnnotations?.list?.() || [];
|
||||
const item = items.find((it) => it.label === target);
|
||||
return Array.isArray(item?.ring) ? item.ring.length : 0;
|
||||
}, scenario.target);
|
||||
if (ringPoints < 4) await settle(250);
|
||||
}
|
||||
const ms = Date.now() - started;
|
||||
const line = resolverLog.slice(logBefore).find((l) => l.includes('bundled')) || '';
|
||||
const pack = /\((us-census|datasf|wof|natural-earth)[,)]/.exec(line)?.[1] || 'none';
|
||||
const packFiles = packTimes.slice(packsBefore);
|
||||
const overpass = requests.slice(requestsBefore).filter(isOverpassRequest);
|
||||
timings.push({ target: scenario.target, ms, pack, overpass: overpass.length });
|
||||
report((r?.drawn ?? 0) >= 1 && ringPoints >= 4 && pack === scenario.pack && overpass.length === 0,
|
||||
label,
|
||||
`drawn=${r?.drawn} ring=${ringPoints} pack=${pack} outlineMs=${ms} overpass=${overpass.length} ` +
|
||||
`packFiles=${packFiles.map((f) => `${f.file}:${f.ms}ms`).join(',') || 'cached'}`);
|
||||
}
|
||||
await run('clear_annotations', {});
|
||||
const overpassTotal = requests.filter(isOverpassRequest);
|
||||
report(overpassTotal.length === 0, 'behavior: no request reached Overpass during the run',
|
||||
`overpass=${overpassTotal.length} ${overpassTotal.slice(0, 2).join(' ')}`);
|
||||
console.log(` bundled outline timings: ${JSON.stringify(timings)}`);
|
||||
console.log(` pack files loaded: ${packTimes.map((f) => `${f.file}:${f.ms}ms`).join(' ')}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Outline acceptance across six countries: each ask teleports the camera
|
||||
* over the place, draws the area once cold and once warm (after clearing),
|
||||
* and records the rung that answered, the parts drawn, the latency, and how
|
||||
* many public Nominatim requests the server spent (its daily count in
|
||||
* .gev-cache/nominatim/state.json). Nothing may reach Overpass.
|
||||
*/
|
||||
const OUTLINE_ASKS = [
|
||||
// group, target, camera lat/lon/height(m), extra annotation args, shot name
|
||||
['country', 'Switzerland', 46.8, 8.2, 1_200_000],
|
||||
['country', 'Iran', 32.4, 53.7, 3_500_000],
|
||||
['state', 'Texas', 31.0, -99.0, 2_000_000],
|
||||
['state', 'Bavaria', 48.9, 11.4, 900_000],
|
||||
['city', 'Austin', 30.2672, -97.7431, 60_000, {}, 'austin-city'],
|
||||
['city', 'Paris', 48.8566, 2.3522, 40_000],
|
||||
['city', 'Tokyo', 35.6762, 139.6503, 90_000],
|
||||
['city', 'São Paulo', -23.5505, -46.6333, 90_000],
|
||||
['city', 'Mumbai', 19.076, 72.8777, 60_000],
|
||||
['city', 'Lagos', 6.5244, 3.3792, 60_000],
|
||||
['neighborhood', 'the Mission District', 37.7599, -122.4148, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'the Marina', 37.8037, -122.4368, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'Notting Hill', 51.5118, -0.2046, 8000, { entityKind: 'district' }, 'notting-hill'],
|
||||
['neighborhood', 'Williamsburg', 40.7081, -73.9571, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'SoHo', 40.7233, -74.003, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'Le Marais', 48.8575, 2.358, 8000, { entityKind: 'district' }],
|
||||
['neighborhood', 'Shibuya', 35.6595, 139.7005, 8000, { entityKind: 'district' }],
|
||||
['landmark', 'the Texas Capitol grounds', 30.2747, -97.7404, 3000, { entityKind: 'compound' }],
|
||||
['landmark', 'Central Park', 40.7829, -73.9654, 6000, {}, 'central-park'],
|
||||
['landmark', 'the Eiffel Tower', 48.8584, 2.2945, 3000],
|
||||
['landmark', 'Stanford', 37.4275, -122.1697, 8000],
|
||||
['landmark', 'Stanford University', 37.4275, -122.1697, 8000],
|
||||
['landmark', 'Lady Bird Lake', 30.2638, -97.7526, 6000],
|
||||
['street', 'Congress Avenue', 30.2672, -97.7425, 3000, { entityKind: 'street' }, 'congress-avenue'],
|
||||
['street', 'Lombard Street', 37.8021, -122.4187, 3000, { entityKind: 'street' }],
|
||||
['building', 'this building', 30.27472, -97.74035, 1200,
|
||||
{ entityKind: 'building', latitude: 30.27472, longitude: -97.74035 }, 'building'],
|
||||
].map(([group, target, lat, lon, height, extra = {}, shot = null]) => ({ group, target, lat, lon, height, extra, shot }));
|
||||
|
||||
const NOMINATIM_STATE = path.join(ROOT, '.gev-cache', 'nominatim', 'state.json');
|
||||
function nominatimCount() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(NOMINATIM_STATE, 'utf8')).count || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function runOutlineAcceptance({ page, run, settle, requests }) {
|
||||
const trace = [];
|
||||
page.on('console', (msg) => {
|
||||
const text = msg.text();
|
||||
if (/^\[(Outline|Resolver)\]/.test(text)) trace.push(text);
|
||||
});
|
||||
const nominatimStart = nominatimCount();
|
||||
const requestsStart = requests.length;
|
||||
if (SHOTS_DIR) fs.mkdirSync(path.resolve(ROOT, SHOTS_DIR), { recursive: true });
|
||||
|
||||
const teleport = (ask) => page.evaluate(({ lat, lon, height }) => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
viewer.camera.cancelFlight();
|
||||
viewer.camera.setView({
|
||||
destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({
|
||||
longitude: (lon * Math.PI) / 180,
|
||||
latitude: (lat * Math.PI) / 180,
|
||||
height,
|
||||
}),
|
||||
});
|
||||
}, ask);
|
||||
|
||||
const rungOf = (lines, state) => {
|
||||
for (const line of lines) {
|
||||
const m = /→ (openfreemap|nominatim)\b/.exec(line) ||
|
||||
/bundled → ([a-z-]+)/.exec(line) ||
|
||||
/\((us-census|datasf|wof|natural-earth),/.exec(line);
|
||||
if (m) return m[1];
|
||||
}
|
||||
if (state?.synthesized) return 'approximate';
|
||||
return state?.ring >= 4 ? 'other' : 'point';
|
||||
};
|
||||
|
||||
const once = async (ask) => {
|
||||
await run('clear_annotations', {});
|
||||
await teleport(ask);
|
||||
await settle(1200);
|
||||
trace.length = 0;
|
||||
const before = nominatimCount();
|
||||
const label = `QA ${ask.target}`;
|
||||
const started = Date.now();
|
||||
const r = await run('annotate_map', {
|
||||
annotations: [{ type: 'area', target: ask.target, label, ...ask.extra }],
|
||||
});
|
||||
let state = null;
|
||||
for (let i = 0; i < 120; i += 1) {
|
||||
state = await page.evaluate((l) => {
|
||||
const it = (window.__gevAnnotations?.list?.() || []).find((a) => a.label === l);
|
||||
if (!it) return null;
|
||||
return {
|
||||
ring: Array.isArray(it.ring) ? it.ring.length : 0,
|
||||
parts: Array.isArray(it.polygons) && it.polygons.length ? it.polygons.length : (it.ring ? 1 : 0),
|
||||
synthesized: Boolean(it.synthesized),
|
||||
unavailable: Boolean(it.outlineUnavailable),
|
||||
pending: Boolean(it.pendingOutline),
|
||||
};
|
||||
}, label);
|
||||
if (state && !state.pending) break;
|
||||
if (!state && r?.drawn === 0 && i > 8) break;
|
||||
await settle(250);
|
||||
}
|
||||
const ms = Date.now() - started;
|
||||
return {
|
||||
ms,
|
||||
drawn: r?.drawn ?? 0,
|
||||
state,
|
||||
rung: rungOf(trace, state),
|
||||
nominatim: nominatimCount() - before,
|
||||
error: r?.error || r?.results?.find?.((x) => x?.error)?.error || null,
|
||||
};
|
||||
};
|
||||
|
||||
const table = [];
|
||||
// --only "Paris|Stanford" reruns a few rows.
|
||||
const only = ONLY ? ONLY.split('|') : null;
|
||||
for (const ask of OUTLINE_ASKS.filter((a) => !only || only.some((o) => a.target.includes(o)))) {
|
||||
const cold = await once(ask);
|
||||
if (SHOTS_DIR && ask.shot) {
|
||||
await settle(HEADFUL ? 6000 : 2500);
|
||||
await page.screenshot({ path: path.resolve(ROOT, SHOTS_DIR, `${ask.shot}.png`) });
|
||||
}
|
||||
const warm = await once(ask);
|
||||
const rendered = Boolean(cold.state?.ring >= 4) && !cold.state?.synthesized;
|
||||
const row = {
|
||||
group: ask.group,
|
||||
ask: ask.target,
|
||||
rung: cold.rung,
|
||||
rendered,
|
||||
approximate: Boolean(cold.state?.synthesized),
|
||||
parts: cold.state?.parts ?? 0,
|
||||
coldMs: cold.ms,
|
||||
warmMs: warm.ms,
|
||||
warmRung: warm.rung,
|
||||
nominatim: cold.nominatim + warm.nominatim,
|
||||
};
|
||||
table.push(row);
|
||||
const detail = `rung=${row.rung} parts=${row.parts} cold=${row.coldMs}ms warm=${row.warmMs}ms ` +
|
||||
`nominatim=${row.nominatim}${row.approximate ? ' approximate' : ''}` +
|
||||
(rendered ? '' : ` trace=${trace.slice(-2).join(' | ').slice(0, 300)}`);
|
||||
if (rendered) report(true, `outlines: ${ask.group} "${ask.target}"`, detail);
|
||||
else skipped(`outlines: ${ask.group} "${ask.target}" (no exact outline)`, detail);
|
||||
}
|
||||
await run('clear_annotations', {});
|
||||
|
||||
const overpass = requests.slice(requestsStart).filter(isOverpassRequest);
|
||||
report(overpass.length === 0, 'outlines: no request reached Overpass during the run',
|
||||
`overpass=${overpass.length} ${overpass.slice(0, 2).join(' ')}`);
|
||||
const used = nominatimCount() - nominatimStart;
|
||||
report(used <= 30, 'outlines: public Nominatim requests within the run budget (30)', `used=${used}`);
|
||||
|
||||
console.log('\n | Group | Ask | Rung | Outline | Parts | Cold ms | Warm ms | Nominatim |');
|
||||
console.log(' |---|---|---|---|---|---|---|---|');
|
||||
for (const row of table) {
|
||||
const outline = row.rendered ? 'yes' : row.approximate ? 'approximate' : 'no (point)';
|
||||
console.log(` | ${row.group} | ${row.ask} | ${row.rung} | ${outline} | ${row.parts} | ${row.coldMs} | ${row.warmMs} | ${row.nominatim} |`);
|
||||
}
|
||||
if (SHOTS_DIR)
|
||||
fs.writeFileSync(path.resolve(ROOT, SHOTS_DIR, 'outline-table.json'), JSON.stringify(table, null, 2));
|
||||
}
|
||||
|
||||
// ── Main ────────────────────────────────────────────────────
|
||||
console.log('\nVoice-surface QA harness');
|
||||
console.log(` App URL : ${APP_URL}`);
|
||||
|
||||
Reference in new issue
Block a user