From 5d8f2b878f5ef204b5efb4519a7fbbbc3699b59f Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 6 Nov 2026 15:47:36 +0400 Subject: [PATCH] chore(datacenters): drop the one-off Norway refresh script The data refresh is committed and its provenance (extract, timestamp, MD5, filter) stays in the dataset README. The script needed osmium-tool and could only re-verify while Geofabrik still listed that extract, so it goes with its test. --- changelog.d/953-datacenters-norway-refresh.md | 6 +- scripts/build-datacenters-norway.py | 259 ------------------ src/data/datacentersNorwayRefresh.test.mjs | 95 ------- src/data/local_data/datacenters/README.md | 12 +- 4 files changed, 3 insertions(+), 369 deletions(-) delete mode 100644 scripts/build-datacenters-norway.py delete mode 100644 src/data/datacentersNorwayRefresh.test.mjs diff --git a/changelog.d/953-datacenters-norway-refresh.md b/changelog.d/953-datacenters-norway-refresh.md index 48a019605..5db897234 100644 --- a/changelog.d/953-datacenters-norway-refresh.md +++ b/changelog.d/953-datacenters-norway-refresh.md @@ -1,6 +1,4 @@ - Refresh Norway's datacenters from a Geofabrik OpenStreetMap extract (2026-10-05). Troll Housing joins the layer (4,352 features) and four - Norwegian sites pick up current OSM tags; no site moved or was removed. - `scripts/build-datacenters-norway.py` reproduces or `--verify`s the merge, - touching only features inside Geofabrik's Norway boundary, and the dataset - README records the extract, checksum and filter (trygvehhh, #953). + Norwegian sites pick up current OSM tags; no site moved or was removed + (trygvehhh, #953). diff --git a/scripts/build-datacenters-norway.py b/scripts/build-datacenters-norway.py deleted file mode 100644 index 3c5eb92dd..000000000 --- a/scripts/build-datacenters-norway.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python3 -"""Maintainer-only Norway refresh of the bundled datacenter extract (osmium-tool). - -Downloads a dated Geofabrik Norway extract, verifies its MD5, filters -telecom=data_center / building=data_center with osmium, and merges the result -into datacenters.geojsonl: features inside the Geofabrik Norway boundary are -added, updated or removed once no longer tagged; every other line is kept -byte-for-byte. - -Geofabrik keeps only its newest dated extract, so --extract defaults to the -current one and a recorded extract can be re-verified only while it is listed. -""" -import argparse -import hashlib -import json -import math -from pathlib import Path -import shutil -import subprocess -import sys -import urllib.request - -ROOT = Path(__file__).resolve().parent.parent -GEOFABRIK = 'https://download.geofabrik.de/europe/' -DATASET = ROOT / 'src/data/local_data/datacenters/datacenters.geojsonl' -CONTACT_TAGS = {'email', 'phone', 'fax', 'mobile', 'whatsapp', 'contact:email', 'contact:phone', - 'contact:fax', 'contact:mobile', 'contact:whatsapp'} -SAME_GEOMETRY_METERS = 0.05 -TIMEOUT_SECONDS = 60 - - -def latest_extract(): - """Resolve norway-latest to the dated name Geofabrik redirects it to.""" - request = urllib.request.Request(GEOFABRIK + 'norway-latest.osm.pbf', method='HEAD') - with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS) as response: - name = response.geturl().rsplit('/', 1)[-1] - if not name.startswith('norway-') or not name.endswith('.osm.pbf') or name == 'norway-latest.osm.pbf': - raise RuntimeError('Unexpected Geofabrik latest target: %s' % name) - return name[:-len('.osm.pbf')] - - -def fetch(url, target): - if not target.exists(): - with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response, \ - open(str(target) + '.part', 'wb') as out: - shutil.copyfileobj(response, out) - Path(str(target) + '.part').rename(target) - return target - - -def md5(path): - digest = hashlib.md5() - with open(path, 'rb') as handle: - for block in iter(lambda: handle.read(1 << 20), b''): - digest.update(block) - return digest.hexdigest() - - -def read_poly(path): - """Geofabrik .poly -> list of (ring, is_hole).""" - rings, ring, hole = [], None, False - for raw in path.read_text(encoding='utf-8').splitlines()[1:]: - line = raw.strip() - if ring is None: - if line and line != 'END': - ring, hole = [], line.startswith('!') - elif line == 'END': - rings.append((ring, hole)) - ring = None - elif line: - lon, lat = (float(v) for v in line.split()) - ring.append((lon, lat)) - return rings - - -def inside(rings, lon, lat): - def in_ring(ring): - hit = False - for (x1, y1), (x2, y2) in zip(ring, ring[1:] + ring[:1]): - if (y1 > lat) != (y2 > lat) and lon < x1 + (lat - y1) * (x2 - x1) / (y2 - y1): - hit = not hit - return hit - return any(in_ring(r) for r, h in rings if not h) and not any(in_ring(r) for r, h in rings if h) - - -def points(coordinates): - if isinstance(coordinates[0], (int, float)): - return [coordinates] - return [p for part in coordinates for p in points(part)] - - -def osm_key(feature): - osm_id = feature['properties']['osm_id'] - if osm_id < 0: - return 'r%d' % -osm_id - return ('n' if feature['geometry']['type'] == 'Point' else 'w') + str(osm_id) - - -def export(osmium, pbf, cache, extract): - """Exported features by key, and the keys of every tagged object in the extract.""" - filtered = cache / (extract + '-datacenters.osm.pbf') - listed = cache / (extract + '-datacenters.opl') - exported = cache / (extract + '-datacenters.geojsonseq') - subprocess.run([osmium, 'tags-filter', '-O', '-o', str(filtered), str(pbf), - 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) - subprocess.run([osmium, 'tags-filter', '-O', '-R', '-f', 'opl', '-o', str(listed), str(filtered), - 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) - tagged = {line.split(' ', 1)[0] for line in listed.read_text(encoding='utf-8').splitlines() if line} - subprocess.run([osmium, 'export', '-O', '-f', 'geojsonseq', '--add-unique-id=type_id', - '-o', str(exported), str(filtered)], check=True) - fresh = {} - for line in exported.read_text(encoding='utf-8').splitlines(): - line = line.lstrip('\x1e').strip() - if not line: - continue - feature = json.loads(line) - tags = {k: v for k, v in feature['properties'].items() if k not in CONTACT_TAGS} - if tags.get('telecom') != 'data_center' and tags.get('building') != 'data_center': - continue # untagged member ways exported only for geometry - key = feature['id'] - geometry = feature['geometry'] - if key[0] == 'a': # osmium area id: way * 2 or relation * 2 + 1 - number = int(key[1:]) - key = ('w%d' % (number // 2)) if number % 2 == 0 else ('r%d' % (number // 2)) - if key[0] == 'r': - tags = {**tags, 'type': 'multipolygon'} - if geometry['type'] == 'MultiPolygon' and len(geometry['coordinates']) == 1: - geometry = {'type': 'Polygon', 'coordinates': geometry['coordinates'][0]} - elif key in fresh or geometry['type'] == 'LineString': - continue # closed ways are taken from their area export - fresh[key] = {'geometry': geometry, 'tags': tags} - return fresh, tagged - - -def same_geometry(old, new): - """Same type and rings within tolerance; ring start, winding and part or hole order may differ.""" - if old['type'] != new['type']: - return False - - def meters(p, q): - dx = (p[0] - q[0]) * 111320 * math.cos(math.radians(p[1])) - return math.hypot(dx, (p[1] - q[1]) * 110540) - - def same_line(a, b): - return len(a) == len(b) and all(meters(p, q) <= SAME_GEOMETRY_METERS for p, q in zip(a, b)) - - def same_ring(a, b): - a, b = a[:-1], b[:-1] # closed rings repeat their first vertex - return len(a) == len(b) and \ - any(same_line(a[i:] + a[:i], c) for c in (b, b[::-1]) for i in range(len(a))) - - def same_parts(a, b, same): - rest = list(b) - for part in a: - match = next((i for i, other in enumerate(rest) if same(part, other)), None) - if match is None: - return False - rest.pop(match) - return not rest - - def same_polygon(a, b): - return same_ring(a[0], b[0]) and same_parts(a[1:], b[1:], same_ring) - - a, b = old['coordinates'], new['coordinates'] - if old['type'] == 'Point': - return same_line([a], [b]) - if old['type'] == 'LineString': - return same_line(a, b) - if old['type'] == 'Polygon': - return same_polygon(a, b) - if old['type'] == 'MultiPolygon': - return same_parts(a, b, same_polygon) - return a == b - - -def feature_line(key, sequence, fresh): - osm_id = int(key[1:]) * (-1 if key[0] == 'r' else 1) - feature = {'id': osm_id, 'type': 'Feature', 'geometry': fresh['geometry'], - 'properties': {'id': sequence, 'tags': fresh['tags'], 'type': 'data_center', - 'osm_id': osm_id}} - return json.dumps(feature, separators=(',', ':'), ensure_ascii=False) - - -def merge(lines, fresh, rings, tagged): - out, known, sequence = [], set(), 0 - report = {'kept': 0, 'updated': [], 'removed': [], 'added': [], 'unexported': [], 'skipped': []} - for line in lines: - feature = json.loads(line) - sequence = max(sequence, feature['properties']['id']) - key = osm_key(feature) - known.add(key) - if key in fresh: - old_tags = feature['properties']['tags'] - if old_tags == fresh[key]['tags'] and same_geometry(feature['geometry'], fresh[key]['geometry']): - out.append(line) - report['kept'] += 1 - else: - out.append(feature_line(key, feature['properties']['id'], fresh[key])) - report['updated'].append(key) - continue - if key in tagged: # still tagged, but osmium built no usable geometry: keep the old line - report['unexported'].append(key) - out.append(line) - continue - lon, lat = points(feature['geometry']['coordinates'])[0][:2] - if inside(rings, lon, lat): - report['removed'].append(key) - continue - out.append(line) - for key in sorted(set(fresh) - known, key=lambda k: (k[0], int(k[1:]))): - sequence += 1 - out.append(feature_line(key, sequence, fresh[key])) - report['added'].append(key) - # newly tagged, but osmium built no usable geometry: nothing to add yet - report['skipped'] = sorted(set(tagged) - set(fresh) - known, key=lambda k: (k[0], int(k[1:]))) - if len({osm_key(json.loads(line)) for line in out}) != len(out): - raise ValueError('Duplicate OSM identity') - return out, report - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--cache', type=Path, required=True) - parser.add_argument('--osmium', default=shutil.which('osmium') or 'osmium') - parser.add_argument('--extract', default='latest', - help="dated Geofabrik name such as norway-261005, or 'latest'") - parser.add_argument('--verify', action='store_true') - args = parser.parse_args() - args.cache.mkdir(parents=True, exist_ok=True) - extract = latest_extract() if args.extract == 'latest' else args.extract - pbf = fetch(GEOFABRIK + extract + '.osm.pbf', args.cache / (extract + '.osm.pbf')) - expected = fetch(GEOFABRIK + extract + '.osm.pbf.md5', - args.cache / (extract + '.osm.pbf.md5')).read_text(encoding='utf-8').split()[0] - actual = md5(pbf) - if actual != expected: - raise RuntimeError('MD5 mismatch for %s: %s != %s' % (pbf.name, actual, expected)) - rings = read_poly(fetch(GEOFABRIK + 'norway.poly', args.cache / 'norway.poly')) - fresh, tagged = export(args.osmium, pbf, args.cache, extract) - current = [line for line in DATASET.read_text(encoding='utf-8').splitlines() if line.strip()] - merged, report = merge(current, fresh, rings, tagged) - if report['unexported']: - print('warning: kept the old line of tagged features osmium did not export: %s' - % ', '.join(report['unexported']), file=sys.stderr) - if report['skipped']: - print('warning: did not add newly tagged features osmium did not export: %s' - % ', '.join(report['skipped']), file=sys.stderr) - payload = '\n'.join(merged) + '\n' - if args.verify: - if payload != DATASET.read_text(encoding='utf-8'): - raise SystemExit('datacenters.geojsonl is not the %s merge' % extract) - else: - with open(DATASET, 'w', encoding='utf-8', newline='\n') as handle: - handle.write(payload) - print(json.dumps({'extract': extract, 'md5': actual, 'norway': len(fresh), - 'features': len(merged), **report})) - - -if __name__ == '__main__': - main() diff --git a/src/data/datacentersNorwayRefresh.test.mjs b/src/data/datacentersNorwayRefresh.test.mjs deleted file mode 100644 index 99278769e..000000000 --- a/src/data/datacentersNorwayRefresh.test.mjs +++ /dev/null @@ -1,95 +0,0 @@ -import test from 'node:test'; -import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; -import { fileURLToPath } from 'node:url'; - -const SCRIPT = fileURLToPath( - new URL('../../scripts/build-datacenters-norway.py', import.meta.url), -); - -function python(t, source) { - const result = spawnSync('python3', ['-c', source, SCRIPT], { - encoding: 'utf8', - }); - if (result.error?.code === 'ENOENT') return t.skip('python3 is not installed'); - assert.equal(result.status, 0, result.stderr); -} - -test('Norway refresh compares rings, not vertex sets', (t) => { - python( - t, - String.raw` -import runpy, sys -same = runpy.run_path(sys.argv[1])['same_geometry'] -poly = lambda *rings: {'type': 'Polygon', 'coordinates': list(rings)} -multi = lambda *polys: {'type': 'MultiPolygon', 'coordinates': [p['coordinates'] for p in polys]} -a, b, c, d, e = [0, 0], [4, 0], [4, 4], [0, 4], [2, 1] -assert not same(poly([a, b, c, d, e, a]), poly([a, b, e, c, d, a])), 'reordered edges' -assert same(poly([a, b, c, d, e, a]), poly([c, b, a, e, d, c])), 'ring start and winding' -assert same(poly([a, b, c, d, a]), poly([[0, 1e-7], b, c, d, [0, 1e-7]])), 'tolerance' -outer = [[0, 0], [9, 0], [9, 9], [0, 9], [0, 0]] -one, two = [[1, 1], [2, 1], [2, 2], [1, 1]], [[5, 5], [6, 5], [6, 6], [5, 5]] -assert same(poly(outer, one, two), poly(outer, two, one)), 'hole order' -assert not same(poly(outer, one), poly(one, outer)), 'outer and inner roles swapped' -assert same(multi(poly(one), poly(two)), multi(poly(two), poly(one))), 'part order' -p, q, r, s, u, v = [0, 0], [1, 0], [0, 1], [5, 5], [6, 5], [5, 6] -assert not same(multi(poly([p, q, r, p]), poly([s, u, v, s])), - multi(poly([p, q, s, p]), poly([r, u, v, r]))), 'vertices moved between parts' -`, - ); -}); - -test('Norway refresh keeps a still-tagged feature osmium did not export', (t) => { - python( - t, - String.raw` -import json, runpy, sys -merge = runpy.run_path(sys.argv[1])['merge'] -square = [[[10, 60], [10.01, 60], [10.01, 60.01], [10, 60.01], [10, 60]]] -def line(osm_id, sequence): - return json.dumps({'id': osm_id, 'type': 'Feature', 'geometry': {'type': 'Polygon', 'coordinates': square}, - 'properties': {'id': sequence, 'tags': {'building': 'data_center'}, - 'type': 'data_center', 'osm_id': osm_id}}) -norway = [([(0, 0), (20, 0), (20, 80), (0, 80)], False)] -lines = [line(7, 1), line(8, 2)] -fresh = {'w13': {'geometry': {'type': 'Polygon', 'coordinates': square}, 'tags': {'building': 'data_center'}}} -out, report = merge(lines, fresh, norway, {'w7', 'w12', 'w13'}) -assert out[0] == lines[0] and len(out) == 2, out -assert report['unexported'] == ['w7'] and report['removed'] == ['w8'], report -assert report['skipped'] == ['w12'] and report['added'] == ['w13'], report -`, - ); -}); - -test('Norway refresh lists tagged objects apart from the exported geometry', (t) => { - python( - t, - String.raw` -import json, runpy, subprocess, sys, tempfile -from pathlib import Path -export = runpy.run_path(sys.argv[1])['export'] -tags = {'building': 'data_center'} -ring = [[10, 60], [10.01, 60], [10.01, 60.01], [10, 60]] -features = [ - {'id': 'n1', 'properties': {'telecom': 'data_center'}, 'geometry': {'type': 'Point', 'coordinates': [10, 60]}}, - {'id': 'w5', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring}}, - {'id': 'a10', 'properties': tags, 'geometry': {'type': 'MultiPolygon', 'coordinates': [[ring]]}}, - {'id': 'w7', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring[:3]}}, -] -outputs = { - '.opl': 'n1 v1 Ttelecom=data_center x10 y60\nw5 v1 Tbuilding=data_center Nn2,n3,n4,n2\n' - 'w7 v1 Tbuilding=data_center Nn2,n3,n4\nr9 v1 Ttelecom=data_center Mw8@outer\n', - '.geojsonseq': ''.join(chr(30) + json.dumps({'type': 'Feature', **f}) + '\n' for f in features), -} -def run(args, check=False): - target = Path(args[args.index('-o') + 1]) - target.write_text(outputs.get(target.suffix, ''), encoding='utf-8') -subprocess.run = run -with tempfile.TemporaryDirectory() as cache: - fresh, tagged = export('osmium', Path(cache) / 'norway-test.osm.pbf', Path(cache), 'norway-test') -assert sorted(fresh) == ['n1', 'w5'], fresh -assert fresh['w5']['geometry'] == {'type': 'Polygon', 'coordinates': [ring]}, fresh['w5'] -assert tagged == {'n1', 'w5', 'w7', 'r9'}, tagged -`, - ); -}); diff --git a/src/data/local_data/datacenters/README.md b/src/data/local_data/datacenters/README.md index 86f573c61..c360ee527 100644 --- a/src/data/local_data/datacenters/README.md +++ b/src/data/local_data/datacenters/README.md @@ -26,14 +26,4 @@ reflects OSM as of the extract below, everything else the original snapshot. - Extract: Geofabrik `norway-261005.osm.pbf`, OSM data 2026-10-05T20:21:35Z, MD5 `1d2cfdeb98980fce11b8891d0f9146c0` - Filter: `osmium tags-filter nwr/telecom=data_center nwr/building=data_center` -- Contact tags are dropped as above; unchanged features keep their original - line, so geometry is only rewritten when tags or rings actually changed. - A feature is removed only once it is no longer tagged; one still tagged but - without an exportable geometry keeps its line and is reported as unexported, - and a newly tagged one osmium cannot export is reported as skipped. -- Refresh: `python3 scripts/build-datacenters-norway.py --cache ` uses - Geofabrik's current Norway extract and prints the dated name it used; record - that name, its OSM timestamp and MD5 here. Requires `osmium-tool`. -- Check: add `--extract --verify` to compare without writing. - Geofabrik keeps only its newest dated extract, so a recorded extract can be - re-verified only while it is still listed; the MD5 above identifies it. +- Contact tags are dropped as above.