mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-07 11:54:02 +00:00
chore(datacenters): drop the one-off Norway refresh script
The data refresh is committed and its provenance (extract, timestamp, MD5, filter) stays in the dataset README. The script needed osmium-tool and could only re-verify while Geofabrik still listed that extract, so it goes with its test.
This commit is contained in:
German Khokhlov committed
1 parent
47ea4aff1d
commit
5d8f2b878f
4 files changed
+3
-369
No files matched your search
@@ -1,6 +1,4 @@
|
||||
- Refresh Norway's datacenters from a Geofabrik OpenStreetMap extract
|
||||
(2026-10-05). Troll Housing joins the layer (4,352 features) and four
|
||||
Norwegian sites pick up current OSM tags; no site moved or was removed.
|
||||
`scripts/build-datacenters-norway.py` reproduces or `--verify`s the merge,
|
||||
touching only features inside Geofabrik's Norway boundary, and the dataset
|
||||
README records the extract, checksum and filter (trygvehhh, #953).
|
||||
Norwegian sites pick up current OSM tags; no site moved or was removed
|
||||
(trygvehhh, #953).
|
||||
@@ -1,259 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maintainer-only Norway refresh of the bundled datacenter extract (osmium-tool).
|
||||
|
||||
Downloads a dated Geofabrik Norway extract, verifies its MD5, filters
|
||||
telecom=data_center / building=data_center with osmium, and merges the result
|
||||
into datacenters.geojsonl: features inside the Geofabrik Norway boundary are
|
||||
added, updated or removed once no longer tagged; every other line is kept
|
||||
byte-for-byte.
|
||||
|
||||
Geofabrik keeps only its newest dated extract, so --extract defaults to the
|
||||
current one and a recorded extract can be re-verified only while it is listed.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
GEOFABRIK = 'https://download.geofabrik.de/europe/'
|
||||
DATASET = ROOT / 'src/data/local_data/datacenters/datacenters.geojsonl'
|
||||
CONTACT_TAGS = {'email', 'phone', 'fax', 'mobile', 'whatsapp', 'contact:email', 'contact:phone',
|
||||
'contact:fax', 'contact:mobile', 'contact:whatsapp'}
|
||||
SAME_GEOMETRY_METERS = 0.05
|
||||
TIMEOUT_SECONDS = 60
|
||||
|
||||
|
||||
def latest_extract():
|
||||
"""Resolve norway-latest to the dated name Geofabrik redirects it to."""
|
||||
request = urllib.request.Request(GEOFABRIK + 'norway-latest.osm.pbf', method='HEAD')
|
||||
with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS) as response:
|
||||
name = response.geturl().rsplit('/', 1)[-1]
|
||||
if not name.startswith('norway-') or not name.endswith('.osm.pbf') or name == 'norway-latest.osm.pbf':
|
||||
raise RuntimeError('Unexpected Geofabrik latest target: %s' % name)
|
||||
return name[:-len('.osm.pbf')]
|
||||
|
||||
|
||||
def fetch(url, target):
|
||||
if not target.exists():
|
||||
with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response, \
|
||||
open(str(target) + '.part', 'wb') as out:
|
||||
shutil.copyfileobj(response, out)
|
||||
Path(str(target) + '.part').rename(target)
|
||||
return target
|
||||
|
||||
|
||||
def md5(path):
|
||||
digest = hashlib.md5()
|
||||
with open(path, 'rb') as handle:
|
||||
for block in iter(lambda: handle.read(1 << 20), b''):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def read_poly(path):
|
||||
"""Geofabrik .poly -> list of (ring, is_hole)."""
|
||||
rings, ring, hole = [], None, False
|
||||
for raw in path.read_text(encoding='utf-8').splitlines()[1:]:
|
||||
line = raw.strip()
|
||||
if ring is None:
|
||||
if line and line != 'END':
|
||||
ring, hole = [], line.startswith('!')
|
||||
elif line == 'END':
|
||||
rings.append((ring, hole))
|
||||
ring = None
|
||||
elif line:
|
||||
lon, lat = (float(v) for v in line.split())
|
||||
ring.append((lon, lat))
|
||||
return rings
|
||||
|
||||
|
||||
def inside(rings, lon, lat):
|
||||
def in_ring(ring):
|
||||
hit = False
|
||||
for (x1, y1), (x2, y2) in zip(ring, ring[1:] + ring[:1]):
|
||||
if (y1 > lat) != (y2 > lat) and lon < x1 + (lat - y1) * (x2 - x1) / (y2 - y1):
|
||||
hit = not hit
|
||||
return hit
|
||||
return any(in_ring(r) for r, h in rings if not h) and not any(in_ring(r) for r, h in rings if h)
|
||||
|
||||
|
||||
def points(coordinates):
|
||||
if isinstance(coordinates[0], (int, float)):
|
||||
return [coordinates]
|
||||
return [p for part in coordinates for p in points(part)]
|
||||
|
||||
|
||||
def osm_key(feature):
|
||||
osm_id = feature['properties']['osm_id']
|
||||
if osm_id < 0:
|
||||
return 'r%d' % -osm_id
|
||||
return ('n' if feature['geometry']['type'] == 'Point' else 'w') + str(osm_id)
|
||||
|
||||
|
||||
def export(osmium, pbf, cache, extract):
|
||||
"""Exported features by key, and the keys of every tagged object in the extract."""
|
||||
filtered = cache / (extract + '-datacenters.osm.pbf')
|
||||
listed = cache / (extract + '-datacenters.opl')
|
||||
exported = cache / (extract + '-datacenters.geojsonseq')
|
||||
subprocess.run([osmium, 'tags-filter', '-O', '-o', str(filtered), str(pbf),
|
||||
'nwr/telecom=data_center', 'nwr/building=data_center'], check=True)
|
||||
subprocess.run([osmium, 'tags-filter', '-O', '-R', '-f', 'opl', '-o', str(listed), str(filtered),
|
||||
'nwr/telecom=data_center', 'nwr/building=data_center'], check=True)
|
||||
tagged = {line.split(' ', 1)[0] for line in listed.read_text(encoding='utf-8').splitlines() if line}
|
||||
subprocess.run([osmium, 'export', '-O', '-f', 'geojsonseq', '--add-unique-id=type_id',
|
||||
'-o', str(exported), str(filtered)], check=True)
|
||||
fresh = {}
|
||||
for line in exported.read_text(encoding='utf-8').splitlines():
|
||||
line = line.lstrip('\x1e').strip()
|
||||
if not line:
|
||||
continue
|
||||
feature = json.loads(line)
|
||||
tags = {k: v for k, v in feature['properties'].items() if k not in CONTACT_TAGS}
|
||||
if tags.get('telecom') != 'data_center' and tags.get('building') != 'data_center':
|
||||
continue # untagged member ways exported only for geometry
|
||||
key = feature['id']
|
||||
geometry = feature['geometry']
|
||||
if key[0] == 'a': # osmium area id: way * 2 or relation * 2 + 1
|
||||
number = int(key[1:])
|
||||
key = ('w%d' % (number // 2)) if number % 2 == 0 else ('r%d' % (number // 2))
|
||||
if key[0] == 'r':
|
||||
tags = {**tags, 'type': 'multipolygon'}
|
||||
if geometry['type'] == 'MultiPolygon' and len(geometry['coordinates']) == 1:
|
||||
geometry = {'type': 'Polygon', 'coordinates': geometry['coordinates'][0]}
|
||||
elif key in fresh or geometry['type'] == 'LineString':
|
||||
continue # closed ways are taken from their area export
|
||||
fresh[key] = {'geometry': geometry, 'tags': tags}
|
||||
return fresh, tagged
|
||||
|
||||
|
||||
def same_geometry(old, new):
|
||||
"""Same type and rings within tolerance; ring start, winding and part or hole order may differ."""
|
||||
if old['type'] != new['type']:
|
||||
return False
|
||||
|
||||
def meters(p, q):
|
||||
dx = (p[0] - q[0]) * 111320 * math.cos(math.radians(p[1]))
|
||||
return math.hypot(dx, (p[1] - q[1]) * 110540)
|
||||
|
||||
def same_line(a, b):
|
||||
return len(a) == len(b) and all(meters(p, q) <= SAME_GEOMETRY_METERS for p, q in zip(a, b))
|
||||
|
||||
def same_ring(a, b):
|
||||
a, b = a[:-1], b[:-1] # closed rings repeat their first vertex
|
||||
return len(a) == len(b) and \
|
||||
any(same_line(a[i:] + a[:i], c) for c in (b, b[::-1]) for i in range(len(a)))
|
||||
|
||||
def same_parts(a, b, same):
|
||||
rest = list(b)
|
||||
for part in a:
|
||||
match = next((i for i, other in enumerate(rest) if same(part, other)), None)
|
||||
if match is None:
|
||||
return False
|
||||
rest.pop(match)
|
||||
return not rest
|
||||
|
||||
def same_polygon(a, b):
|
||||
return same_ring(a[0], b[0]) and same_parts(a[1:], b[1:], same_ring)
|
||||
|
||||
a, b = old['coordinates'], new['coordinates']
|
||||
if old['type'] == 'Point':
|
||||
return same_line([a], [b])
|
||||
if old['type'] == 'LineString':
|
||||
return same_line(a, b)
|
||||
if old['type'] == 'Polygon':
|
||||
return same_polygon(a, b)
|
||||
if old['type'] == 'MultiPolygon':
|
||||
return same_parts(a, b, same_polygon)
|
||||
return a == b
|
||||
|
||||
|
||||
def feature_line(key, sequence, fresh):
|
||||
osm_id = int(key[1:]) * (-1 if key[0] == 'r' else 1)
|
||||
feature = {'id': osm_id, 'type': 'Feature', 'geometry': fresh['geometry'],
|
||||
'properties': {'id': sequence, 'tags': fresh['tags'], 'type': 'data_center',
|
||||
'osm_id': osm_id}}
|
||||
return json.dumps(feature, separators=(',', ':'), ensure_ascii=False)
|
||||
|
||||
|
||||
def merge(lines, fresh, rings, tagged):
|
||||
out, known, sequence = [], set(), 0
|
||||
report = {'kept': 0, 'updated': [], 'removed': [], 'added': [], 'unexported': [], 'skipped': []}
|
||||
for line in lines:
|
||||
feature = json.loads(line)
|
||||
sequence = max(sequence, feature['properties']['id'])
|
||||
key = osm_key(feature)
|
||||
known.add(key)
|
||||
if key in fresh:
|
||||
old_tags = feature['properties']['tags']
|
||||
if old_tags == fresh[key]['tags'] and same_geometry(feature['geometry'], fresh[key]['geometry']):
|
||||
out.append(line)
|
||||
report['kept'] += 1
|
||||
else:
|
||||
out.append(feature_line(key, feature['properties']['id'], fresh[key]))
|
||||
report['updated'].append(key)
|
||||
continue
|
||||
if key in tagged: # still tagged, but osmium built no usable geometry: keep the old line
|
||||
report['unexported'].append(key)
|
||||
out.append(line)
|
||||
continue
|
||||
lon, lat = points(feature['geometry']['coordinates'])[0][:2]
|
||||
if inside(rings, lon, lat):
|
||||
report['removed'].append(key)
|
||||
continue
|
||||
out.append(line)
|
||||
for key in sorted(set(fresh) - known, key=lambda k: (k[0], int(k[1:]))):
|
||||
sequence += 1
|
||||
out.append(feature_line(key, sequence, fresh[key]))
|
||||
report['added'].append(key)
|
||||
# newly tagged, but osmium built no usable geometry: nothing to add yet
|
||||
report['skipped'] = sorted(set(tagged) - set(fresh) - known, key=lambda k: (k[0], int(k[1:])))
|
||||
if len({osm_key(json.loads(line)) for line in out}) != len(out):
|
||||
raise ValueError('Duplicate OSM identity')
|
||||
return out, report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--cache', type=Path, required=True)
|
||||
parser.add_argument('--osmium', default=shutil.which('osmium') or 'osmium')
|
||||
parser.add_argument('--extract', default='latest',
|
||||
help="dated Geofabrik name such as norway-261005, or 'latest'")
|
||||
parser.add_argument('--verify', action='store_true')
|
||||
args = parser.parse_args()
|
||||
args.cache.mkdir(parents=True, exist_ok=True)
|
||||
extract = latest_extract() if args.extract == 'latest' else args.extract
|
||||
pbf = fetch(GEOFABRIK + extract + '.osm.pbf', args.cache / (extract + '.osm.pbf'))
|
||||
expected = fetch(GEOFABRIK + extract + '.osm.pbf.md5',
|
||||
args.cache / (extract + '.osm.pbf.md5')).read_text(encoding='utf-8').split()[0]
|
||||
actual = md5(pbf)
|
||||
if actual != expected:
|
||||
raise RuntimeError('MD5 mismatch for %s: %s != %s' % (pbf.name, actual, expected))
|
||||
rings = read_poly(fetch(GEOFABRIK + 'norway.poly', args.cache / 'norway.poly'))
|
||||
fresh, tagged = export(args.osmium, pbf, args.cache, extract)
|
||||
current = [line for line in DATASET.read_text(encoding='utf-8').splitlines() if line.strip()]
|
||||
merged, report = merge(current, fresh, rings, tagged)
|
||||
if report['unexported']:
|
||||
print('warning: kept the old line of tagged features osmium did not export: %s'
|
||||
% ', '.join(report['unexported']), file=sys.stderr)
|
||||
if report['skipped']:
|
||||
print('warning: did not add newly tagged features osmium did not export: %s'
|
||||
% ', '.join(report['skipped']), file=sys.stderr)
|
||||
payload = '\n'.join(merged) + '\n'
|
||||
if args.verify:
|
||||
if payload != DATASET.read_text(encoding='utf-8'):
|
||||
raise SystemExit('datacenters.geojsonl is not the %s merge' % extract)
|
||||
else:
|
||||
with open(DATASET, 'w', encoding='utf-8', newline='\n') as handle:
|
||||
handle.write(payload)
|
||||
print(json.dumps({'extract': extract, 'md5': actual, 'norway': len(fresh),
|
||||
'features': len(merged), **report}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,95 +0,0 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const SCRIPT = fileURLToPath(
|
||||
new URL('../../scripts/build-datacenters-norway.py', import.meta.url),
|
||||
);
|
||||
|
||||
function python(t, source) {
|
||||
const result = spawnSync('python3', ['-c', source, SCRIPT], {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
if (result.error?.code === 'ENOENT') return t.skip('python3 is not installed');
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
}
|
||||
|
||||
test('Norway refresh compares rings, not vertex sets', (t) => {
|
||||
python(
|
||||
t,
|
||||
String.raw`
|
||||
import runpy, sys
|
||||
same = runpy.run_path(sys.argv[1])['same_geometry']
|
||||
poly = lambda *rings: {'type': 'Polygon', 'coordinates': list(rings)}
|
||||
multi = lambda *polys: {'type': 'MultiPolygon', 'coordinates': [p['coordinates'] for p in polys]}
|
||||
a, b, c, d, e = [0, 0], [4, 0], [4, 4], [0, 4], [2, 1]
|
||||
assert not same(poly([a, b, c, d, e, a]), poly([a, b, e, c, d, a])), 'reordered edges'
|
||||
assert same(poly([a, b, c, d, e, a]), poly([c, b, a, e, d, c])), 'ring start and winding'
|
||||
assert same(poly([a, b, c, d, a]), poly([[0, 1e-7], b, c, d, [0, 1e-7]])), 'tolerance'
|
||||
outer = [[0, 0], [9, 0], [9, 9], [0, 9], [0, 0]]
|
||||
one, two = [[1, 1], [2, 1], [2, 2], [1, 1]], [[5, 5], [6, 5], [6, 6], [5, 5]]
|
||||
assert same(poly(outer, one, two), poly(outer, two, one)), 'hole order'
|
||||
assert not same(poly(outer, one), poly(one, outer)), 'outer and inner roles swapped'
|
||||
assert same(multi(poly(one), poly(two)), multi(poly(two), poly(one))), 'part order'
|
||||
p, q, r, s, u, v = [0, 0], [1, 0], [0, 1], [5, 5], [6, 5], [5, 6]
|
||||
assert not same(multi(poly([p, q, r, p]), poly([s, u, v, s])),
|
||||
multi(poly([p, q, s, p]), poly([r, u, v, r]))), 'vertices moved between parts'
|
||||
`,
|
||||
);
|
||||
});
|
||||
|
||||
test('Norway refresh keeps a still-tagged feature osmium did not export', (t) => {
|
||||
python(
|
||||
t,
|
||||
String.raw`
|
||||
import json, runpy, sys
|
||||
merge = runpy.run_path(sys.argv[1])['merge']
|
||||
square = [[[10, 60], [10.01, 60], [10.01, 60.01], [10, 60.01], [10, 60]]]
|
||||
def line(osm_id, sequence):
|
||||
return json.dumps({'id': osm_id, 'type': 'Feature', 'geometry': {'type': 'Polygon', 'coordinates': square},
|
||||
'properties': {'id': sequence, 'tags': {'building': 'data_center'},
|
||||
'type': 'data_center', 'osm_id': osm_id}})
|
||||
norway = [([(0, 0), (20, 0), (20, 80), (0, 80)], False)]
|
||||
lines = [line(7, 1), line(8, 2)]
|
||||
fresh = {'w13': {'geometry': {'type': 'Polygon', 'coordinates': square}, 'tags': {'building': 'data_center'}}}
|
||||
out, report = merge(lines, fresh, norway, {'w7', 'w12', 'w13'})
|
||||
assert out[0] == lines[0] and len(out) == 2, out
|
||||
assert report['unexported'] == ['w7'] and report['removed'] == ['w8'], report
|
||||
assert report['skipped'] == ['w12'] and report['added'] == ['w13'], report
|
||||
`,
|
||||
);
|
||||
});
|
||||
|
||||
test('Norway refresh lists tagged objects apart from the exported geometry', (t) => {
|
||||
python(
|
||||
t,
|
||||
String.raw`
|
||||
import json, runpy, subprocess, sys, tempfile
|
||||
from pathlib import Path
|
||||
export = runpy.run_path(sys.argv[1])['export']
|
||||
tags = {'building': 'data_center'}
|
||||
ring = [[10, 60], [10.01, 60], [10.01, 60.01], [10, 60]]
|
||||
features = [
|
||||
{'id': 'n1', 'properties': {'telecom': 'data_center'}, 'geometry': {'type': 'Point', 'coordinates': [10, 60]}},
|
||||
{'id': 'w5', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring}},
|
||||
{'id': 'a10', 'properties': tags, 'geometry': {'type': 'MultiPolygon', 'coordinates': [[ring]]}},
|
||||
{'id': 'w7', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring[:3]}},
|
||||
]
|
||||
outputs = {
|
||||
'.opl': 'n1 v1 Ttelecom=data_center x10 y60\nw5 v1 Tbuilding=data_center Nn2,n3,n4,n2\n'
|
||||
'w7 v1 Tbuilding=data_center Nn2,n3,n4\nr9 v1 Ttelecom=data_center Mw8@outer\n',
|
||||
'.geojsonseq': ''.join(chr(30) + json.dumps({'type': 'Feature', **f}) + '\n' for f in features),
|
||||
}
|
||||
def run(args, check=False):
|
||||
target = Path(args[args.index('-o') + 1])
|
||||
target.write_text(outputs.get(target.suffix, ''), encoding='utf-8')
|
||||
subprocess.run = run
|
||||
with tempfile.TemporaryDirectory() as cache:
|
||||
fresh, tagged = export('osmium', Path(cache) / 'norway-test.osm.pbf', Path(cache), 'norway-test')
|
||||
assert sorted(fresh) == ['n1', 'w5'], fresh
|
||||
assert fresh['w5']['geometry'] == {'type': 'Polygon', 'coordinates': [ring]}, fresh['w5']
|
||||
assert tagged == {'n1', 'w5', 'w7', 'r9'}, tagged
|
||||
`,
|
||||
);
|
||||
});
|
||||
@@ -26,14 +26,4 @@ reflects OSM as of the extract below, everything else the original snapshot.
|
||||
- Extract: Geofabrik `norway-261005.osm.pbf`, OSM data
|
||||
2026-10-05T20:21:35Z, MD5 `1d2cfdeb98980fce11b8891d0f9146c0`
|
||||
- Filter: `osmium tags-filter nwr/telecom=data_center nwr/building=data_center`
|
||||
- Contact tags are dropped as above; unchanged features keep their original
|
||||
line, so geometry is only rewritten when tags or rings actually changed.
|
||||
A feature is removed only once it is no longer tagged; one still tagged but
|
||||
without an exportable geometry keeps its line and is reported as unexported,
|
||||
and a newly tagged one osmium cannot export is reported as skipped.
|
||||
- Refresh: `python3 scripts/build-datacenters-norway.py --cache <dir>` uses
|
||||
Geofabrik's current Norway extract and prints the dated name it used; record
|
||||
that name, its OSM timestamp and MD5 here. Requires `osmium-tool`.
|
||||
- Check: add `--extract <dated name> --verify` to compare without writing.
|
||||
Geofabrik keeps only its newest dated extract, so a recorded extract can be
|
||||
re-verified only while it is still listed; the MD5 above identifies it.
|
||||
- Contact tags are dropped as above.
|
||||
Reference in new issue
Block a user