feat(intro): add the First Light intro sequence

A deterministic WebGL film of God's Eye View coming online: a seed on Null
Island grows into a culture that draws the planet, indexes the surface
tile by tile, ingests the live feeds and outgrows its dish, ending on the
logo. Renders 16:9 and 9:16 frame by frame in headless Chromium; the
procedural score is locked to the same clock. Data and imagery are
fetched by the scripts in tools/ and are not committed.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01KqP857pippaUs6konSDfe8

intro-sequence/tools/fetch/cctv_count.py

@@ -0,0 +1,511 @@
+#!/usr/bin/env python3
+"""Count distinct cameras in every keyless CCTV catalog the app supports.
+
+Fetches each source's FULL public catalog (no anchor selection, no
+DEFAULT_*_MAX_SOURCES caps) and counts distinct cameras two ways:
+  raw   - distinct camera ids listed in the catalog at all
+  valid - distinct ids passing the app's own validity filters (in service /
+          online / official still or stream URL / plausible coordinates),
+          i.e. what server/providers/cctv/* would accept before capping.
+The headline number written to data/extra_counts.json is the conservative
+`valid` count. Sources that fail are skipped, never estimated.
+
+Usage: python3 -I cctv_count.py <out_json> [<detail_json>]
+"""
+import json, math, re, sys, time
+from concurrent.futures import ThreadPoolExecutor
+from urllib.parse import urlparse, parse_qs
+import requests
+
+UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
+REPO = '/home/user/gods-eye-view'
+
+
+def sess():
+    s = requests.Session()
+    s.headers.update({'User-Agent': UA, 'Digitraffic-User': 'gods-eye-view'})
+    return s
+
+
+def get(url, timeout=60, **kw):
+    r = sess().get(url, timeout=timeout, **kw)
+    r.raise_for_status()
+    return r
+
+
+def fnum(v):
+    try:
+        f = float(v)
+        return f if math.isfinite(f) else None
+    except (TypeError, ValueError):
+        return None
+
+
+def inbox(lat, lon, la0, la1, lo0, lo1):
+    return lat is not None and lon is not None and la0 <= lat <= la1 and lo0 <= lon <= lo1
+
+
+SRC = {}
+
+
+def source(label, country):
+    def deco(fn):
+        SRC[label] = (fn, country)
+        return fn
+    return deco
+
+
+@source('TfL JamCams', 'United Kingdom')
+def tfl():
+    d = get('https://api.tfl.gov.uk/Place/Type/JamCam').json()
+    raw, ok = set(), set()
+    for p in d:
+        raw.add(p['id'])
+        props = {a['key']: a['value'] for a in p.get('additionalProperties', [])}
+        if str(props.get('available')).lower() != 'true':
+            continue
+        if not (props.get('imageUrl') or '').startswith('https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/'):
+            continue
+        if fnum(p.get('lat')) is None or fnum(p.get('lon')) is None:
+            continue
+        ok.add(p['id'])
+    return raw, ok, 'one Place per camera; available=true + official S3 still'
+
+
+@source('NYC DOT', 'United States')
+def nyc():
+    d = get('https://webcams.nyctmc.org/api/cameras').json()
+    raw = {c['id'] for c in d}
+    ok = {c['id'] for c in d if str(c.get('isOnline')).strip().lower() == 'true'
+          and inbox(fnum(c.get('latitude')), fnum(c.get('longitude')), 40.4, 41.0, -74.3, -73.6)}
+    return raw, ok, 'one row per camera; isOnline=true'
+
+
+@source('Caltrans', 'United States')
+def caltrans():
+    raw, ok = set(), set()
+    notes = []
+
+    def one(dist):
+        err = None
+        for attempt in range(5):  # cwwp2 resets connections under parallel load
+            try:
+                return dist, get(f'https://cwwp2.dot.ca.gov/data/d{dist}/cctv/cctvStatusD{dist:02d}.json').json()
+            except Exception as e:
+                err = e
+                time.sleep(1 + attempt)
+        return dist, err
+    with ThreadPoolExecutor(3) as ex:
+        res = list(ex.map(one, range(1, 13)))
+    for dist, d in res:
+        if isinstance(d, Exception):
+            notes.append(f'D{dist} failed: {str(d)[:60]}')
+            continue
+        for i, row in enumerate(d.get('data', [])):
+            c = row.get('cctv') or {}
+            loc = c.get('location') or {}
+            name = (loc.get('locationName') or '').strip()
+            m = re.match(r'^([A-Za-z0-9_-]+)\s*--', name)
+            key = f"d{dist}-{(m.group(1) if m else c.get('index') or name or i)}".lower()
+            raw.add(key)
+            if str(c.get('inService')).lower() != 'true':
+                continue
+            if fnum(loc.get('latitude')) is None or fnum(loc.get('longitude')) is None:
+                continue
+            url = ((c.get('imageData') or {}).get('static') or {}).get('currentImageURL') or ''
+            if not url.startswith('https://cwwp2.dot.ca.gov/'):
+                continue
+            ok.add(key)
+    return raw, ok, 'all 12 districts; inService=true + cwwp2 still. ' + '; '.join(notes)
+
+
+@source('WSDOT', 'United States')
+def wsdot():
+    d = json.loads(get('https://data.wsdot.wa.gov/travelcenter/Cameras.json').content.decode('cp1252'))
+    raw, ok = set(), set()
+    for f in d['features']:
+        a = f.get('attributes') or {}
+        cid = a.get('CameraID')
+        if not isinstance(cid, int) or cid <= 0:
+            continue
+        raw.add(cid)
+        if (a.get('ImageURL') or '').strip().startswith('https://images.wsdot.wa.gov/') and fnum((f.get('geometry') or {}).get('x')) is not None:
+            ok.add(cid)
+    return raw, ok, 'one feature per CameraID; official images.wsdot.wa.gov still'
+
+
+@source('King County DOT', 'United States')
+def kingco():
+    d = get('https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson').json()
+    raw, ok = set(), set()
+    for f in d['features']:
+        p = f.get('properties') or {}
+        aid = p.get('AssetID')
+        if not isinstance(aid, int) or aid <= 0:
+            continue
+        raw.add(aid)
+        if p.get('Live') != 1 or str(p.get('CurrentStatus') or '').strip() != 'Active':
+            continue
+        if str(p.get('Owner') or '').strip() == 'WSDOT':
+            continue  # WSDOT-owned rows are WSDOT cameras (counted under WSDOT)
+        url = (p.get('ImageURL') or '').strip().replace('http://info.kingcounty.gov/', 'https://info.kingcounty.gov/')
+        if not url.startswith('https://info.kingcounty.gov/'):
+            continue
+        ok.add(aid)
+    note = 'Live=1 + Active, WSDOT-owned rows excluded (avoid double count)'
+    if d.get('exceededTransferLimit') or (d.get('properties') or {}).get('exceededTransferLimit'):
+        note += ' WARNING: transfer limit exceeded'
+    return raw, ok, note
+
+
+@source('DriveBC', 'Canada')
+def drivebc():
+    d = get('https://www.drivebc.ca/api/webcams/').json()
+    raw, ok = set(), set()
+    for c in d:
+        cid = c.get('id')
+        if not isinstance(cid, int) or cid <= 0:
+            continue
+        raw.add(cid)
+        if c.get('is_on') is not True or c.get('should_appear') is not True:
+            continue
+        coords = (c.get('location') or {}).get('coordinates') or [None, None]
+        if not inbox(fnum(coords[1]), fnum(coords[0]), 48.0, 60.5, -139.5, -113.5):
+            continue
+        ok.add(cid)
+    return raw, ok, 'one webcam per id; is_on + should_appear'
+
+
+@source('Fintraffic Digitraffic', 'Finland')
+def fintraffic():
+    d = get('https://tie.digitraffic.fi/api/weathercam/v1/stations', headers={'Accept-Encoding': 'gzip'}).json()
+    raw, ok, presets = set(), set(), 0
+    for f in d['features']:
+        p = f.get('properties') or {}
+        sid = str(p.get('id') or '')
+        if not sid:
+            continue
+        raw.add(sid)
+        if str(p.get('collectionStatus') or '').upper() != 'GATHERING':
+            continue
+        live = [pr for pr in p.get('presets') or [] if pr.get('inCollection') is True and re.fullmatch(r'C\d{7}', str(pr.get('id'))) and str(pr['id']).startswith(sid)]
+        if live:
+            ok.add(sid)
+            presets += len(live)
+    return raw, ok, f'counted per STATION (camera site), not per preset view; {presets} live presets (the app treats each preset as a camera)'
+
+
+@source('Statens vegvesen', 'Norway')
+def vegvesen():
+    d = get('https://ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items?f=application/geo%2Bjson&limit=5000',
+            headers={'Accept': 'application/geo+json,application/json'}).json()
+    raw, ok = set(), set()
+    for f in d['features']:
+        p = f.get('properties') or {}
+        rid = str(p.get('cameraId') or '').strip()
+        if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', rid):
+            continue
+        raw.add(rid)
+        av = str(p.get('status.stillImageAvailability') or '').strip()
+        if av and av != 'videoOrImagesAvailable':
+            continue
+        if (p.get('stillImageUrl') or '') != f'https://kamera.atlas.vegvesen.no/api/images/{rid}':
+            continue
+        c = (f.get('geometry') or {}).get('coordinates') or [None, None]
+        if not inbox(fnum(c[1]), fnum(c[0]), 57.5, 71.5, 4.0, 31.5):
+            continue
+        ok.add(rid)
+    note = f"numberMatched={d.get('numberMatched')} numberReturned={d.get('numberReturned')}"
+    return raw, ok, 'stillImageAvailability ok + official still; ' + note
+
+
+@source('City of Tallinn', 'Estonia')
+def tallinn():
+    d = json.load(open(f'{REPO}/config/cctv_sources.tallinn.json'))
+    raw, ok = set(), set()
+    for c in d:
+        cid = str(c.get('id') or '').strip()
+        if not cid:
+            continue
+        raw.add(cid)
+        url = c.get('snapshotUrl') or c.get('url') or ''
+        if inbox(fnum(c.get('lat')), fnum(c.get('lon')), 59.2, 59.7, 24.3, 25.4) and url.startswith('https://ristmikud.tallinn.ee/'):
+            ok.add(url)
+    return raw, ok, 'curated repo catalog (config/cctv_sources.tallinn.json) of ristmikud.tallinn.ee stills; distinct frame URLs'
+
+
+@source('Transpordiamet Tarktee', 'Estonia')
+def tarktee():
+    lx = get('https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraLocations', headers={'Accept': 'application/xml'}).text
+    ix = get('https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraImages', headers={'Accept': 'application/xml'}).text
+    locs = {}
+    for m in re.finditer(r'<predefinedLocation\s+id="([^"]+)"[^>]*>([\s\S]*?)</predefinedLocation>', lx):
+        la = re.search(r'<latitude>\s*(-?\d+(?:\.\d+)?)\s*</latitude>', m.group(2), re.I)
+        lo = re.search(r'<longitude>\s*(-?\d+(?:\.\d+)?)\s*</longitude>', m.group(2), re.I)
+        if la and lo:
+            locs[m.group(1)] = (float(la.group(1)), float(lo.group(1)))
+    imgs = {}
+    for m in re.finditer(r'<trafficView\b[^>]*>([\s\S]*?)</trafficView>', ix):
+        r = re.search(r'<linearPredefinedLocationReference\b[^>]*\bid="([^"]+)"', m.group(1), re.I)
+        u = re.search(r'<urlLinkAddress>\s*([^<\s]+)\s*</urlLinkAddress>', m.group(1), re.I)
+        if r and u:
+            imgs[r.group(1)] = u.group(1)
+    raw = set(locs) | set(imgs)
+    ok = set()
+    for lid, (lat, lon) in locs.items():
+        u = imgs.get(lid, '')
+        if not u.startswith('https://tarktee.transpordiamet.ee/images/'):
+            continue
+        if not inbox(lat, lon, 57.4, 59.9, 21.5, 28.4):
+            continue
+        n = re.search(r'/images/(\d+)/', u)
+        ok.add(n.group(1) if n else lid)
+    return raw, ok, 'DATEX location joined to image; distinct camera numbers'
+
+
+@source('Via Lietuva eismoinfo', 'Lithuania')
+def lithuania():
+    rows = get('https://eismoinfo.lt/eismoinfo-backend/camera-info-table', headers={'Accept': 'application/json'}).json()
+    layers = get('https://eismoinfo.lt/eismoinfo-backend/layer-static-features/VKR', headers={'Accept': 'application/json'}).json()
+    feats = {}
+    for L in layers:
+        if L.get('layer') == 'VKR':
+            feats = {str(f.get('id')): f for f in L.get('features') or [] if f}
+    raw, ok = set(), set()
+    for r in rows:
+        rid = str(r.get('id') or '')
+        if not re.fullmatch(r'[1-9]\d{0,8}', rid):
+            continue
+        raw.add(rid)
+        f = feats.get(rid)
+        try:
+            lat, lon = f['points'][0]['point']
+        except Exception:
+            continue
+        if not inbox(fnum(lat), fnum(lon), 53.8, 56.5, 20.8, 26.9):
+            continue
+        u = urlparse(str(r.get('image') or ''))
+        if f'{u.scheme}://{u.netloc}' != 'https://eismoinfo.lt' or u.path != '/eismoinfo-backend/image-provider/camera/last' or parse_qs(u.query).get('id', [''])[0] != rid:
+            continue
+        ok.add(rid)
+    return raw, ok, 'camera table joined to VKR layer; official still endpoint'
+
+
+@source('Live Traffic NSW', 'Australia')
+def nsw():
+    d = get('https://data.livetraffic.com/cameras/traffic-cam.json').json()
+    raw, ok = set(), set()
+    for f in d['features']:
+        p = f.get('properties') or {}
+        fid = str(f.get('id') or p.get('id') or p.get('title') or '').strip()
+        if not fid:
+            continue
+        raw.add(fid)
+        href = (p.get('href') or '').replace('http://', 'https://')
+        c = (f.get('geometry') or {}).get('coordinates') or [None, None]
+        if href.startswith('https://webcams.transport.nsw.gov.au/') and inbox(fnum(c[1]), fnum(c[0]), -38.0, -28.0, 140.0, 154.5):
+            ok.add(fid)
+    return raw, ok, 'one feature per camera; official webcams.transport.nsw.gov.au still'
+
+
+@source('City of Calgary', 'Canada')
+def calgary():
+    d = get('https://data.calgary.ca/resource/k7p9-kppz.json?$limit=5000').json()
+    raw, ok = set(), set()
+    for c in d:
+        url = ((c.get('camera_url') or {}).get('url') or '').strip()
+        if not url:
+            continue
+        u = re.sub(r'^http://', 'https://', url)
+        m = re.search(r'loc(\d+)\.jpg$', urlparse(u).path, re.I)
+        key = m.group(1) if m else urlparse(u).path
+        raw.add(key)
+        coords = (c.get('point') or {}).get('coordinates') or [None, None]
+        if u.startswith('https://trafficcam.calgary.ca/') and inbox(fnum(coords[1]), fnum(coords[0]), 50.7, 51.4, -114.5, -113.7):
+            ok.add(key)
+    return raw, ok, f'{len(d)} rows ($limit=5000); distinct trafficcam.calgary.ca image ids'
+
+
+@source('City of Toronto RESCU', 'Canada')
+def toronto():
+    t = get('https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json').text
+    d = json.loads(t[t.index('(') + 1: t.rindex(')')])
+    raw, ok = set(), set()
+    for c in d['Data']:
+        n = str(c.get('Number') or '').strip()
+        if not n:
+            continue
+        raw.add(n)
+        if re.fullmatch(r'\d{1,6}', n) and inbox(fnum(c.get('Latitude')), fnum(c.get('Longitude')), 43.4, 44.0, -79.8, -79.0):
+            ok.add(n)
+    return raw, ok, 'one row per numbered RESCU camera'
+
+
+@source('Servei Català de Trànsit', 'Spain')
+def catalonia():
+    x = get('https://www.gencat.cat/transit/opendata/cameres.xml').text
+    hosts = {'mct.gencat.cat': '/mct2bo/', 'www.bcn.cat': '/transit/imatges/', 'emap.terrassa.cat': '/it_terrassa/'}
+    raw, ok = set(), set()
+    for m in re.finditer(r'<gml:featureMember>([\s\S]*?)</gml:featureMember>', x):
+        b = m.group(1)
+        co = re.search(r'<gml:coordinates[^>]*>([^<]+)</gml:coordinates>', b)
+        lk = re.search(r'<cite:link>([^<]*)</cite:link>', b)
+        if not co or not lk:
+            continue
+        link = lk.group(1).replace('&amp;', '&').strip()
+        u = urlparse(link)
+        cam = parse_qs(u.query).get('sctidcam', [''])[0]
+        key = f'sct:{cam}' if u.hostname == 'mct.gencat.cat' and cam else f'{u.hostname}{u.path}'
+        raw.add(key)
+        lon, lat = (fnum(v) for v in co.group(1).split(',')[:2])
+        if not inbox(lat, lon, 40.4, 42.95, 0.1, 3.4):
+            continue
+        d = hosts.get(u.hostname or '')
+        if not d or not u.path.startswith(d) or u.username or u.password or u.port:
+            continue
+        f = u.path[len(d):]
+        if not f or '/' in f:
+            continue
+        if u.hostname == 'mct.gencat.cat' and (f != 'RenderService' or not re.fullmatch(r'[a-z0-9]+\.(gif|jpe?g)', cam, re.I)):
+            continue
+        ok.add(key)
+    return raw, ok, 'SCT + Barcelona IMI + Terrassa; keyed by frame file (feed fid is unstable)'
+
+
+@source('City of Austin', 'United States')
+def austin():
+    d = get('https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD').json()
+    cols = [c['fieldName'] for c in d['meta']['view']['columns']]
+    raw, ok = set(), set()
+    for row in d['data']:
+        r = dict(zip(cols, row))
+        cid = str(r.get('camera_id') or '').strip()
+        if not cid:
+            continue
+        raw.add(cid)
+        st = (r.get('camera_status') or '').strip().upper()
+        if st and st != 'TURNED_ON':
+            continue
+        m = re.match(r'POINT \(([-\d.]+) ([-\d.]+)\)', r.get('location') or '')
+        if not m or not inbox(fnum(m.group(2)), fnum(m.group(1)), 29.9, 30.8, -98.2, -97.3):
+            continue
+        ok.add(cid)
+    return raw, ok, 'camera_status TURNED_ON'
+
+
+TXDOT = 'ABL AMA ATL AUS BMT BWD BRY CHS CRP DAL ELP FTW HOU LRD LBB LFK ODA PAR PHR SJT SAT TYL WAC WFS YKM'.split()
+
+
+@source('TxDOT ITS', 'United States')
+def txdot():
+    def one(code):
+        for attempt in range(2):
+            try:
+                return code, get(f'https://its.txdot.gov/its/DistrictIts/GetCctvStatusListByDistrict?districtCode={code}',
+                                 headers={'Accept': 'application/json'}, timeout=60).json()
+            except Exception as e:
+                err = e
+        return code, err
+    with ThreadPoolExecutor(8) as ex:
+        res = list(ex.map(one, TXDOT))
+    raw, ok, notes, per = set(), set(), [], {}
+    for code, d in res:
+        if isinstance(d, Exception):
+            notes.append(f'{code} failed: {str(d)[:50]}')
+            continue
+        n = 0
+        for rows in (d.get('roadwayCctvStatuses') or {}).values():
+            for r in rows or []:
+                icd = str(r.get('icd_Id') or '').strip()
+                if not icd:
+                    continue
+                raw.add(icd)
+                if r.get('statusDescription') != 'Device Online' or r.get('hasSnapshot') is False:
+                    continue
+                lat, lon = r.get('latitude'), r.get('longitude')
+                if not (isinstance(lat, (int, float)) and isinstance(lon, (int, float)) and inbox(lat, lon, 25.5, 36.7, -106.8, -93.3)):
+                    continue
+                if icd not in ok:
+                    n += 1
+                ok.add(icd)
+        per[code] = n
+    return raw, ok, f'all 25 districts; Device Online + snapshot; per-district {per}. ' + '; '.join(notes)
+
+
+@source('Ontario 511', 'Canada')
+def ontario():
+    d = get('https://511on.ca/api/v2/get/cameras?format=json&lang=en', headers={'Accept': 'application/json'}).json()
+    raw, ok, views = set(), set(), 0
+    for r in d:
+        rid = str(r.get('Id') or r.get('id') or '').strip()
+        if not rid:
+            continue
+        raw.add(rid)
+        if not inbox(fnum(r.get('Latitude')), fnum(r.get('Longitude')), 41.0, 57.5, -95.6, -74.0):
+            continue
+        good = []
+        for v in r.get('Views') or r.get('views') or []:
+            if str(v.get('Status') or '').strip().lower() != 'enabled':
+                continue
+            u = urlparse(str(v.get('Url') or '').strip())
+            if u.scheme == 'https' and (u.hostname == '511on.ca' or (u.hostname or '').endswith('.traveliq.co')) and re.fullmatch(r'/map/Cctv/[A-Za-z0-9_.-]+', u.path):
+                good.append(v)
+        if good:
+            ok.add(rid)
+            views += len(good)
+    return raw, ok, f'counted per camera row, not per view; {views} enabled views'
+
+
+@source('DelDOT', 'United States')
+def deldot():
+    d = get('https://tmc.deldot.gov/json/videocamera.json', headers={'Accept': 'application/json'}).json()
+    raw, ok = set(), set()
+    for r in d.get('videoCameras') or []:
+        cid = str(r.get('id') or '').strip()
+        if not cid:
+            continue
+        raw.add(cid)
+        if str(r.get('status')).lower() != 'active':
+            continue
+        if not inbox(fnum(r.get('lat')), fnum(r.get('lon')), 38.4, 39.9, -75.85, -75.0):
+            continue
+        u = urlparse(str((r.get('urls') or {}).get('m3u8s') or ''))
+        if u.scheme != 'https' or u.hostname != 'video.deldot.gov' or u.port not in (None, 443) or u.username or not re.fullmatch(r'/live/[A-Za-z0-9_.-]+/playlist\.m3u8', u.path):
+            continue
+        if not re.fullmatch(r'[A-Za-z0-9_-]{1,80}', cid):
+            continue
+        ok.add(cid)
+    return raw, ok, 'status active + official HLS stream'
+
+
+def run(label):
+    fn, country = SRC[label]
+    t0 = time.time()
+    try:
+        raw, ok, note = fn()
+        if not ok:
+            raise RuntimeError('catalog parsed to zero cameras')
+        return label, dict(country=country, raw=len(raw), valid=len(ok), note=note, secs=round(time.time() - t0, 1))
+    except Exception as e:
+        return label, dict(country=country, error=f'{type(e).__name__}: {str(e)[:200]}', secs=round(time.time() - t0, 1))
+
+
+if __name__ == '__main__':
+    out = sys.argv[1]
+    detail = sys.argv[2] if len(sys.argv) > 2 else None
+    with ThreadPoolExecutor(8) as ex:
+        results = dict(ex.map(run, list(SRC)))
+    counted = {k: v for k, v in results.items() if 'valid' in v}
+    for k, v in results.items():
+        print(f"{k:28s} {v.get('country', ''):15s} raw={v.get('raw', '-'):>6} valid={v.get('valid', '-'):>6}  {v.get('error') or v.get('note')}")
+    total = sum(v['valid'] for v in counted.values())
+    res = {
+        'cctv_catalog': total,
+        'cctv_sources': {k: v['valid'] for k, v in counted.items()},
+        'cctv_countries': len({v['country'] for v in counted.values()}),
+        'fetched_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
+    }
+    print('TOTAL valid', total, 'raw', sum(v['raw'] for v in counted.values()), 'countries', res['cctv_countries'])
+    json.dump(res, open(out, 'w'), ensure_ascii=False, indent=2)
+    if detail:
+        json.dump(results, open(detail, 'w'), ensure_ascii=False, indent=2)

intro-sequence/tools/fetch/cctv_fetch.py

@@ -0,0 +1,406 @@
+#!/usr/bin/env python3
+"""Download real current still frames from keyless public traffic-camera feeds.
+
+Stage 1 (this script): build candidate lists from each source's public catalog
+(endpoints mirror server/providers/cctv/*), sample a spread of cameras, download
+frames, compute quality metrics, write <work>/frames/*.jpg + <work>/candidates.json.
+Stage 2 (cctv_select.py) filters, crops to 640x360 and writes cctv/index.json.
+
+Usage: python3 -I cctv_fetch.py <work_dir>
+"""
+import base64, hashlib, io, json, math, random, re, sys, time
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor
+import requests
+from PIL import Image, ImageStat, ImageFilter
+
+WORK = sys.argv[1]
+import os
+os.makedirs(f'{WORK}/frames', exist_ok=True)
+UA_BROWSER = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
+S = requests.Session()
+S.headers.update({'User-Agent': UA_BROWSER, 'Digitraffic-User': 'gods-eye-intro'})
+rng = random.Random(42)
+
+
+def get(url, **kw):
+    kw.setdefault('timeout', 40)
+    r = S.get(url, **kw)
+    r.raise_for_status()
+    return r
+
+
+def C(source, name, city, country, lat, lon, url, kind='image'):
+    return dict(source=source, name=name.strip(), city=city, country=country, lat=round(float(lat), 5), lon=round(float(lon), 5), url=url, kind=kind)
+
+
+def nearest(cands, anchors, n):
+    """Spread a pick across anchors: round-robin, nearest cameras to each anchor first, randomized a bit."""
+    buckets = []
+    for (alat, alon, label) in anchors:
+        b = sorted(cands, key=lambda c: (c['lat'] - alat) ** 2 + ((c['lon'] - alon) * math.cos(math.radians(alat))) ** 2)[:max(n * 3, 12)]
+        rng.shuffle(b)
+        for c in b:
+            c['city'] = label if c.get('city') in (None, '', '?') else c['city']
+        buckets.append(b)
+    out, seen = [], set()
+    while len(out) < n and any(buckets):
+        for b in buckets:
+            while b:
+                c = b.pop()
+                if c['url'] not in seen:
+                    seen.add(c['url']); out.append(c); break
+    return out[:n]
+
+
+def sample(cands, n):
+    cands = list(cands)
+    rng.shuffle(cands)
+    return cands[:n]
+
+
+SOURCES = {}
+
+
+def source(fn):
+    SOURCES[fn.__name__] = fn
+    return fn
+
+
+@source
+def tfl(n):
+    d = get('https://api.tfl.gov.uk/Place/Type/JamCam').json()
+    out = []
+    for p in d:
+        props = {a['key']: a['value'] for a in p.get('additionalProperties', [])}
+        if props.get('available') != 'true' or not props.get('imageUrl'):
+            continue
+        out.append(C('TfL JamCams', p['commonName'], 'London', 'United Kingdom', p['lat'], p['lon'], props['imageUrl']))
+    return nearest(out, [(51.5074, -0.1278, 'London')], n)
+
+
+@source
+def nyc(n):
+    d = get('https://webcams.nyctmc.org/api/cameras').json()
+    out = [C('NYC DOT', c['name'], f"New York ({c.get('area')})", 'United States', c['latitude'], c['longitude'],
+             f"https://webcams.nyctmc.org/api/cameras/{c['id']}/image") for c in d if str(c.get('isOnline')).lower() == 'true']
+    return nearest(out, [(40.758, -73.9855, 'New York'), (40.7128, -74.006, 'New York'), (40.6782, -73.9442, 'New York'), (40.7282, -73.7949, 'New York')], n)
+
+
+@source
+def caltrans(n):
+    out = []
+    for dist, city in [(4, 'San Francisco Bay Area'), (7, 'Los Angeles'), (11, 'San Diego'), (3, 'Sacramento')]:
+        try:
+            d = get(f'https://cwwp2.dot.ca.gov/data/d{dist}/cctv/cctvStatusD{dist:02d}.json').json()
+        except Exception as e:
+            print('caltrans', dist, e); continue
+        for row in d['data']:
+            c = row['cctv']
+            url = (c.get('imageData', {}).get('static') or {}).get('currentImageURL')
+            if c.get('inService') != 'true' or not url:
+                continue
+            loc = c['location']
+            out.append(C('Caltrans CCTV', loc['locationName'], loc.get('nearbyPlace') or city, 'United States', loc['latitude'], loc['longitude'], url))
+    return nearest(out, [(37.7793, -122.4193, 'San Francisco'), (34.0537, -118.2428, 'Los Angeles'), (32.7157, -117.1611, 'San Diego'), (38.5816, -121.4944, 'Sacramento')], n)
+
+
+@source
+def wsdot(n):
+    d = json.loads(get('https://data.wsdot.wa.gov/travelcenter/Cameras.json').content.decode('cp1252'))
+    out = []
+    for f in d['features']:
+        a, g = f['attributes'], f['geometry']
+        url = a.get('ImageURL') or ''
+        if not url.startswith('https://images.wsdot.wa.gov/'):
+            continue
+        lon = g['x'] / 6378137 * 180 / math.pi
+        lat = (2 * math.atan(math.exp(g['y'] / 6378137)) - math.pi / 2) * 180 / math.pi
+        out.append(C('WSDOT', a['CameraTitle'], '', 'United States', lat, lon, url))
+    return nearest(out, [(47.6062, -122.3321, 'Seattle'), (47.6588, -117.426, 'Spokane')], n)
+
+
+@source
+def kingco(n):
+    d = get('https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson').json()
+    out = []
+    for f in d['features']:
+        p = f['properties']
+        url = (p.get('ImageURL') or '').replace('http://info.kingcounty.gov/', 'https://info.kingcounty.gov/')
+        if p.get('CurrentStatus') != 'Active' or not url.startswith('https://info.kingcounty.gov/'):
+            continue
+        lon, lat = f['geometry']['coordinates'][:2]
+        out.append(C('King County DOT', p['Location'], 'King County, WA', 'United States', lat, lon, url))
+    return sample(out, n)
+
+
+@source
+def drivebc(n):
+    d = get('https://www.drivebc.ca/api/webcams/').json()
+    out = []
+    for c in d:
+        if not c.get('is_on') or c.get('marked_stale') or not c.get('should_appear', True):
+            continue
+        lon, lat = c['location']['coordinates'][:2]
+        out.append(C('DriveBC', c['name'], c.get('region_name') or 'British Columbia', 'Canada', lat, lon, f"https://www.drivebc.ca/images/{c['id']}.jpg"))
+    return nearest(out, [(49.2827, -123.1207, 'Vancouver'), (48.4284, -123.3656, 'Victoria'), (50.1163, -122.9574, 'Whistler')], n)
+
+
+@source
+def calgary(n):
+    d = get('https://data.calgary.ca/resource/k7p9-kppz.json?$limit=500').json()
+    out = []
+    for c in d:
+        url = (c.get('camera_url') or {}).get('url', '').replace('http://', 'https://')
+        if not url.startswith('https://trafficcam.calgary.ca/'):
+            continue
+        lon, lat = c['point']['coordinates'][:2]
+        out.append(C('City of Calgary', c['camera_location'], 'Calgary', 'Canada', lat, lon, url))
+    return nearest(out, [(51.0461, -114.0626, 'Calgary')], n)
+
+
+@source
+def toronto(n):
+    t = get('https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json').text
+    d = json.loads(t[t.index('(') + 1: t.rindex(')')])
+    out = [C('City of Toronto RESCU', c['Name'].title(), 'Toronto', 'Canada', c['Latitude'], c['Longitude'],
+             f"https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/loc{c['Number']}.jpg")
+           for c in d['Data'] if re.fullmatch(r'\d+', str(c['Number']))]
+    return nearest(out, [(43.6532, -79.3832, 'Toronto')], n)
+
+
+@source
+def txdot(n):
+    out = []
+    for code, city in [('HOU', 'Houston'), ('DAL', 'Dallas'), ('AUS', 'Austin'), ('SAT', 'San Antonio'), ('ELP', 'El Paso')]:
+        try:
+            d = get(f'https://its.txdot.gov/its/DistrictIts/GetCctvStatusListByDistrict?districtCode={code}').json()
+        except Exception as e:
+            print('txdot', code, e); continue
+        seen = set()
+        for rows in d['roadwayCctvStatuses'].values():
+            for r in rows:
+                if r.get('statusDescription') != 'Device Online' or r.get('hasSnapshot') is False or r['icd_Id'] in seen:
+                    continue
+                if not isinstance(r.get('latitude'), (int, float)) or not r['latitude']:
+                    continue
+                seen.add(r['icd_Id'])
+                url = 'https://its.txdot.gov/its/DistrictIts/GetCctvSnapshotByIcdId?' + requests.compat.urlencode({'icdId': r['icd_Id'], 'districtCode': code})
+                out.append(C('TxDOT ITS', r['name'], city, 'United States', r['latitude'], r['longitude'], url, kind='txdot'))
+    return nearest(out, [(29.7604, -95.3698, 'Houston'), (32.7767, -96.797, 'Dallas'), (29.4241, -98.4936, 'San Antonio'), (31.7619, -106.485, 'El Paso')], n)
+
+
+@source
+def austin(n):
+    d = get('https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD').json()
+    cols = [c['fieldName'] for c in d['meta']['view']['columns']]
+    out = []
+    for row in d['data']:
+        r = dict(zip(cols, row))
+        if (r.get('camera_status') or '').strip().upper() != 'TURNED_ON':
+            continue
+        m = re.match(r'POINT \(([-\d.]+) ([-\d.]+)\)', r.get('location') or '')
+        if not m or not r.get('camera_id'):
+            continue
+        out.append(C('City of Austin', (r.get('location_name') or '').strip(), 'Austin', 'United States', m.group(2), m.group(1),
+                     f"https://cctv.austinmobility.io/image/{r['camera_id']}.jpg"))
+    return nearest(out, [(30.2672, -97.7431, 'Austin')], n)
+
+
+@source
+def fintraffic(n):
+    d = get('https://tie.digitraffic.fi/api/weathercam/v1/stations').json()
+    out = []
+    for f in d['features']:
+        p = f['properties']
+        if p.get('collectionStatus') != 'GATHERING':
+            continue
+        lon, lat = f['geometry']['coordinates'][:2]
+        for pr in p.get('presets', []):
+            if pr.get('inCollection'):
+                out.append(C('Fintraffic Digitraffic', p['name'].replace('_', ' '), '', 'Finland', lat, lon, f"https://weathercam.digitraffic.fi/{pr['id']}.jpg"))
+                break
+    return nearest(out, [(60.1699, 24.9384, 'Helsinki'), (61.4978, 23.761, 'Tampere'), (65.0121, 25.4651, 'Oulu'), (66.5039, 25.7294, 'Rovaniemi'), (60.4518, 22.2666, 'Turku')], n)
+
+
+@source
+def vegvesen(n):
+    d = get('https://ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items?f=application/geo%2Bjson&limit=5000').json()
+    out = []
+    for f in d['features']:
+        p = f['properties']
+        url = p.get('stillImageUrl') or ''
+        if not url.startswith('https://kamera.atlas.vegvesen.no/api/images/'):
+            continue
+        lon, lat = f['geometry']['coordinates'][:2]
+        name = ' - '.join(x for x in [p.get('description'), p.get('roadNumber')] if x)
+        out.append(C('Statens vegvesen', name, '', 'Norway', lat, lon, url))
+    return nearest(out, [(59.9139, 10.7522, 'Oslo'), (60.3913, 5.3221, 'Bergen'), (63.4305, 10.3951, 'Trondheim'), (69.6492, 18.9553, 'Tromsø'), (58.97, 5.7331, 'Stavanger')], n)
+
+
+@source
+def tallinn(n):
+    d = json.load(open('/home/user/gods-eye-view/config/cctv_sources.tallinn.json'))
+    out = [C('City of Tallinn', c['name'], 'Tallinn', 'Estonia', c['lat'], c['lon'], c['snapshotUrl']) for c in d if c.get('snapshotUrl', '').startswith('https://ristmikud.tallinn.ee/')]
+    return sample(out, n)
+
+
+@source
+def tarktee(n):
+    ns = {'d': 'http://datex2.eu/schema/2/2_0'}
+    loc = ET.fromstring(get('https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraLocations').content)
+    locs = {}
+    for pl in loc.iter('{http://datex2.eu/schema/2/2_0}predefinedLocation'):
+        lat = pl.find('.//d:latitude', ns); lon = pl.find('.//d:longitude', ns); nm = pl.find('.//d:value', ns)
+        if lat is not None and lon is not None:
+            locs[pl.get('id')] = (float(lat.text), float(lon.text), nm.text if nm is not None else '')
+    img = ET.fromstring(get('https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraImages').content)
+    out = []
+    for tv in img.iter('{http://datex2.eu/schema/2/2_0}linearTrafficView'):
+        ref = tv.find('d:linearPredefinedLocationReference', ns)
+        u = tv.find('.//d:urlLinkAddress', ns)
+        if ref is None or u is None or ref.get('id') not in locs:
+            continue
+        lat, lon, nm = locs[ref.get('id')]
+        out.append(C('Transpordiamet Tarktee', nm, '', 'Estonia', lat, lon, u.text))
+    return nearest(out, [(59.437, 24.753, 'Tallinn'), (58.378, 26.729, 'Tartu'), (58.3859, 24.4971, 'Pärnu')], n)
+
+
+@source
+def lithuania(n):
+    rows = get('https://eismoinfo.lt/eismoinfo-backend/camera-info-table').json()
+    layer = get('https://eismoinfo.lt/eismoinfo-backend/layer-static-features/VKR').json()
+    feats = {str(f.get('id')): f for L in layer for f in L.get('features', [])}
+    out = []
+    for r in rows:
+        f = feats.get(str(r.get('id')))
+        try:
+            lat, lon = f['points'][0]['point']
+        except Exception:
+            continue
+        out.append(C('Via Lietuva eismoinfo', r['name'], '', 'Lithuania', lat, lon, f"https://eismoinfo.lt/eismoinfo-backend/image-provider/camera/last?id={r['id']}"))
+    return nearest(out, [(54.6872, 25.2797, 'Vilnius'), (54.8985, 23.9036, 'Kaunas'), (55.7033, 21.1443, 'Klaipėda')], n)
+
+
+@source
+def catalonia(n):
+    x = get('https://www.gencat.cat/transit/opendata/cameres.xml').text
+    out = []
+    for m in re.finditer(r'<gml:coordinates[^>]*>([-\d.]+),([-\d.]+)</gml:coordinates>.*?<cite:carretera>(.*?)</cite:carretera>.*?<cite:municipi>(.*?)</cite:municipi>.*?<cite:link>(.*?)</cite:link>.*?<cite:font>(.*?)</cite:font>', x, re.S):
+        lon, lat, road, muni, link, font = m.groups()
+        link = link.replace('&amp;', '&').replace('http://', 'https://')
+        if font not in ('SCT', 'IMI', 'Terrassa'):
+            continue
+        city = 'Barcelona' if font == 'IMI' else ('Terrassa' if font == 'Terrassa' else muni)
+        out.append(C('Servei Català de Trànsit', f'{road} {muni}', city, 'Spain', lat, lon, link))
+    return nearest(out, [(41.3874, 2.1686, 'Barcelona'), (41.5636, 2.0111, 'Terrassa'), (41.45, 2.2, 'Barcelona area')], n)
+
+
+@source
+def nsw(n):
+    d = get('https://data.livetraffic.com/cameras/traffic-cam.json').json()
+    out = []
+    for f in d['features']:
+        p = f['properties']
+        href = (p.get('href') or '').replace('http://', 'https://')
+        if not href.startswith('https://webcams.transport.nsw.gov.au/'):
+            continue
+        lon, lat = f['geometry']['coordinates'][:2]
+        out.append(C('Live Traffic NSW', p['title'], '', 'Australia', lat, lon, href))
+    return nearest(out, [(-33.8688, 151.2093, 'Sydney'), (-32.9283, 151.7817, 'Newcastle'), (-34.4278, 150.8931, 'Wollongong'), (-33.8, 151.0, 'Western Sydney')], n)
+
+
+@source
+def singapore(n):
+    d = get('https://api.data.gov.sg/v1/transport/traffic-images').json()
+    out = [C('LTA Singapore (data.gov.sg)', f"Camera {c['camera_id']}", 'Singapore', 'Singapore', c['location']['latitude'], c['location']['longitude'], c['image'])
+           for c in d['items'][0]['cameras']]
+    return sample(out, n)
+
+
+@source
+def hongkong(n):
+    x = ET.fromstring(get('https://static.data.gov.hk/td/traffic-snapshot-images/code/Traffic_Camera_Locations_En.xml').content)
+    out = []
+    for im in x.iter('image'):
+        g = lambda k: (im.findtext(k) or '').strip()
+        if not g('url').startswith('https://tdcctv.data.one.gov.hk/'):
+            continue
+        out.append(C('HK Transport Department', re.sub(r'\s*\[.*?\]$', '', g('description')), f"Hong Kong ({g('district')})", 'Hong Kong', g('latitude'), g('longitude'), g('url')))
+    return sample(out, n)
+
+
+@source
+def nz(n):
+    d = get('https://trafficnz.info/service/traffic/rest/4/cameras/all', headers={'Accept': 'application/json'}).json()
+    out = []
+    for c in d['response']['camera']:
+        if c.get('offline') or c.get('underMaintenance'):
+            continue
+        out.append(C('NZTA Waka Kotahi', (c.get('name') or '').strip() + (' - ' + c['description'] if c.get('description') else ''),
+                     (c.get('region') or {}).get('name', ''), 'New Zealand', c['latitude'], c['longitude'], 'https://trafficnz.info' + c['imageUrl']))
+    return nearest(out, [(-36.8485, 174.7633, 'Auckland'), (-41.2865, 174.7762, 'Wellington'), (-43.5321, 172.6362, 'Christchurch'), (-37.787, 175.279, 'Hamilton'), (-45.0312, 168.6626, 'Queenstown')], n)
+
+
+TARGETS = dict(nsw=26, hongkong=24, singapore=18, nz=20, nyc=20, caltrans=18, wsdot=12, kingco=6, drivebc=14, calgary=8,
+               toronto=10, txdot=10, austin=8, tfl=14, fintraffic=10, vegvesen=10, tallinn=6, tarktee=6, lithuania=8, catalonia=10)
+
+
+def fetch_frame(c):
+    try:
+        hdr = {}
+        if c['kind'] == 'txdot':
+            r = S.get(c['url'], timeout=30, headers={'Accept': 'application/json'})
+            r.raise_for_status()
+            js = r.json()
+            b64 = js.get('snippet') or js.get('Snippet') or ''
+            data = base64.b64decode(b64.split(',')[-1])
+        else:
+            r = S.get(c['url'], timeout=30)
+            r.raise_for_status()
+            data = r.content
+        im = Image.open(io.BytesIO(data))
+        im.load()
+        im = im.convert('RGB')
+        g = im.convert('L')
+        st = ImageStat.Stat(g)
+        edges = ImageStat.Stat(g.resize((320, 180)).filter(ImageFilter.FIND_EDGES)).mean[0]
+        sat = ImageStat.Stat(im.convert('HSV').split()[1]).mean[0]
+        small = g.resize((16, 9))
+        px = list(small.tobytes())
+        ahash = ''.join('1' if p > sum(px) / len(px) else '0' for p in px)
+        h = hashlib.md5(c['url'].encode()).hexdigest()[:12]
+        path = f'{WORK}/frames/{h}.jpg'
+        im.save(path, quality=95)
+        c.update(ok=True, file=path, w=im.width, h=im.height, mean=round(st.mean[0], 1), std=round(st.stddev[0], 1),
+                 edges=round(edges, 2), sat=round(sat, 1), ahash=ahash, bytes=len(data), fetched=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
+    except Exception as e:
+        c.update(ok=False, err=str(e)[:160])
+    return c
+
+
+if __name__ == '__main__':
+    only = sys.argv[2].split(',') if len(sys.argv) > 2 else list(SOURCES)
+    cands = []
+    for name in only:
+        try:
+            got = SOURCES[name](TARGETS.get(name, 8))
+            print(f'{name}: {len(got)} candidates')
+            for g in got:
+                g['src_key'] = name
+            cands += got
+        except Exception as e:
+            print(f'{name}: CATALOG FAILED {e}')
+    with ThreadPoolExecutor(24) as ex:
+        res = list(ex.map(fetch_frame, cands))
+    prev = []
+    pth = f'{WORK}/candidates.json'
+    if os.path.exists(pth):
+        prev = [p for p in json.load(open(pth)) if p['src_key'] not in only]
+    json.dump(prev + res, open(pth, 'w'), ensure_ascii=False, indent=0)
+    from collections import Counter
+    print('ok per source', Counter(c['src_key'] for c in res if c['ok']))
+    print('fail per source', Counter(c['src_key'] for c in res if not c['ok']))
+    for c in res:
+        if not c['ok']:
+            print('  FAIL', c['src_key'], c['url'][:90], c['err'][:100])

intro-sequence/tools/fetch/cctv_select.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""Stage 2: pick a spread of good frames from cctv_fetch.py output, cover-crop to 640x360, write cctv/index.json.
+Usage: python3 -I cctv_select.py <work_dir> <out_cctv_dir>
+EXCLUDE lists frames rejected on visual review of the contact sheets (placeholders such as
+'Temporarily Unavailable', 'Kamera er midlertidig ute av drift', 'NB! ÜHENDUS KAAMERAGA PUUDUB',
+'Interrupció temporal de la imatge', and an all-black King County frame)."""
+import json, os, sys
+from collections import Counter
+from PIL import Image
+W, OUT = sys.argv[1], sys.argv[2]
+ok = json.load(open(f'{W}/ok.json'))
+EXCLUDE = {40, 43, 45, 46, 49, 66, 126, 139, 148, 150}
+PREFER = {'hongkong': [187, 195, 197, 198, 201, 205, 206, 196, 200, 202]}  # fewest privacy-mask polygons
+QUOTA = dict(nsw=12, hongkong=10, singapore=6, nz=12, nyc=9, caltrans=6, wsdot=6, kingco=3, drivebc=6, calgary=4, toronto=6,
+             txdot=4, austin=4, tfl=8, fintraffic=4, vegvesen=4, tallinn=3, tarktee=3, lithuania=4, catalonia=3)
+hash_count = Counter(x['ahash'] for x in ok)
+good = [x for x in ok if x['i'] not in EXCLUDE and hash_count[x['ahash']] == 1 and x['w'] >= 320 and x['mean'] > 25 and x['std'] > 15]
+pick = []
+for src, q in QUOTA.items():
+    pool = [x for x in good if x['src_key'] == src]
+    pref = [x for i in PREFER.get(src, []) for x in pool if x['i'] == i]
+    rest = sorted([x for x in pool if x not in pref], key=lambda x: -(x['std'] * x['edges']))
+    pick += (pref + rest)[:q]
+os.makedirs(OUT, exist_ok=True)
+for f in os.listdir(OUT):
+    if f.startswith('cam_') and f.endswith('.jpg'):
+        os.remove(os.path.join(OUT, f))
+index = []
+for n, x in enumerate(pick, 1):
+    im = Image.open(x['file']).convert('RGB')
+    w, h = im.size
+    s = max(640 / w, 360 / h)
+    im = im.resize((max(640, round(w * s)), max(360, round(h * s))), Image.LANCZOS)
+    w, h = im.size
+    l, t = (w - 640) // 2, (h - 360) // 2
+    im = im.crop((l, t, l + 640, t + 360))
+    fn = f'cam_{n:03d}.jpg'
+    im.save(os.path.join(OUT, fn), quality=88)
+    index.append(dict(file=fn, name=x['name'], city=x['city'], country=x['country'], lat=x['lat'], lon=x['lon'], source=x['source'],
+                      fetched_utc=x['fetched'], orig_size=[x['w'], x['h']], url=x['url']))
+json.dump(index, open(os.path.join(OUT, 'index.json'), 'w'), ensure_ascii=False, indent=1)
+print(len(index), Counter(i['country'] for i in index))

intro-sequence/tools/fetch/fetch_basic.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Re-fetch the raw keyless feeds used by process_basic.py / process_tle.py / process_vessels.py.
+# Usage: fetch_basic.sh <raw_dir>   then:
+#   python3 -I process_basic.py <raw_dir> ../../data
+#   python3 -I process_tle.py <raw_dir> ../../data
+#   python3 -I process_vessels.py <raw_dir> ../../data
+#   <venv with eccodes,numpy,requests>/python -I fetch_wind.py https://noaa-gfs-bdp-pds.s3.amazonaws.com/gfs.YYYYMMDD/HH/atmos/gfs.tHHz.pgrb2.1p00.f006 ../../data/wind.json
+#   python -I fetch_bonus.py ../../data <raw_dir>
+#   python -I cctv_fetch.py <work_dir> && python -I cctv_select.py <work_dir> ../../cctv   (after rebuilding ok.json/contact sheets and re-reviewing EXCLUDE)
+set -u
+R="$1"; mkdir -p "$R"; cd "$R"
+date -u +%Y-%m-%dT%H:%M:%SZ > fetch_start.txt
+CT='gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)'  # CelesTrak 403s bulk groups without a descriptive UA
+f() { curl -sS -m 300 --compressed "${@:3}" -o "$1" -w "$1 %{http_code} %{size_download}\n" "$2"; }
+f opensky.json https://opensky-network.org/api/states/all &
+f mil.json https://api.adsb.lol/v2/mil &
+f mil_adsbfi.json https://opendata.adsb.fi/api/v2/mil &
+f quakes.geojson https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.geojson &
+f viirs_snpp.csv https://firms.modaps.eosdis.nasa.gov/data/active_fire/suomi-npp-viirs-c2/csv/SUOMI_VIIRS_C2_Global_24h.csv &
+f viirs_n20.csv https://firms.modaps.eosdis.nasa.gov/data/active_fire/noaa-20-viirs-c2/csv/J1_VIIRS_C2_Global_24h.csv &
+f modis.csv https://firms.modaps.eosdis.nasa.gov/data/active_fire/modis-c6.1/csv/MODIS_C6_1_Global_24h.csv &
+f cables.json https://www.submarinecablemap.com/api/v3/cable/cable-geo.json &
+f coast50.geojson https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_50m_coastline.geojson &
+f countries110.geojson https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_110m_admin_0_countries.geojson &
+f places.geojson https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_populated_places_simple.geojson &
+f airports.csv https://davidmegginson.github.io/ourairports-data/airports.csv &
+f kystverket_ais.json https://kystdatahuset.kystverket.no/ws/api/ais/realtime/geojson &
+f dt_ais_from.json "https://meri.digitraffic.fi/api/ais/v1/locations?from=$(( ($(date +%s)-172800)*1000 ))" -H "Digitraffic-User: gods-eye-intro" &
+f digitraffic_vessels.json https://meri.digitraffic.fi/api/ais/v1/vessels -H "Digitraffic-User: gods-eye-intro" &
+wait
+# CelesTrak: sequential (parallel requests get connection resets); FORMAT=3le endpoint answered when FORMAT=tle timed out
+f tle_active3.txt "https://celestrak.org/NORAD/elements/gp.php?GROUP=active&FORMAT=3le" -A "$CT"
+for g in stations starlink gps-ops geo weather; do sleep 3; f tle_$g.txt "https://celestrak.org/NORAD/elements/gp.php?GROUP=$g&FORMAT=tle" -A "$CT"; done

intro-sequence/tools/fetch/fetch_bonus.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Bonus layers: radio stations (Radio Browser), GBFS bike stations, launch pads (Launch Library 2),
+power plants (WRI Global Power Plant Database).  Usage: python3 -I fetch_bonus.py <out_data_dir> <raw_dir>"""
+import csv, io, json, os, sys, time, zipfile
+import requests
+
+OUT, RAW = sys.argv[1], sys.argv[2]
+only = sys.argv[3].split(',') if len(sys.argv) > 3 else ['radio', 'gbfs', 'pads', 'power']
+S = requests.Session()
+S.headers['User-Agent'] = 'GodsEyeView/1.0 intro-sequence data snapshot'
+log = {}
+
+
+def dump(name, obj):
+    with open(os.path.join(OUT, name), 'w') as f:
+        json.dump(obj, f, separators=(',', ':'), ensure_ascii=False)
+
+
+if 'radio' in only:
+    rows, used = [], None
+    for host in ['https://de1.api.radio-browser.info', 'https://de2.api.radio-browser.info', 'https://nl1.api.radio-browser.info']:
+        try:
+            r = S.get(f'{host}/json/stations/search', params={'has_geo_info': 'true', 'hidebroken': 'true', 'limit': 100000}, timeout=180,
+                      headers={'User-Agent': 'GodsEyeView/1.0 (Radio Browser directory client)'})
+            r.raise_for_status()
+            data = r.json(); used = host
+            break
+        except Exception as e:
+            print('radio', host, e)
+    seen = set()
+    for s in data:
+        lat, lon = s.get('geo_lat'), s.get('geo_long')
+        if lat is None or lon is None or (abs(lat) < 0.01 and abs(lon) < 0.01):
+            continue
+        key = (round(lon, 3), round(lat, 3), s['name'].strip())
+        if key in seen:
+            continue
+        seen.add(key)
+        rows.append([round(lon, 4), round(lat, 4), s['name'].strip(), s.get('countrycode') or ''])
+    dump('radio.json', {'cols': ['lon', 'lat', 'name', 'cc'], 'rows': rows})
+    log['radio'] = dict(url=f'{used}/json/stations/search?has_geo_info=true&hidebroken=true&limit=100000', rows=len(rows))
+    print('radio', len(rows))
+
+if 'gbfs' in only:
+    SYSTEMS = {
+        'Citi Bike (New York)': 'https://gbfs.lyft.com/gbfs/2.3/bkn/en/station_information.json',
+        'Divvy (Chicago)': 'https://gbfs.lyft.com/gbfs/2.3/chi/en/station_information.json',
+        'Bay Wheels (SF Bay)': 'https://gbfs.lyft.com/gbfs/2.3/bay/en/station_information.json',
+        'Capital Bikeshare (DC)': 'https://gbfs.lyft.com/gbfs/2.3/dca-cabi/en/station_information.json',
+        'Bluebikes (Boston)': 'https://gbfs.bluebikes.com/gbfs/en/station_information.json',
+        'Bike Share Toronto': 'https://toronto.publicbikesystem.net/customer/gbfs/v2/en/station_information.json',
+        'BIXI (Montreal)': 'https://gbfs.velobixi.com/gbfs/en/station_information.json',
+        'Ecobici (Mexico City)': 'https://gbfs.mex.lyftbikes.com/gbfs/en/station_information.json',
+        'Velib Metropole (Paris)': 'https://velib-metropole-opendata.smovengo.cloud/opendata/Velib_Metropole/station_information.json',
+        'Bicing (Barcelona)': 'https://barcelona.publicbikesystem.net/customer/gbfs/v2/en/station_information.json',
+        'BiciMAD (Madrid)': 'https://madrid.publicbikesystem.net/customer/gbfs/v2/en/station_information.json',
+        'Bike Sampa (Sao Paulo)': 'https://saopaulo.publicbikesystem.net/customer/gbfs/v2/en/station_information.json',
+        'Bike Rio (Rio de Janeiro)': 'https://riodejaneiro.publicbikesystem.net/customer/gbfs/v2/en/station_information.json',
+        'Bike Santiago': 'https://santiago.publicbikesystem.net/customer/gbfs/v2/en/station_information.json',
+        'Tembici Bogota': 'https://bogota.publicbikesystem.net/customer/gbfs/v2/en/station_information.json',
+        'Indego (Philadelphia)': 'https://gbfs.bcycle.com/bcycle_indego/station_information.json',
+        'Metro Bike Share (LA)': 'https://gbfs.bcycle.com/bcycle_lametro/station_information.json',
+        'Oslo Bysykkel': 'https://gbfs.urbansharing.com/oslobysykkel.no/station_information.json',
+    }
+    rows, per = [], {}
+    for name, url in SYSTEMS.items():
+        try:
+            d = S.get(url, timeout=60, headers={'Client-Identifier': 'gods-eye-view'}).json()
+            st = d['data']['stations']
+            for s in st:
+                rows.append([round(float(s['lon']), 5), round(float(s['lat']), 5)])
+            per[name] = len(st)
+        except Exception as e:
+            per[name] = f'FAILED {str(e)[:80]}'
+    # nextbike: one keyless global live feed covering every nextbike city
+    try:
+        d = S.get('https://api.nextbike.net/maps/nextbike-live.json', timeout=120).json()
+        n = 0
+        for c in d['countries']:
+            for city in c.get('cities', []):
+                for p in city.get('places', []):
+                    if p.get('spot'):  # stations only, not free-floating bikes
+                        rows.append([round(p['lng'], 5), round(p['lat'], 5)]); n += 1
+        per['nextbike (all cities, global)'] = n
+    except Exception as e:
+        per['nextbike'] = f'FAILED {e}'
+    dump('bikes.json', {'cols': ['lon', 'lat'], 'systems': per, 'rows': rows})
+    log['gbfs'] = dict(systems=per, urls=list(SYSTEMS.values()) + ['https://api.nextbike.net/maps/nextbike-live.json'], rows=len(rows))
+    print('bikes', len(rows), per)
+
+if 'pads' in only:
+    url = 'https://ll.thespacedevs.com/2.3.0/pads/?limit=100&mode=list'
+    rows = []
+    while url:
+        r = S.get(url, timeout=60)
+        r.raise_for_status()
+        d = r.json()
+        for p in d['results']:
+            try:
+                lat, lon = float(p['latitude']), float(p['longitude'])
+            except (TypeError, ValueError, KeyError):
+                continue
+            loc = p.get('location') or {}
+            rows.append([round(lon, 4), round(lat, 4), p['name'], loc.get('name') if isinstance(loc, dict) else None,
+                         p.get('total_launch_count') or 0, bool(p.get('active'))])
+        url = d.get('next')
+        time.sleep(1)
+    rows.sort(key=lambda x: -x[4])
+    dump('launch_sites.json', {'cols': ['lon', 'lat', 'pad', 'location', 'total_launches', 'active'], 'rows': rows})
+    log['pads'] = dict(url='https://ll.thespacedevs.com/2.3.0/pads/?limit=100&mode=list (paged)', rows=len(rows))
+    print('pads', len(rows))
+
+if 'power' in only:
+    url = 'https://raw.githubusercontent.com/wri/global-power-plant-database/master/output_database/global_power_plant_database.csv'
+    r = S.get(url, timeout=180)
+    r.raise_for_status()
+    rows = []
+    for row in csv.DictReader(io.StringIO(r.content.decode('utf-8'))):
+        try:
+            rows.append([round(float(row['longitude']), 4), round(float(row['latitude']), 4), round(float(row['capacity_mw']), 1), row['primary_fuel'], row['name'], row['country']])
+        except ValueError:
+            pass
+    rows.sort(key=lambda x: -x[2])
+    dump('power_plants.json', {'cols': ['lon', 'lat', 'capacity_mw', 'fuel', 'name', 'iso3'], 'rows': rows})
+    log['power'] = dict(url=url, rows=len(rows))
+    print('power', len(rows))
+
+prev = {}
+p = os.path.join(RAW, 'bonus_log.json')
+if os.path.exists(p):
+    prev = json.load(open(p))
+prev.update(log)
+json.dump(prev, open(p, 'w'), indent=1)

intro-sequence/tools/fetch/fetch_wind.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+"""GFS 1.0 deg 10 m wind U/V from NOAA's AWS Open Data bucket (noaa-gfs-bdp-pds) via .idx byte
+ranges, same approach as server/providers/wind/gfs.js; decoded with python `eccodes` (pip).
+Usage: python fetch_wind.py <gfs file base url (no .idx)> <out wind.json>"""
+import json, sys, requests, numpy as np, eccodes
+base, out = sys.argv[1], sys.argv[2]
+lines = [l.split(':') for l in requests.get(base + '.idx', timeout=60).text.strip().split('\n')]
+def field(var):
+    i = next(k for k, p in enumerate(lines) if p[3] == var and p[4] == '10 m above ground')
+    s, e = int(lines[i][1]), int(lines[i + 1][1]) - 1
+    b = requests.get(base, headers={'Range': f'bytes={s}-{e}'}, timeout=120).content
+    h = eccodes.codes_new_from_message(b)
+    g = {k: eccodes.codes_get(h, k) for k in ['Ni', 'Nj', 'dataDate', 'dataTime', 'step']}
+    g.update({k: eccodes.codes_get_double(h, k) for k in ['longitudeOfFirstGridPointInDegrees', 'latitudeOfFirstGridPointInDegrees', 'iDirectionIncrementInDegrees', 'jDirectionIncrementInDegrees']})
+    g['values'] = eccodes.codes_get_values(h).reshape(g['Nj'], g['Ni'])
+    eccodes.codes_release(h)
+    return g
+U, V = field('UGRD'), field('VGRD')
+assert U['longitudeOfFirstGridPointInDegrees'] == 0 and U['latitudeOfFirstGridPointInDegrees'] == 90
+dx = U['iDirectionIncrementInDegrees']; shift = int(round(180 / dx))
+u = np.roll(U['values'], -shift, axis=1); v = np.roll(V['values'], -shift, axis=1)  # lon 0..359 -> -180..179
+ny, nx = u.shape
+json.dump({'source': 'NOAA GFS 1.0deg ' + base.split('/')[-1], 'refTime': f"{U['dataDate']} {U['dataTime']:04d}Z", 'forecastHour': U['step'],
+           'level': '10 m above ground', 'units': 'm/s', 'nx': nx, 'ny': ny, 'lo1': -180, 'la1': 90, 'dx': dx, 'dy': U['jDirectionIncrementInDegrees'],
+           'u': [round(float(x), 2) for x in u.ravel()], 'v': [round(float(x), 2) for x in v.ravel()]},
+          open(out, 'w'), separators=(',', ':'))
+sp = np.hypot(u, v)
+print(nx, ny, 'max', sp.max().round(1), 'mean', sp.mean().round(2), U['dataDate'], U['dataTime'], 'f', U['step'])

intro-sequence/tools/fetch/process_basic.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Convert raw fetched feeds (see fetch_basic.sh) into compact intro-sequence JSON.
+
+Usage: python3 -I process_basic.py <raw_dir> <out_data_dir>
+All inputs are real downloads; this only filters, reprojects units and rounds.
+"""
+import csv, json, os, sys, time
+
+RAW, OUT = sys.argv[1], sys.argv[2]
+meta = {}
+
+
+def r(x, n):
+    return None if x is None else round(float(x), n)
+
+
+def dump(name, obj):
+    p = os.path.join(OUT, name)
+    with open(p, 'w') as f:
+        json.dump(obj, f, separators=(',', ':'), ensure_ascii=False)
+    print(f'{name}: {os.path.getsize(p)/1e6:.2f} MB')
+
+
+def raw(name):
+    return os.path.join(RAW, name)
+
+
+# 1. flights (OpenSky)
+o = json.load(open(raw('opensky.json')))
+rows = []
+for s in o['states']:
+    lon, lat, on_ground = s[5], s[6], s[8]
+    if lon is None or lat is None or on_ground:
+        continue
+    alt = s[13] if s[13] is not None else s[7]
+    rows.append([r(lon, 4), r(lat, 4), r(alt, 0), r(s[10], 1), r(s[9], 1),
+                 (s[1] or '').strip(), s[2]])
+dump('flights.json', {'t': o['time'], 'cols': ['lon', 'lat', 'alt_m', 'track_deg', 'vel_ms', 'callsign', 'country'], 'rows': rows})
+meta['flights'] = len(rows)
+
+# 2. military (adsb.lol /v2/mil, unioned with adsb.fi /v2/mil, dedup by hex; adsb.lol wins)
+FT, KT = 0.3048, 0.514444
+seen, rows, tmax = set(), [], 0
+for fname in ['mil.json', 'mil_adsbfi.json']:
+    if not os.path.exists(raw(fname)):
+        continue
+    d = json.load(open(raw(fname)))
+    tmax = max(tmax, int((d.get('now') or 0) / (1000 if (d.get('now') or 0) > 1e11 else 1)))
+    for a in d.get('ac') or d.get('aircraft') or []:
+        lat, lon = a.get('lat'), a.get('lon')
+        if lat is None or lon is None or a['hex'] in seen:
+            continue
+        alt = a.get('alt_geom', a.get('alt_baro'))
+        if a.get('alt_baro') == 'ground' or alt == 'ground':
+            continue  # airborne only
+        seen.add(a['hex'])
+        track = a.get('track', a.get('true_heading'))
+        gs = a.get('gs')
+        rows.append([r(lon, 4), r(lat, 4), r(alt * FT, 0) if isinstance(alt, (int, float)) else None,
+                     r(track, 1), r(gs * KT, 1) if gs is not None else None,
+                     (a.get('flight') or '').strip(), None, a.get('t')])
+dump('military.json', {'t': tmax, 'cols': ['lon', 'lat', 'alt_m', 'track_deg', 'vel_ms', 'callsign', 'country', 'type'], 'rows': rows,
+                       'note': 'country not provided by adsb.lol/adsb.fi feeds (null)'})
+meta['military'] = len(rows)
+
+# 4. quakes
+q = json.load(open(raw('quakes.geojson')))
+rows = []
+for f in q['features']:
+    c, p = f['geometry']['coordinates'], f['properties']
+    if p.get('mag') is None:
+        continue
+    rows.append([r(c[0], 3), r(c[1], 3), r(c[2], 1), r(p['mag'], 2), p['time'], p.get('place') or ''])
+rows.sort(key=lambda x: x[4])
+dump('quakes.json', {'t': q['metadata']['generated'], 'cols': ['lon', 'lat', 'depth_km', 'mag', 'time_ms', 'place'], 'rows': rows})
+meta['quakes'] = len(rows)
+
+# 5. fires (FIRMS 24h global: SNPP VIIRS, NOAA-20 VIIRS, MODIS)
+rows, per = [], {}
+for fname, bcol, tag in [('viirs_snpp.csv', 'bright_ti4', 'VIIRS_SNPP'), ('viirs_n20.csv', 'bright_ti4', 'VIIRS_NOAA20'), ('modis.csv', 'brightness', 'MODIS')]:
+    if not os.path.exists(raw(fname)):
+        continue
+    n = 0
+    with open(raw(fname), newline='') as fh:
+        for row in csv.DictReader(fh):
+            try:
+                rows.append([r(row['longitude'], 3), r(row['latitude'], 3), r(row['frp'], 1), r(row[bcol], 1)])
+                n += 1
+            except (KeyError, ValueError):
+                pass
+    per[tag] = n
+dump('fires.json', {'cols': ['lon', 'lat', 'frp', 'brightness'], 'sources': per, 'rows': rows})
+meta['fires'] = len(rows)
+meta['fires_per'] = per
+
+# 8. cables
+c = json.load(open(raw('cables.json')))
+lines = []
+for f in c['features']:
+    g = f['geometry']
+    parts = g['coordinates'] if g['type'] == 'MultiLineString' else [g['coordinates']]
+    for part in parts:
+        pl = [[r(x, 2), r(y, 2)] for x, y in (pt[:2] for pt in part)]
+        if len(pl) >= 2:
+            lines.append(pl)
+dump('cables.json', {'cables': lines})
+meta['cables'] = (len(c['features']), len(lines))
+
+
+# 9. coastline + borders
+def lines_of(geo):
+    out = []
+    for f in geo['features']:
+        g = f['geometry']
+        if g is None:
+            continue
+        t, co = g['type'], g['coordinates']
+        if t == 'LineString':
+            parts = [co]
+        elif t == 'MultiLineString':
+            parts = co
+        elif t == 'Polygon':
+            parts = co
+        elif t == 'MultiPolygon':
+            parts = [ring for poly in co for ring in poly]
+        else:
+            parts = []
+        for part in parts:
+            pl, last = [], None
+            for x, y in (pt[:2] for pt in part):
+                pt = [r(x, 2), r(y, 2)]
+                if pt != last:
+                    pl.append(pt)
+                last = pt
+            if len(pl) >= 2:
+                out.append(pl)
+    return out
+
+
+coast = lines_of(json.load(open(raw('coast50.geojson'))))
+dump('coastline.json', {'lines': coast})
+borders = lines_of(json.load(open(raw('countries110.geojson'))))
+dump('borders.json', {'lines': borders})
+meta['coast'] = len(coast)
+meta['borders'] = len(borders)
+
+# 10. cities
+pp = json.load(open(raw('places.geojson')))
+rows = []
+for f in pp['features']:
+    p = f['properties']
+    lon, lat = f['geometry']['coordinates'][:2]
+    rows.append([r(lon, 3), r(lat, 3), int(p.get('pop_max') or 0), p.get('name') or p.get('nameascii'), p.get('adm0name') or p.get('sov0name')])
+rows.sort(key=lambda x: -x[2])
+rows = rows[:3000]
+dump('cities.json', {'cols': ['lon', 'lat', 'pop_max', 'name', 'country'], 'rows': rows})
+meta['cities'] = len(rows)
+
+# 11. airports (OurAirports large + medium)
+rows = []
+with open(raw('airports.csv'), newline='', encoding='utf-8') as fh:
+    for row in csv.DictReader(fh):
+        if row['type'] not in ('large_airport', 'medium_airport'):
+            continue
+        rows.append([r(row['longitude_deg'], 4), r(row['latitude_deg'], 4), row['iata_code'] or row['ident'], row['name'],
+                     'L' if row['type'] == 'large_airport' else 'M'])
+rows.sort(key=lambda x: x[4])
+dump('airports.json', {'cols': ['lon', 'lat', 'iata', 'name', 'size'], 'rows': rows})
+meta['airports'] = (len(rows), sum(1 for x in rows if x[4] == 'L'))
+
+json.dump(meta, open(os.path.join(RAW, 'meta_basic.json'), 'w'))
+print(meta)

intro-sequence/tools/fetch/process_tle.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+"""tle.txt (3-line TLEs, CelesTrak GROUP=active) + tle_groups.json (NORAD ids per group).
+Usage: python3 -I process_tle.py <raw_dir> <out_dir>"""
+import json, os, sys
+RAW, OUT = sys.argv[1], sys.argv[2]
+def parse(path):
+    L = [l.rstrip() for l in open(path) if l.strip()]
+    out = []
+    i = 0
+    while i + 2 < len(L) + 0 and i + 2 <= len(L) - 1:
+        if L[i+1].startswith('1 ') and L[i+2].startswith('2 '):
+            name = L[i][2:] if L[i].startswith('0 ') else L[i]
+            out.append((name.strip(), L[i+1], L[i+2])); i += 3
+        else:
+            i += 1
+    return out
+act = parse(os.path.join(RAW, 'tle_active3.txt'))
+with open(os.path.join(OUT, 'tle.txt'), 'w') as f:
+    for n, a, b in act:
+        f.write(f'{n}\n{a}\n{b}\n')
+groups, notable = {}, {}
+for g in ['stations', 'starlink', 'gps-ops', 'geo', 'weather']:
+    p = os.path.join(RAW, f'tle_{g}.txt')
+    if os.path.exists(p) and os.path.getsize(p) > 0:
+        sats = parse(p)
+        groups[g] = sorted(int(a[2:7]) for _, a, _ in sats)
+        for n, _, _ in sats[: (40 if g != 'starlink' else 3)]:
+            notable[n] = g
+epochs = sorted(a[18:32] for _, a, _ in act)
+json.dump({'groups_norad': groups, 'notable': notable}, open(os.path.join(OUT, 'tle_groups.json'), 'w'), separators=(',', ':'))
+print(len(act), {k: len(v) for k, v in groups.items()}, 'epoch range', epochs[0], epochs[-1])

intro-sequence/tools/fetch/process_vessels.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""vessels.json from keyless AIS: Fintraffic Digitraffic (Finland/Baltic) + Kystverket Kystdatahuset realtime (Norway).
+Usage: python3 -I process_vessels.py <raw_dir> <out_dir>"""
+import json, os, sys, time
+RAW, OUT = sys.argv[1], sys.argv[2]
+rows, src = {}, {}
+# Norway (Kystverket) - LineString trails, last coordinate = latest position
+k = json.load(open(os.path.join(RAW, 'kystverket_ais.json')))
+for f in k['features']:
+    c, p = f['geometry']['coordinates'], f['properties']
+    if not c:
+        continue
+    lon, lat = c[-1]
+    cog = p.get('cog')
+    cog = None if cog is None or cog >= 360 else round(cog, 1)
+    sog = p.get('speed')
+    sog = None if sog is None or sog >= 102.3 else round(sog, 1)
+    rows[p['mmsi']] = [round(lon, 4), round(lat, 4), sog, cog, p['mmsi'], (p.get('ship_name') or '').strip() or None]
+    src[p['mmsi']] = 'NO'
+# Finland / Baltic (Digitraffic)
+names = {v['mmsi']: (v.get('name') or '').strip() or None for v in json.load(open(os.path.join(RAW, 'digitraffic_vessels.json')))}
+d = json.load(open(os.path.join(RAW, 'dt_ais_from.json')))
+for f in d['features']:
+    p = f['properties']
+    lon, lat = f['geometry']['coordinates'][:2]
+    if p['mmsi'] in rows:
+        continue
+    cog = p.get('cog'); cog = None if cog is None or cog >= 360 else round(cog, 1)
+    sog = p.get('sog'); sog = None if sog is None or sog >= 102.3 else round(sog, 1)
+    rows[p['mmsi']] = [round(lon, 4), round(lat, 4), sog, cog, p['mmsi'], names.get(p['mmsi'])]
+    src[p['mmsi']] = 'FI'
+out = list(rows.values())
+json.dump({'t': int(time.time()), 'cols': ['lon', 'lat', 'sog_kn', 'cog_deg', 'mmsi', 'name'],
+           'coverage': 'Norwegian coast/EEZ (Kystverket) + Finnish waters & Baltic Sea (Fintraffic Digitraffic); not global',
+           'counts': {'kystverket_norway': sum(1 for v in src.values() if v == 'NO'), 'digitraffic_finland_baltic': sum(1 for v in src.values() if v == 'FI')},
+           'rows': out}, open(os.path.join(OUT, 'vessels.json'), 'w'), separators=(',', ':'), ensure_ascii=False)
+lons = [r[0] for r in out]; lats = [r[1] for r in out]
+print(len(out), 'bbox', min(lons), max(lons), min(lats), max(lats))