mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
feat(intro): add the First Light intro sequence
A deterministic WebGL film of God's Eye View coming online: a seed on Null Island grows into a culture that draws the planet, indexes the surface tile by tile, ingests the live feeds and outgrows its dish, ending on the logo. Renders 16:9 and 9:16 frame by frame in headless Chromium; the procedural score is locked to the same clock. Data and imagery are fetched by the scripts in tools/ and are not committed. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01KqP857pippaUs6konSDfe8
This commit is contained in:
37 files changed
+15507
No files matched your search
@@ -0,0 +1,511 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Count distinct cameras in every keyless CCTV catalog the app supports.
|
||||
|
||||
Fetches each source's FULL public catalog (no anchor selection, no
|
||||
DEFAULT_*_MAX_SOURCES caps) and counts distinct cameras two ways:
|
||||
raw - distinct camera ids listed in the catalog at all
|
||||
valid - distinct ids passing the app's own validity filters (in service /
|
||||
online / official still or stream URL / plausible coordinates),
|
||||
i.e. what server/providers/cctv/* would accept before capping.
|
||||
The headline number written to data/extra_counts.json is the conservative
|
||||
`valid` count. Sources that fail are skipped, never estimated.
|
||||
|
||||
Usage: python3 -I cctv_count.py <out_json> [<detail_json>]
|
||||
"""
|
||||
import json, math, re, sys, time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
import requests
|
||||
|
||||
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
|
||||
REPO = '/home/user/gods-eye-view'
|
||||
|
||||
|
||||
def sess():
|
||||
s = requests.Session()
|
||||
s.headers.update({'User-Agent': UA, 'Digitraffic-User': 'gods-eye-view'})
|
||||
return s
|
||||
|
||||
|
||||
def get(url, timeout=60, **kw):
|
||||
r = sess().get(url, timeout=timeout, **kw)
|
||||
r.raise_for_status()
|
||||
return r
|
||||
|
||||
|
||||
def fnum(v):
|
||||
try:
|
||||
f = float(v)
|
||||
return f if math.isfinite(f) else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def inbox(lat, lon, la0, la1, lo0, lo1):
|
||||
return lat is not None and lon is not None and la0 <= lat <= la1 and lo0 <= lon <= lo1
|
||||
|
||||
|
||||
SRC = {}
|
||||
|
||||
|
||||
def source(label, country):
|
||||
def deco(fn):
|
||||
SRC[label] = (fn, country)
|
||||
return fn
|
||||
return deco
|
||||
|
||||
|
||||
@source('TfL JamCams', 'United Kingdom')
|
||||
def tfl():
|
||||
d = get('https://api.tfl.gov.uk/Place/Type/JamCam').json()
|
||||
raw, ok = set(), set()
|
||||
for p in d:
|
||||
raw.add(p['id'])
|
||||
props = {a['key']: a['value'] for a in p.get('additionalProperties', [])}
|
||||
if str(props.get('available')).lower() != 'true':
|
||||
continue
|
||||
if not (props.get('imageUrl') or '').startswith('https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/'):
|
||||
continue
|
||||
if fnum(p.get('lat')) is None or fnum(p.get('lon')) is None:
|
||||
continue
|
||||
ok.add(p['id'])
|
||||
return raw, ok, 'one Place per camera; available=true + official S3 still'
|
||||
|
||||
|
||||
@source('NYC DOT', 'United States')
|
||||
def nyc():
|
||||
d = get('https://webcams.nyctmc.org/api/cameras').json()
|
||||
raw = {c['id'] for c in d}
|
||||
ok = {c['id'] for c in d if str(c.get('isOnline')).strip().lower() == 'true'
|
||||
and inbox(fnum(c.get('latitude')), fnum(c.get('longitude')), 40.4, 41.0, -74.3, -73.6)}
|
||||
return raw, ok, 'one row per camera; isOnline=true'
|
||||
|
||||
|
||||
@source('Caltrans', 'United States')
|
||||
def caltrans():
|
||||
raw, ok = set(), set()
|
||||
notes = []
|
||||
|
||||
def one(dist):
|
||||
err = None
|
||||
for attempt in range(5): # cwwp2 resets connections under parallel load
|
||||
try:
|
||||
return dist, get(f'https://cwwp2.dot.ca.gov/data/d{dist}/cctv/cctvStatusD{dist:02d}.json').json()
|
||||
except Exception as e:
|
||||
err = e
|
||||
time.sleep(1 + attempt)
|
||||
return dist, err
|
||||
with ThreadPoolExecutor(3) as ex:
|
||||
res = list(ex.map(one, range(1, 13)))
|
||||
for dist, d in res:
|
||||
if isinstance(d, Exception):
|
||||
notes.append(f'D{dist} failed: {str(d)[:60]}')
|
||||
continue
|
||||
for i, row in enumerate(d.get('data', [])):
|
||||
c = row.get('cctv') or {}
|
||||
loc = c.get('location') or {}
|
||||
name = (loc.get('locationName') or '').strip()
|
||||
m = re.match(r'^([A-Za-z0-9_-]+)\s*--', name)
|
||||
key = f"d{dist}-{(m.group(1) if m else c.get('index') or name or i)}".lower()
|
||||
raw.add(key)
|
||||
if str(c.get('inService')).lower() != 'true':
|
||||
continue
|
||||
if fnum(loc.get('latitude')) is None or fnum(loc.get('longitude')) is None:
|
||||
continue
|
||||
url = ((c.get('imageData') or {}).get('static') or {}).get('currentImageURL') or ''
|
||||
if not url.startswith('https://cwwp2.dot.ca.gov/'):
|
||||
continue
|
||||
ok.add(key)
|
||||
return raw, ok, 'all 12 districts; inService=true + cwwp2 still. ' + '; '.join(notes)
|
||||
|
||||
|
||||
@source('WSDOT', 'United States')
|
||||
def wsdot():
|
||||
d = json.loads(get('https://data.wsdot.wa.gov/travelcenter/Cameras.json').content.decode('cp1252'))
|
||||
raw, ok = set(), set()
|
||||
for f in d['features']:
|
||||
a = f.get('attributes') or {}
|
||||
cid = a.get('CameraID')
|
||||
if not isinstance(cid, int) or cid <= 0:
|
||||
continue
|
||||
raw.add(cid)
|
||||
if (a.get('ImageURL') or '').strip().startswith('https://images.wsdot.wa.gov/') and fnum((f.get('geometry') or {}).get('x')) is not None:
|
||||
ok.add(cid)
|
||||
return raw, ok, 'one feature per CameraID; official images.wsdot.wa.gov still'
|
||||
|
||||
|
||||
@source('King County DOT', 'United States')
|
||||
def kingco():
|
||||
d = get('https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson').json()
|
||||
raw, ok = set(), set()
|
||||
for f in d['features']:
|
||||
p = f.get('properties') or {}
|
||||
aid = p.get('AssetID')
|
||||
if not isinstance(aid, int) or aid <= 0:
|
||||
continue
|
||||
raw.add(aid)
|
||||
if p.get('Live') != 1 or str(p.get('CurrentStatus') or '').strip() != 'Active':
|
||||
continue
|
||||
if str(p.get('Owner') or '').strip() == 'WSDOT':
|
||||
continue # WSDOT-owned rows are WSDOT cameras (counted under WSDOT)
|
||||
url = (p.get('ImageURL') or '').strip().replace('http://info.kingcounty.gov/', 'https://info.kingcounty.gov/')
|
||||
if not url.startswith('https://info.kingcounty.gov/'):
|
||||
continue
|
||||
ok.add(aid)
|
||||
note = 'Live=1 + Active, WSDOT-owned rows excluded (avoid double count)'
|
||||
if d.get('exceededTransferLimit') or (d.get('properties') or {}).get('exceededTransferLimit'):
|
||||
note += ' WARNING: transfer limit exceeded'
|
||||
return raw, ok, note
|
||||
|
||||
|
||||
@source('DriveBC', 'Canada')
|
||||
def drivebc():
|
||||
d = get('https://www.drivebc.ca/api/webcams/').json()
|
||||
raw, ok = set(), set()
|
||||
for c in d:
|
||||
cid = c.get('id')
|
||||
if not isinstance(cid, int) or cid <= 0:
|
||||
continue
|
||||
raw.add(cid)
|
||||
if c.get('is_on') is not True or c.get('should_appear') is not True:
|
||||
continue
|
||||
coords = (c.get('location') or {}).get('coordinates') or [None, None]
|
||||
if not inbox(fnum(coords[1]), fnum(coords[0]), 48.0, 60.5, -139.5, -113.5):
|
||||
continue
|
||||
ok.add(cid)
|
||||
return raw, ok, 'one webcam per id; is_on + should_appear'
|
||||
|
||||
|
||||
@source('Fintraffic Digitraffic', 'Finland')
|
||||
def fintraffic():
|
||||
d = get('https://tie.digitraffic.fi/api/weathercam/v1/stations', headers={'Accept-Encoding': 'gzip'}).json()
|
||||
raw, ok, presets = set(), set(), 0
|
||||
for f in d['features']:
|
||||
p = f.get('properties') or {}
|
||||
sid = str(p.get('id') or '')
|
||||
if not sid:
|
||||
continue
|
||||
raw.add(sid)
|
||||
if str(p.get('collectionStatus') or '').upper() != 'GATHERING':
|
||||
continue
|
||||
live = [pr for pr in p.get('presets') or [] if pr.get('inCollection') is True and re.fullmatch(r'C\d{7}', str(pr.get('id'))) and str(pr['id']).startswith(sid)]
|
||||
if live:
|
||||
ok.add(sid)
|
||||
presets += len(live)
|
||||
return raw, ok, f'counted per STATION (camera site), not per preset view; {presets} live presets (the app treats each preset as a camera)'
|
||||
|
||||
|
||||
@source('Statens vegvesen', 'Norway')
|
||||
def vegvesen():
|
||||
d = get('https://ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items?f=application/geo%2Bjson&limit=5000',
|
||||
headers={'Accept': 'application/geo+json,application/json'}).json()
|
||||
raw, ok = set(), set()
|
||||
for f in d['features']:
|
||||
p = f.get('properties') or {}
|
||||
rid = str(p.get('cameraId') or '').strip()
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', rid):
|
||||
continue
|
||||
raw.add(rid)
|
||||
av = str(p.get('status.stillImageAvailability') or '').strip()
|
||||
if av and av != 'videoOrImagesAvailable':
|
||||
continue
|
||||
if (p.get('stillImageUrl') or '') != f'https://kamera.atlas.vegvesen.no/api/images/{rid}':
|
||||
continue
|
||||
c = (f.get('geometry') or {}).get('coordinates') or [None, None]
|
||||
if not inbox(fnum(c[1]), fnum(c[0]), 57.5, 71.5, 4.0, 31.5):
|
||||
continue
|
||||
ok.add(rid)
|
||||
note = f"numberMatched={d.get('numberMatched')} numberReturned={d.get('numberReturned')}"
|
||||
return raw, ok, 'stillImageAvailability ok + official still; ' + note
|
||||
|
||||
|
||||
@source('City of Tallinn', 'Estonia')
|
||||
def tallinn():
|
||||
d = json.load(open(f'{REPO}/config/cctv_sources.tallinn.json'))
|
||||
raw, ok = set(), set()
|
||||
for c in d:
|
||||
cid = str(c.get('id') or '').strip()
|
||||
if not cid:
|
||||
continue
|
||||
raw.add(cid)
|
||||
url = c.get('snapshotUrl') or c.get('url') or ''
|
||||
if inbox(fnum(c.get('lat')), fnum(c.get('lon')), 59.2, 59.7, 24.3, 25.4) and url.startswith('https://ristmikud.tallinn.ee/'):
|
||||
ok.add(url)
|
||||
return raw, ok, 'curated repo catalog (config/cctv_sources.tallinn.json) of ristmikud.tallinn.ee stills; distinct frame URLs'
|
||||
|
||||
|
||||
@source('Transpordiamet Tarktee', 'Estonia')
|
||||
def tarktee():
|
||||
lx = get('https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraLocations', headers={'Accept': 'application/xml'}).text
|
||||
ix = get('https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraImages', headers={'Accept': 'application/xml'}).text
|
||||
locs = {}
|
||||
for m in re.finditer(r'<predefinedLocation\s+id="([^"]+)"[^>]*>([\s\S]*?)</predefinedLocation>', lx):
|
||||
la = re.search(r'<latitude>\s*(-?\d+(?:\.\d+)?)\s*</latitude>', m.group(2), re.I)
|
||||
lo = re.search(r'<longitude>\s*(-?\d+(?:\.\d+)?)\s*</longitude>', m.group(2), re.I)
|
||||
if la and lo:
|
||||
locs[m.group(1)] = (float(la.group(1)), float(lo.group(1)))
|
||||
imgs = {}
|
||||
for m in re.finditer(r'<trafficView\b[^>]*>([\s\S]*?)</trafficView>', ix):
|
||||
r = re.search(r'<linearPredefinedLocationReference\b[^>]*\bid="([^"]+)"', m.group(1), re.I)
|
||||
u = re.search(r'<urlLinkAddress>\s*([^<\s]+)\s*</urlLinkAddress>', m.group(1), re.I)
|
||||
if r and u:
|
||||
imgs[r.group(1)] = u.group(1)
|
||||
raw = set(locs) | set(imgs)
|
||||
ok = set()
|
||||
for lid, (lat, lon) in locs.items():
|
||||
u = imgs.get(lid, '')
|
||||
if not u.startswith('https://tarktee.transpordiamet.ee/images/'):
|
||||
continue
|
||||
if not inbox(lat, lon, 57.4, 59.9, 21.5, 28.4):
|
||||
continue
|
||||
n = re.search(r'/images/(\d+)/', u)
|
||||
ok.add(n.group(1) if n else lid)
|
||||
return raw, ok, 'DATEX location joined to image; distinct camera numbers'
|
||||
|
||||
|
||||
@source('Via Lietuva eismoinfo', 'Lithuania')
|
||||
def lithuania():
|
||||
rows = get('https://eismoinfo.lt/eismoinfo-backend/camera-info-table', headers={'Accept': 'application/json'}).json()
|
||||
layers = get('https://eismoinfo.lt/eismoinfo-backend/layer-static-features/VKR', headers={'Accept': 'application/json'}).json()
|
||||
feats = {}
|
||||
for L in layers:
|
||||
if L.get('layer') == 'VKR':
|
||||
feats = {str(f.get('id')): f for f in L.get('features') or [] if f}
|
||||
raw, ok = set(), set()
|
||||
for r in rows:
|
||||
rid = str(r.get('id') or '')
|
||||
if not re.fullmatch(r'[1-9]\d{0,8}', rid):
|
||||
continue
|
||||
raw.add(rid)
|
||||
f = feats.get(rid)
|
||||
try:
|
||||
lat, lon = f['points'][0]['point']
|
||||
except Exception:
|
||||
continue
|
||||
if not inbox(fnum(lat), fnum(lon), 53.8, 56.5, 20.8, 26.9):
|
||||
continue
|
||||
u = urlparse(str(r.get('image') or ''))
|
||||
if f'{u.scheme}://{u.netloc}' != 'https://eismoinfo.lt' or u.path != '/eismoinfo-backend/image-provider/camera/last' or parse_qs(u.query).get('id', [''])[0] != rid:
|
||||
continue
|
||||
ok.add(rid)
|
||||
return raw, ok, 'camera table joined to VKR layer; official still endpoint'
|
||||
|
||||
|
||||
@source('Live Traffic NSW', 'Australia')
|
||||
def nsw():
|
||||
d = get('https://data.livetraffic.com/cameras/traffic-cam.json').json()
|
||||
raw, ok = set(), set()
|
||||
for f in d['features']:
|
||||
p = f.get('properties') or {}
|
||||
fid = str(f.get('id') or p.get('id') or p.get('title') or '').strip()
|
||||
if not fid:
|
||||
continue
|
||||
raw.add(fid)
|
||||
href = (p.get('href') or '').replace('http://', 'https://')
|
||||
c = (f.get('geometry') or {}).get('coordinates') or [None, None]
|
||||
if href.startswith('https://webcams.transport.nsw.gov.au/') and inbox(fnum(c[1]), fnum(c[0]), -38.0, -28.0, 140.0, 154.5):
|
||||
ok.add(fid)
|
||||
return raw, ok, 'one feature per camera; official webcams.transport.nsw.gov.au still'
|
||||
|
||||
|
||||
@source('City of Calgary', 'Canada')
|
||||
def calgary():
|
||||
d = get('https://data.calgary.ca/resource/k7p9-kppz.json?$limit=5000').json()
|
||||
raw, ok = set(), set()
|
||||
for c in d:
|
||||
url = ((c.get('camera_url') or {}).get('url') or '').strip()
|
||||
if not url:
|
||||
continue
|
||||
u = re.sub(r'^http://', 'https://', url)
|
||||
m = re.search(r'loc(\d+)\.jpg$', urlparse(u).path, re.I)
|
||||
key = m.group(1) if m else urlparse(u).path
|
||||
raw.add(key)
|
||||
coords = (c.get('point') or {}).get('coordinates') or [None, None]
|
||||
if u.startswith('https://trafficcam.calgary.ca/') and inbox(fnum(coords[1]), fnum(coords[0]), 50.7, 51.4, -114.5, -113.7):
|
||||
ok.add(key)
|
||||
return raw, ok, f'{len(d)} rows ($limit=5000); distinct trafficcam.calgary.ca image ids'
|
||||
|
||||
|
||||
@source('City of Toronto RESCU', 'Canada')
|
||||
def toronto():
|
||||
t = get('https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json').text
|
||||
d = json.loads(t[t.index('(') + 1: t.rindex(')')])
|
||||
raw, ok = set(), set()
|
||||
for c in d['Data']:
|
||||
n = str(c.get('Number') or '').strip()
|
||||
if not n:
|
||||
continue
|
||||
raw.add(n)
|
||||
if re.fullmatch(r'\d{1,6}', n) and inbox(fnum(c.get('Latitude')), fnum(c.get('Longitude')), 43.4, 44.0, -79.8, -79.0):
|
||||
ok.add(n)
|
||||
return raw, ok, 'one row per numbered RESCU camera'
|
||||
|
||||
|
||||
@source('Servei Català de Trànsit', 'Spain')
|
||||
def catalonia():
|
||||
x = get('https://www.gencat.cat/transit/opendata/cameres.xml').text
|
||||
hosts = {'mct.gencat.cat': '/mct2bo/', 'www.bcn.cat': '/transit/imatges/', 'emap.terrassa.cat': '/it_terrassa/'}
|
||||
raw, ok = set(), set()
|
||||
for m in re.finditer(r'<gml:featureMember>([\s\S]*?)</gml:featureMember>', x):
|
||||
b = m.group(1)
|
||||
co = re.search(r'<gml:coordinates[^>]*>([^<]+)</gml:coordinates>', b)
|
||||
lk = re.search(r'<cite:link>([^<]*)</cite:link>', b)
|
||||
if not co or not lk:
|
||||
continue
|
||||
link = lk.group(1).replace('&', '&').strip()
|
||||
u = urlparse(link)
|
||||
cam = parse_qs(u.query).get('sctidcam', [''])[0]
|
||||
key = f'sct:{cam}' if u.hostname == 'mct.gencat.cat' and cam else f'{u.hostname}{u.path}'
|
||||
raw.add(key)
|
||||
lon, lat = (fnum(v) for v in co.group(1).split(',')[:2])
|
||||
if not inbox(lat, lon, 40.4, 42.95, 0.1, 3.4):
|
||||
continue
|
||||
d = hosts.get(u.hostname or '')
|
||||
if not d or not u.path.startswith(d) or u.username or u.password or u.port:
|
||||
continue
|
||||
f = u.path[len(d):]
|
||||
if not f or '/' in f:
|
||||
continue
|
||||
if u.hostname == 'mct.gencat.cat' and (f != 'RenderService' or not re.fullmatch(r'[a-z0-9]+\.(gif|jpe?g)', cam, re.I)):
|
||||
continue
|
||||
ok.add(key)
|
||||
return raw, ok, 'SCT + Barcelona IMI + Terrassa; keyed by frame file (feed fid is unstable)'
|
||||
|
||||
|
||||
@source('City of Austin', 'United States')
|
||||
def austin():
|
||||
d = get('https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD').json()
|
||||
cols = [c['fieldName'] for c in d['meta']['view']['columns']]
|
||||
raw, ok = set(), set()
|
||||
for row in d['data']:
|
||||
r = dict(zip(cols, row))
|
||||
cid = str(r.get('camera_id') or '').strip()
|
||||
if not cid:
|
||||
continue
|
||||
raw.add(cid)
|
||||
st = (r.get('camera_status') or '').strip().upper()
|
||||
if st and st != 'TURNED_ON':
|
||||
continue
|
||||
m = re.match(r'POINT \(([-\d.]+) ([-\d.]+)\)', r.get('location') or '')
|
||||
if not m or not inbox(fnum(m.group(2)), fnum(m.group(1)), 29.9, 30.8, -98.2, -97.3):
|
||||
continue
|
||||
ok.add(cid)
|
||||
return raw, ok, 'camera_status TURNED_ON'
|
||||
|
||||
|
||||
TXDOT = 'ABL AMA ATL AUS BMT BWD BRY CHS CRP DAL ELP FTW HOU LRD LBB LFK ODA PAR PHR SJT SAT TYL WAC WFS YKM'.split()
|
||||
|
||||
|
||||
@source('TxDOT ITS', 'United States')
|
||||
def txdot():
|
||||
def one(code):
|
||||
for attempt in range(2):
|
||||
try:
|
||||
return code, get(f'https://its.txdot.gov/its/DistrictIts/GetCctvStatusListByDistrict?districtCode={code}',
|
||||
headers={'Accept': 'application/json'}, timeout=60).json()
|
||||
except Exception as e:
|
||||
err = e
|
||||
return code, err
|
||||
with ThreadPoolExecutor(8) as ex:
|
||||
res = list(ex.map(one, TXDOT))
|
||||
raw, ok, notes, per = set(), set(), [], {}
|
||||
for code, d in res:
|
||||
if isinstance(d, Exception):
|
||||
notes.append(f'{code} failed: {str(d)[:50]}')
|
||||
continue
|
||||
n = 0
|
||||
for rows in (d.get('roadwayCctvStatuses') or {}).values():
|
||||
for r in rows or []:
|
||||
icd = str(r.get('icd_Id') or '').strip()
|
||||
if not icd:
|
||||
continue
|
||||
raw.add(icd)
|
||||
if r.get('statusDescription') != 'Device Online' or r.get('hasSnapshot') is False:
|
||||
continue
|
||||
lat, lon = r.get('latitude'), r.get('longitude')
|
||||
if not (isinstance(lat, (int, float)) and isinstance(lon, (int, float)) and inbox(lat, lon, 25.5, 36.7, -106.8, -93.3)):
|
||||
continue
|
||||
if icd not in ok:
|
||||
n += 1
|
||||
ok.add(icd)
|
||||
per[code] = n
|
||||
return raw, ok, f'all 25 districts; Device Online + snapshot; per-district {per}. ' + '; '.join(notes)
|
||||
|
||||
|
||||
@source('Ontario 511', 'Canada')
|
||||
def ontario():
|
||||
d = get('https://511on.ca/api/v2/get/cameras?format=json&lang=en', headers={'Accept': 'application/json'}).json()
|
||||
raw, ok, views = set(), set(), 0
|
||||
for r in d:
|
||||
rid = str(r.get('Id') or r.get('id') or '').strip()
|
||||
if not rid:
|
||||
continue
|
||||
raw.add(rid)
|
||||
if not inbox(fnum(r.get('Latitude')), fnum(r.get('Longitude')), 41.0, 57.5, -95.6, -74.0):
|
||||
continue
|
||||
good = []
|
||||
for v in r.get('Views') or r.get('views') or []:
|
||||
if str(v.get('Status') or '').strip().lower() != 'enabled':
|
||||
continue
|
||||
u = urlparse(str(v.get('Url') or '').strip())
|
||||
if u.scheme == 'https' and (u.hostname == '511on.ca' or (u.hostname or '').endswith('.traveliq.co')) and re.fullmatch(r'/map/Cctv/[A-Za-z0-9_.-]+', u.path):
|
||||
good.append(v)
|
||||
if good:
|
||||
ok.add(rid)
|
||||
views += len(good)
|
||||
return raw, ok, f'counted per camera row, not per view; {views} enabled views'
|
||||
|
||||
|
||||
@source('DelDOT', 'United States')
|
||||
def deldot():
|
||||
d = get('https://tmc.deldot.gov/json/videocamera.json', headers={'Accept': 'application/json'}).json()
|
||||
raw, ok = set(), set()
|
||||
for r in d.get('videoCameras') or []:
|
||||
cid = str(r.get('id') or '').strip()
|
||||
if not cid:
|
||||
continue
|
||||
raw.add(cid)
|
||||
if str(r.get('status')).lower() != 'active':
|
||||
continue
|
||||
if not inbox(fnum(r.get('lat')), fnum(r.get('lon')), 38.4, 39.9, -75.85, -75.0):
|
||||
continue
|
||||
u = urlparse(str((r.get('urls') or {}).get('m3u8s') or ''))
|
||||
if u.scheme != 'https' or u.hostname != 'video.deldot.gov' or u.port not in (None, 443) or u.username or not re.fullmatch(r'/live/[A-Za-z0-9_.-]+/playlist\.m3u8', u.path):
|
||||
continue
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,80}', cid):
|
||||
continue
|
||||
ok.add(cid)
|
||||
return raw, ok, 'status active + official HLS stream'
|
||||
|
||||
|
||||
def run(label):
|
||||
fn, country = SRC[label]
|
||||
t0 = time.time()
|
||||
try:
|
||||
raw, ok, note = fn()
|
||||
if not ok:
|
||||
raise RuntimeError('catalog parsed to zero cameras')
|
||||
return label, dict(country=country, raw=len(raw), valid=len(ok), note=note, secs=round(time.time() - t0, 1))
|
||||
except Exception as e:
|
||||
return label, dict(country=country, error=f'{type(e).__name__}: {str(e)[:200]}', secs=round(time.time() - t0, 1))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
out = sys.argv[1]
|
||||
detail = sys.argv[2] if len(sys.argv) > 2 else None
|
||||
with ThreadPoolExecutor(8) as ex:
|
||||
results = dict(ex.map(run, list(SRC)))
|
||||
counted = {k: v for k, v in results.items() if 'valid' in v}
|
||||
for k, v in results.items():
|
||||
print(f"{k:28s} {v.get('country', ''):15s} raw={v.get('raw', '-'):>6} valid={v.get('valid', '-'):>6} {v.get('error') or v.get('note')}")
|
||||
total = sum(v['valid'] for v in counted.values())
|
||||
res = {
|
||||
'cctv_catalog': total,
|
||||
'cctv_sources': {k: v['valid'] for k, v in counted.items()},
|
||||
'cctv_countries': len({v['country'] for v in counted.values()}),
|
||||
'fetched_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
}
|
||||
print('TOTAL valid', total, 'raw', sum(v['raw'] for v in counted.values()), 'countries', res['cctv_countries'])
|
||||
json.dump(res, open(out, 'w'), ensure_ascii=False, indent=2)
|
||||
if detail:
|
||||
json.dump(results, open(detail, 'w'), ensure_ascii=False, indent=2)
|
||||
Reference in new issue
Block a user