Merge upstream main into pr/street-level-show-action

Locale conflicts: main's catalogs plus the PR's 10 street-level strings.

scripts/build-ucs-satellites.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+'''Maintainer-only UCS Satellite Database pack build (openpyxl).
+
+Reads the UCS Excel release and writes the compact pack the satellites layer
+loads: one row per satellite, keyed by NORAD number, with a string-table index
+for its purpose, the one column the layer's classes read. Starlink rows are
+left out: the layer files every STARLINK object under its own class by name.
+
+Usage: python scripts/build-ucs-satellites.py "<path>/UCS-Satellite-Database 5-1-2023.xlsx"
+'''
+import argparse
+import json
+import logging
+from pathlib import Path
+import re
+
+import openpyxl
+
+LOG = logging.getLogger('build-ucs-satellites')
+
+ROOT = Path(__file__).resolve().parent.parent
+PACK = ROOT / 'src/data/local_data/ucs_satellites'
+VERSION = 'UCS-Satellite-Database 5-1-2023'
+DATA_CURRENT_TO = '2023-05-01'
+URL = 'https://www.ucsusa.org/satellite_database'
+
+# Output column -> spreadsheet header.
+COLUMNS = {
+    'purpose': 'Purpose',
+}
+STARLINK = re.compile(r'\bstarlink\b', re.I)
+
+
+def clean(value: object) -> str:
+    '''One cell as tidy text; UCS's placeholders for nothing become ''.'''
+    text = re.sub(r'\s+', ' ', str(value or '')).strip()
+    return '' if text.lower() in ('', 'none', 'n/a') else text
+
+
+def build(xlsx: Path) -> dict:
+    '''Read the release's first sheet into the pack the layer loads.'''
+    sheet = openpyxl.load_workbook(xlsx, read_only=True, data_only=True).worksheets[0]
+    rows = sheet.iter_rows(values_only=True)
+    header = {str(h).strip(): i for i, h in enumerate(next(rows)) if h}
+    names = [header['Current Official Name of Satellite'], header['Name of Satellite, Alternate Names']]
+    norad_col = header['NORAD Number']
+    tables = {key: [''] for key in COLUMNS}
+    index = {key: {'': 0} for key in COLUMNS}
+    records = {}
+    source_rows = skipped_starlink = invalid_norad = 0
+    for row in rows:
+        if not any(row):
+            continue
+        source_rows += 1
+        if any(STARLINK.search(clean(row[i])) for i in names):
+            skipped_starlink += 1
+            continue
+        try:
+            norad = int(float(row[norad_col]))
+        except (TypeError, ValueError):
+            invalid_norad += 1
+            continue
+        if norad <= 0:
+            invalid_norad += 1
+            continue
+        record = [norad]
+        for key, column in COLUMNS.items():
+            text = clean(row[header[column]])
+            if text not in index[key]:
+                index[key][text] = len(tables[key])
+                tables[key].append(text)
+            record.append(index[key][text])
+        # A duplicate NORAD number keeps the first row, as UCS lists it.
+        records.setdefault(norad, record)
+    ordered = [records[n] for n in sorted(records)]
+    return {
+        'meta': {
+            'source': 'Union of Concerned Scientists Satellite Database',
+            'version': VERSION,
+            'url': URL,
+            'dataCurrentTo': DATA_CURRENT_TO,
+            'license': 'Free, unrestricted use; acknowledgement requested (UCS database manual, section 3)',
+            'curation': {
+                'sourceRows': source_rows,
+                'starlinkRowsDropped': skipped_starlink,
+                'invalidNoradDropped': invalid_norad,
+                'duplicateNoradDropped': source_rows - skipped_starlink - invalid_norad - len(ordered),
+                'columns': COLUMNS,
+            },
+            'recordCount': len(ordered),
+        },
+        'fields': ['noradId', *COLUMNS],
+        'strings': tables,
+        'records': ordered,
+    }
+
+
+def main() -> None:
+    '''Build the pack from the release given on the command line.'''
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+    parser = argparse.ArgumentParser(description='Build the UCS satellite pack from the UCS Excel release.')
+    parser.add_argument('xlsx', type=Path, help='UCS-Satellite-Database <version>.xlsx')
+    pack = build(parser.parse_args().xlsx)
+    PACK.mkdir(parents=True, exist_ok=True)
+    out = PACK / 'ucs_satellites.json'
+    # LF on every platform: the repository stores text with LF endings.
+    out.write_text(
+        json.dumps(pack, ensure_ascii=False, separators=(',', ':')) + '\n',
+        encoding='utf-8',
+        newline='\n',
+    )
+    LOG.info('%s: %d records, %d bytes', out.relative_to(ROOT), pack['meta']['recordCount'], out.stat().st_size)
+
+
+if __name__ == '__main__':
+    main()

scripts/package-boundaries.json

@@ -999,6 +999,7 @@
       "src/data/idSwitches.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateCoordinator.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/militaryAwarenessEngine.js",
@@ -1381,6 +1382,7 @@
       "src/data/idSwitches.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/modelVisualAnchor.js",
       "src/data/aircraftModelLighting.js",
@@ -1446,6 +1448,7 @@
       "src/data/idSwitches.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/modelVisualAnchor.js",
       "src/data/aircraftModelLighting.js",
@@ -1502,6 +1505,7 @@
       "src/data/idSwitches.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateCoordinator.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/modelVisualAnchor.js",
@@ -1643,6 +1647,7 @@
     "exports": ["./layers/satellites"],
     "modules": [
       "src/data/satelliteClass.js",
+      "src/data/satelliteClassMask.js",
       "src/data/satellitePass.js",
       "src/sources/tle.js",
       "src/layers/satellites/catalog.js",
@@ -1675,6 +1680,10 @@
       "src/i18n/locales/zh-CN.json",
       "src/i18n/locales/zh-Hant.json",
       "src/ui/unitLabels.js",
+      "src/layers/satellites/classes.js",
+      "src/data/ucsSatellites.js",
+      "src/data/bundledJson.js",
+      "src/data/retryableLoad.js",
       "src/time/worldClock.js"
     ],
     "external": ["cesium", "satellite.js"]
@@ -2090,6 +2099,7 @@
       "src/data/idSwitches.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/radioCountry.js",
       "src/layers/radio/catalogModel.js",
@@ -2482,6 +2492,7 @@
       "src/data/labelArbiter.js",
       "src/data/idSwitches.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateCoordinator.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/lifecycle.js",
@@ -3368,7 +3379,9 @@
       "src/ui/wildlifeSightingPlacement.js",
       "src/overlays/worldOverlayChrome.js",
       "src/layers/wildlifeSightings/photoText.js",
-      "src/layers/wildlifeSightings/performance.js"
+      "src/layers/wildlifeSightings/performance.js",
+      "src/layers/satellites/classes.js",
+      "src/data/ucsSatellites.js"
     ],
     "external": [
       "hls.js",
@@ -3480,6 +3493,7 @@
       "src/data/labelArbiter.js",
       "src/data/idSwitches.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/localGeojsonCore.js",
       "src/data/localGeojsonLod.js",
@@ -4021,7 +4035,9 @@
       "src/layers/wildlifeSightings/sprites.js",
       "src/overlays/worldOverlayChrome.js",
       "src/layers/wildlifeSightings/photoText.js",
-      "src/layers/wildlifeSightings/performance.js"
+      "src/layers/wildlifeSightings/performance.js",
+      "src/layers/satellites/classes.js",
+      "src/data/ucsSatellites.js"
     ],
     "external": [
       "hls.js",
@@ -4155,6 +4171,7 @@
       "src/data/idSwitches.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateCoordinator.js",
       "src/data/layerStateTokenReservations.json",
       "src/loadingFeedback.js",
@@ -4524,7 +4541,13 @@
   },
   "satellites-source": {
     "exports": ["./layers/satellites/source"],
-    "modules": ["src/layers/satellites/source.js"],
+    "modules": [
+      "src/layers/satellites/source.js",
+      "src/layers/satellites/classes.js",
+      "src/data/ucsSatellites.js",
+      "src/data/bundledJson.js",
+      "src/data/retryableLoad.js"
+    ],
     "external": []
   },
   "traffic-source-factory": {
@@ -4895,6 +4918,7 @@
       "src/view/index.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateTokenReservations.json",
       "src/data/idSwitches.js",
       "src/data/detectionPolicy.js",
@@ -5021,6 +5045,7 @@
       "src/data/idSwitches.js",
       "src/data/vesselTypes.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateTokenReservations.json",
       "src/sources/gbfsStations.js",
       "src/sources/tle.js",
@@ -5144,7 +5169,9 @@
       "src/layers/alpr/records.js",
       "src/layers/alpr/tileRecords.js",
       "src/sources/infrastructureData.js",
-      "src/sources/eventPacks.js"
+      "src/sources/eventPacks.js",
+      "src/layers/satellites/classes.js",
+      "src/data/ucsSatellites.js"
     ],
     "external": ["@mapbox/vector-tile", "pbf"]
   },
@@ -5155,6 +5182,7 @@
       "src/data/idSwitches.js",
       "src/data/firmsIntensity.js",
       "src/data/layerState.js",
+      "src/data/satelliteClassMask.js",
       "src/data/layerStateTokenReservations.json",
       "src/visualStylePolicy.js"
     ],

scripts/track-regression.mjs

@@ -409,6 +409,15 @@ async function main() {
             status: 'ready',
           }));
         }
+        // CelesTrak SATCAT, which the satellite classes read beside the TLEs.
+        // An empty CSV (header only) keeps the run hermetic: the classes fall
+        // back to names and groups, as they do when SATCAT is unavailable.
+        if (isAppRequest && url.pathname.startsWith('/api/celestrak-satcat/')) {
+          return Promise.resolve(new Response(
+            'OBJECT_NAME,OBJECT_ID,NORAD_CAT_ID,OBJECT_TYPE,OPS_STATUS_CODE,LAUNCH_DATE\n',
+            { status: 200, headers: { 'Content-Type': 'text/csv' } },
+          ));
+        }
         // CelesTrak TLE groups. Served from a fixed two-satellite catalog so
         // the satellite assertions are deterministic and the run never depends
         // on an upstream that rate-limits (it was answering 403 the night this