City-Lifeline-Cockpit/tools/city-massing/build-streaming-tiles.mjs

// Convert existing merged cells into small spatial GLBs. Source assets remain untouched.
// node tools/city-massing/build-streaming-tiles.mjs <batch-dir> [--max-triangles 20000]
import assert from 'node:assert/strict'
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { createHash } from 'node:crypto'
import draco3d from 'draco3d'
import { rasterLimits, tileRasterBudget } from './raster-budget.mjs'

const args = process.argv.slice(2)
const option = (key, fallback) => { const index = args.indexOf(key); return index < 0 ? fallback : Number(args.splice(index, 2)[1]) }
const maxTriangles = option('--max-triangles', 20000)
const coarseDistance = option('--coarse-distance', 1500)
const detailDistance = option('--detail-distance', 250)
// Distance-first refinement retains local request spheres; effectiveness depends on Cesium traversal mode.
const distanceFirst = args.includes('--distance-first')
if (distanceFirst) args.splice(args.indexOf('--distance-first'), 1)
const generationIndex = args.indexOf('--generation')
const generation = generationIndex < 0 ? 'streaming-v2' : args.splice(generationIndex, 2)[1]
assert.match(generation, /^streaming-v\d+$/)
assert.ok(Number.isInteger(maxTriangles) && maxTriangles > 0 && maxTriangles <= 20000)
assert.ok(coarseDistance > detailDistance && detailDistance > 0)
const batch = resolve(args[0])
const out = join(batch, generation)
// Never regenerate a directory currently served by the app. A new generation gets a new URL.
assert.ok(!existsSync(join(out, 'manifest.json')), `output already exists: ${out}`)
mkdirSync(out, { recursive: true })
const manifest = JSON.parse(readFileSync(join(batch, 'manifest.json'), 'utf8'))
const originalTree = JSON.parse(readFileSync(join(batch, 'tileset.json'), 'utf8'))
const draco = await draco3d.createDecoderModule({})
const encoderModule = await draco3d.createEncoderModule({})
const report = { generation, sourceBatch: batch, count: manifest.count, encoding: 'draco-small-tiles',
    provenance: manifest.provenance || { gisFootprints: false, roadsProcedural: null, facadeDetails: null },
    budget: { maxTriangles, maxVertices: maxTriangles * 3, ...rasterLimits, coarseDistance, detailDistance },
    origin: manifest.origin, cells: [], files: [], totals: { lod1: { triangles: 0, bytes: 0 }, lod0: { triangles: 0, bytes: 0 } } }

function parseGlb(path) {
    const bytes = readFileSync(path)
    assert.equal(bytes.toString('ascii', 0, 4), 'glTF')
    assert.equal(bytes.readUInt32LE(8), bytes.length)
    const length = bytes.readUInt32LE(12)
    const json = JSON.parse(bytes.toString('utf8', 20, 20 + length))
    const offset = length + 20
    assert.equal(bytes.toString('ascii', offset + 4, offset + 8), 'BIN\0')
    return { bytes, json, bin: bytes.subarray(offset + 8) }
}
function decode(path) {
    const { bytes, json, bin } = parseGlb(path)
    assert.ok(!json.skins?.length, 'skinned meshes are not supported')
    // Preserve embedded raster bytes; export remaps each partition's dependency indices.
    const images = (json.images || []).map((image) => {
        assert.ok(image.bufferView !== undefined && !image.uri, 'images must be embedded in the source GLB')
        assert.ok(['image/png', 'image/jpeg'].includes(image.mimeType), 'unsupported raster image')
        const view = json.bufferViews[image.bufferView]
        return { mimeType: image.mimeType, name: image.name,
            bytes: bin.subarray(view.byteOffset || 0, (view.byteOffset || 0) + view.byteLength) }
    })
    const nodes = json.nodes.filter((node) => node.mesh !== undefined)
    assert.equal(nodes.length, 1)
    assert.ok(!nodes[0].matrix && !nodes[0].translation && !nodes[0].scale && !nodes[0].rotation,
        'source node transforms must already be baked')
    const primitives = []
    for (const primitive of json.meshes[nodes[0].mesh].primitives) {
        assert.equal(primitive.mode ?? 4, 4)
        const extension = primitive.extensions.KHR_draco_mesh_compression
        const view = json.bufferViews[extension.bufferView]
        const encoded = bin.subarray(view.byteOffset || 0, (view.byteOffset || 0) + view.byteLength)
        const decoder = new draco.Decoder(), buffer = new draco.DecoderBuffer(), mesh = new draco.Mesh()
        let status
        try {
            buffer.Init(new Int8Array(encoded.buffer, encoded.byteOffset, encoded.byteLength), encoded.byteLength)
            status = decoder.DecodeBufferToMesh(buffer, mesh)
            assert.ok(status.ok(), status.error_msg())
            const attributes = {}
            for (const [semantic, uniqueId] of Object.entries(extension.attributes)) {
                assert.ok(['POSITION', 'NORMAL', 'TEXCOORD_0'].includes(semantic), `unsupported source attribute ${semantic}`)
                const attribute = decoder.GetAttributeByUniqueId(mesh, uniqueId)
                const components = semantic === 'TEXCOORD_0' ? 2 : 3
                assert.equal(attribute.num_components(), components)
                const size = mesh.num_points() * components * 4
                const ptr = draco._malloc(size)
                try {
                    decoder.GetAttributeDataArrayForAllPoints(mesh, attribute, draco.DT_FLOAT32, size, ptr)
                    attributes[semantic] = new Float32Array(draco.HEAPF32.buffer, ptr, mesh.num_points() * components).slice()
                } finally { draco._free(ptr) }
            }
            const size = mesh.num_faces() * 3 * 4
            const ptr = draco._malloc(size)
            let indices
            try {
                decoder.GetTrianglesUInt32Array(mesh, size, ptr)
                indices = new Uint32Array(draco.HEAPU32.buffer, ptr, mesh.num_faces() * 3).slice()
            } finally { draco._free(ptr) }
            primitives.push({ material: primitive.material, attributes, indices })
        } finally {
            if (status) draco.destroy(status)
            draco.destroy(mesh); draco.destroy(buffer); draco.destroy(decoder)
        }
    }
    const triangleCount = primitives.reduce((sum, primitive) => sum + primitive.indices.length / 3, 0)
    const primitiveIndex = new Uint16Array(triangleCount), faceIndex = new Uint32Array(triangleCount)
    const centers = new Float32Array(triangleCount * 3)
    let current = 0
    primitives.forEach((primitive, pi) => {
        const positions = primitive.attributes.POSITION
        for (let face = 0; face < primitive.indices.length / 3; face++, current++) {
            primitiveIndex[current] = pi; faceIndex[current] = face
            for (let axis = 0; axis < 3; axis++) {
                let value = 0
                for (let corner = 0; corner < 3; corner++) value += positions[primitive.indices[face * 3 + corner] * 3 + axis] / 3
                centers[current * 3 + axis] = value
            }
        }
    })
    return { primitives, materials: json.materials, images, textures: json.textures, samplers: json.samplers,
        extensionsUsed: (json.extensionsUsed || []).filter((x) => x !== 'KHR_draco_mesh_compression'),
        primitiveIndex, faceIndex, centers, ids: Uint32Array.from({ length: triangleCount }, (_, index) => index), triangleCount,
        sourceHash: createHash('sha256').update(bytes).digest('hex') }
}
function subsetBounds(levels, selections) {
    const min = [Infinity, Infinity, Infinity], max = [-Infinity, -Infinity, -Infinity]
    for (let level = 0; level < 2; level++) {
        const data = levels[level]
        for (const id of selections[level]) {
            const primitive = data.primitives[data.primitiveIndex[id]], positions = primitive.attributes.POSITION
            const face = data.faceIndex[id]
            for (let corner = 0; corner < 3; corner++) {
                const index = primitive.indices[face * 3 + corner]
                // glTF Y-up -> source ENU. The root retains the original ECEF transform.
                const enu = [positions[index * 3], -positions[index * 3 + 2], positions[index * 3 + 1]]
                for (let axis = 0; axis < 3; axis++) { min[axis] = Math.min(min[axis], enu[axis]); max[axis] = Math.max(max[axis], enu[axis]) }
            }
        }
    }
    return { min, max }
}
function partition(levels, selections, leaves) {
    if (Math.max(...selections.map((ids) => ids.length)) <= maxTriangles) { leaves.push(selections); return }
    const min = [Infinity, Infinity, Infinity], max = [-Infinity, -Infinity, -Infinity]
    levels.forEach((data, level) => {
        for (const id of selections[level]) for (let axis = 0; axis < 3; axis++) {
            const value = data.centers[id * 3 + axis]
            min[axis] = Math.min(min[axis], value); max[axis] = Math.max(max[axis], value)
        }
    })
    const spans = max.map((value, axis) => value - min[axis])
    const axis = spans.indexOf(Math.max(...spans)), midpoint = (min[axis] + max[axis]) / 2
    const left = [], right = []
    levels.forEach((data, level) => {
        const ids = selections[level], a = [], b = []
        for (const id of ids) (data.centers[id * 3 + axis] < midpoint ? a : b).push(id)
        left.push(Uint32Array.from(a)); right.push(Uint32Array.from(b))
    })
    // Coincident geometry still needs a bounded upload; split deterministically by source index.
    if (left.every((ids) => !ids.length) || right.every((ids) => !ids.length)) {
        selections.forEach((ids, level) => { left[level] = ids.slice(0, Math.ceil(ids.length / 2)); right[level] = ids.slice(Math.ceil(ids.length / 2)) })
    }
    partition(levels, left, leaves); partition(levels, right, leaves)
}
function box(bounds) {
    const center = bounds.min.map((value, axis) => (value + bounds.max[axis]) / 2)
    const half = bounds.min.map((value, axis) => Math.max(0.001, (bounds.max[axis] - value) / 2))
    return [center[0], center[1], center[2], half[0], 0, 0, 0, half[1], 0, 0, 0, half[2]]
}
function requestVolume(bounds, distance) {
    const center = bounds.min.map((value, axis) => (value + bounds.max[axis]) / 2)
    const radius = Math.hypot(...bounds.min.map((value, axis) => (bounds.max[axis] - value) / 2))
    return { sphere: [...center, radius + distance] }
}
function exportGlb(name, data, ids, bounds, level) {
    const center = bounds.min.map((value, axis) => (value + bounds.max[axis]) / 2)
    const gltfCenter = [center[0], center[2], -center[1]]
    const chunks = [], views = [], accessors = [], primitives = []
    let byteOffset = 0, vertexCount = 0
    function appendPayload(bytes) {
        const viewIndex = views.length
        views.push({ buffer: 0, byteOffset, byteLength: bytes.length })
        chunks.push(bytes); byteOffset += bytes.length
        if (byteOffset % 4) { const padding = Buffer.alloc(4 - byteOffset % 4); chunks.push(padding); byteOffset += padding.length }
        return viewIndex
    }
    function accessor(type, count, components, range = {}) {
        const index = accessors.length
        accessors.push({ componentType: type, count, type: components === 1 ? 'SCALAR' : components === 2 ? 'VEC2' : 'VEC3', ...range })
        return index
    }
    const byPrimitive = new Map()
    for (const id of ids) {
        const pi = data.primitiveIndex[id]
        if (!byPrimitive.has(pi)) byPrimitive.set(pi, [])
        byPrimitive.get(pi).push(data.faceIndex[id])
    }
    for (const [pi, faces] of byPrimitive) {
        const source = data.primitives[pi], used = new Map(), oldIndices = [], indices = []
        for (const face of faces) for (let corner = 0; corner < 3; corner++) {
            const index = source.indices[face * 3 + corner]
            if (!used.has(index)) { used.set(index, oldIndices.length); oldIndices.push(index) }
            indices.push(used.get(index))
        }
        const builder = new encoderModule.MeshBuilder(), mesh = new encoderModule.Mesh()
        const encoder = new encoderModule.Encoder(), encoded = new encoderModule.DracoInt8Array()
        try {
            builder.AddFacesToMesh(mesh, indices.length / 3, Uint32Array.from(indices))
            const attributeIds = {}, ranges = {}
            for (const [semantic, values] of Object.entries(source.attributes)) {
                const components = semantic === 'TEXCOORD_0' ? 2 : 3
                const result = new Float32Array(oldIndices.length * components), min = [Infinity, Infinity, Infinity], max = [-Infinity, -Infinity, -Infinity]
                oldIndices.forEach((index, output) => {
                    for (let axis = 0; axis < components; axis++) {
                        const value = values[index * components + axis] - (semantic === 'POSITION' ? gltfCenter[axis] : 0)
                        result[output * components + axis] = value
                        min[axis] = Math.min(min[axis], value); max[axis] = Math.max(max[axis], value)
                    }
                })
                attributeIds[semantic] = builder.AddFloatAttributeToMesh(mesh,
                    encoderModule[semantic === 'TEXCOORD_0' ? 'TEX_COORD' : semantic], oldIndices.length, components, result)
                ranges[semantic] = semantic === 'POSITION' ? { min, max } : {}
            }
            // Prefer cheap decode over maximum compression. Tiles are tiny, and have local origins.
            encoder.SetSpeedOptions(7, 10)
            encoder.SetEncodingMethod(encoderModule.MESH_SEQUENTIAL_ENCODING)
            // Near architectural roofs need extra precision to retain surveyed near-collinear slivers.
            encoder.SetAttributeQuantization(encoderModule.POSITION, manifest.provenance?.nearArchitecture ? 20 : 16)
            encoder.SetAttributeQuantization(encoderModule.NORMAL, 12)
            encoder.SetAttributeQuantization(encoderModule.TEX_COORD, 14)
            encoder.SetTrackEncodedProperties(true)
            const length = encoder.EncodeMeshToDracoBuffer(mesh, encoded)
            assert.ok(length > 0, `${name}: Draco encoding failed`)
            const points = encoder.GetNumberOfEncodedPoints(), faces = encoder.GetNumberOfEncodedFaces()
            assert.equal(faces, indices.length / 3, `${name}: compression changed triangle count`)
            vertexCount += points
            const bytes = Buffer.alloc(length)
            for (let index = 0; index < length; index++) bytes[index] = encoded.GetValue(index) & 255
            const attributes = Object.fromEntries(Object.keys(attributeIds).map((semantic) => [semantic, accessor(5126, points, semantic === 'TEXCOORD_0' ? 2 : 3, ranges[semantic])]))
            primitives.push({ attributes, indices: accessor(points < 65536 ? 5123 : 5125, faces * 3, 1),
                material: source.material, mode: 4,
                extensions: { KHR_draco_mesh_compression: { bufferView: appendPayload(bytes), attributes: attributeIds } } })
        } finally {
            encoderModule.destroy(encoded); encoderModule.destroy(encoder); encoderModule.destroy(mesh); encoderModule.destroy(builder)
        }
    }
    assert.ok(vertexCount <= maxTriangles * 3, `${name}: vertex budget`)
    // Keep only this partition's dependency closure; no unused cell-wide roof rasters.
    const materialIds = new Map(), textureIds = new Map(), imageIds = new Map(), samplerIds = new Map()
    const materials = [], textures = [], images = [], samplers = []
    const rasterImages = []
    function copyImage(index) {
        if (!imageIds.has(index)) {
            const { bytes, ...image } = data.images[index]
            rasterImages.push({ bytes, mimeType: image.mimeType })
            imageIds.set(index, images.length)
            images.push({ ...image, bufferView: appendPayload(bytes) })
        }
        return imageIds.get(index)
    }
    function copyTexture(index) {
        if (!textureIds.has(index)) {
            const texture = structuredClone(data.textures[index])
            assert.ok(!texture.extensions, 'texture extensions require explicit dependency remapping')
            texture.source = copyImage(texture.source)
            if (texture.sampler !== undefined) {
                const original = texture.sampler
                if (!samplerIds.has(original)) {
                    samplerIds.set(original, samplers.length)
                    samplers.push(structuredClone(data.samplers[original]))
                }
                texture.sampler = samplerIds.get(original)
            }
            textureIds.set(index, textures.length)
            textures.push(texture)
        }
        return textureIds.get(index)
    }
    function remapTextureInfo(value) {
        for (const [key, child] of Object.entries(value)) {
            if (!child || typeof child !== 'object') continue
            if (key.endsWith('Texture') && Number.isInteger(child.index)) child.index = copyTexture(child.index)
            else remapTextureInfo(child)
        }
    }
    for (const primitive of primitives) {
        const original = primitive.material
        if (!materialIds.has(original)) {
            const material = structuredClone(data.materials[original])
            remapTextureInfo(material)
            materialIds.set(original, materials.length)
            materials.push(material)
        }
        primitive.material = materialIds.get(original)
    }
    const rasterBudget = tileRasterBudget(rasterImages, textures)
    const payload = Buffer.concat(chunks)
    const bin = Buffer.concat([payload, Buffer.alloc((-payload.length) & 3)])
    const json = { asset: { version: '2.0', generator: 'CAD bounded spatial streaming' }, scene: 0,
        scenes: [{ nodes: [0] }], nodes: [{ mesh: 0, translation: gltfCenter, name }], meshes: [{ primitives }],
        materials, images: images.length ? images : undefined, textures: textures.length ? textures : undefined,
        samplers: samplers.length ? samplers : undefined, accessors, bufferViews: views, buffers: [{ byteLength: bin.length }],
        extensionsUsed: [...data.extensionsUsed, 'KHR_draco_mesh_compression'], extensionsRequired: ['KHR_draco_mesh_compression'] }
    let jsonBytes = Buffer.from(JSON.stringify(json))
    jsonBytes = Buffer.concat([jsonBytes, Buffer.alloc((4 - jsonBytes.length % 4) % 4, 0x20)])
    const header = Buffer.alloc(20), binHeader = Buffer.alloc(8)
    header.write('glTF'); header.writeUInt32LE(2, 4); header.writeUInt32LE(28 + jsonBytes.length + bin.length, 8)
    header.writeUInt32LE(jsonBytes.length, 12); header.write('JSON', 16)
    binHeader.writeUInt32LE(bin.length, 0); binHeader.write('BIN\0', 4)
    const bytes = Buffer.concat([header, jsonBytes, binHeader, bin])
    writeFileSync(join(out, name), bytes)
    const record = { uri: name, level, triangles: ids.length, vertices: vertexCount, bytes: bytes.length,
        boundsENU: bounds, materials: primitives.length, ...rasterBudget }
    report.files.push(record)
    report.totals[level].triangles += ids.length; report.totals[level].bytes += bytes.length
    return record
}

const rootChildren = []
for (const cell of manifest.cells) {
    const levels = [decode(join(batch, cell.lod1.uri)), decode(join(batch, cell.lod0.uri))]
    assert.equal(levels[0].triangleCount, cell.lod1.triangles)
    assert.equal(levels[1].triangleCount, cell.lod0.triangles)
    const leaves = []
    partition(levels, levels.map((data) => data.ids), leaves)
    const children = []
    leaves.forEach((selections, index) => {
        const bounds = subsetBounds(levels, selections), volume = { box: box(bounds) }
        const records = selections.map((ids, level) => ids.length
            ? exportGlb(`${cell.id}-p${String(index).padStart(3, '0')}-lod${1 - level}.glb`, levels[level], ids, bounds, level ? 'lod0' : 'lod1') : null)
        const fine = records[1] ? { boundingVolume: volume, geometricError: 0,
            viewerRequestVolume: requestVolume(bounds, detailDistance), content: { uri: records[1].uri } } : null
        if (records[0]) children.push({ boundingVolume: volume, geometricError: fine ? (distanceFirst ? 1000000 : 16) : 0, refine: 'REPLACE',
            viewerRequestVolume: requestVolume(bounds, coarseDistance), content: { uri: records[0].uri }, ...(fine ? { children: [fine] } : {}) })
        else if (fine) children.push(fine)
    })
    // Empty structural parents do not load a huge GLB. Only their small visible children stream.
    rootChildren.push({ boundingVolume: { box: box(cell.boundsENU) }, geometricError: 1000000, refine: 'REPLACE', children })
    report.cells.push({ id: cell.id, buildingIds: cell.buildingIds, partitions: leaves.length,
        source: levels.map((data, index) => ({ level: index ? 'lod0' : 'lod1', triangles: data.triangleCount, sha256: data.sourceHash })) })
    console.log(`${cell.id}: ${cell.count} buildings, ${leaves.length} bounded partitions`)
}
const anchor = readFileSync(join(batch, 'root-anchor.glb'))
writeFileSync(join(out, 'root-anchor.glb'), anchor)
const tileset = { asset: { version: '1.1', generator: 'CAD bounded spatial streaming' }, geometricError: 1000000,
    root: { transform: originalTree.root.transform, boundingVolume: originalTree.root.boundingVolume, geometricError: 1000000,
        refine: 'REPLACE', content: { uri: 'root-anchor.glb' }, children: rootChildren } }
writeFileSync(join(out, 'tileset.json'), JSON.stringify(tileset, null, 2) + '\n')
writeFileSync(join(out, 'manifest.json'), JSON.stringify(report, null, 2) + '\n')
console.log(JSON.stringify({ output: out, buildings: report.count, files: report.files.length,
    maxVertices: Math.max(...report.files.map((file) => file.vertices)), totals: report.totals }))