From be3a1eda8cfd644d7dfb834bea1092a23f120ca3 Mon Sep 17 00:00:00 2001 From: Disy Date: Tue, 20 Oct 2026 19:14:58 +0800 Subject: [PATCH] fix(ci): publish skin site releases with chunked uploads Use bounded chunks with retries, processing deadlines and verified activation. Fail publication explicitly on upload errors. Add a manual recovery workflow for existing release packages. --- .github/workflows/build-on-tag.yml | 39 +--- .github/workflows/publish-skin-site.yml | 56 +++++ scripts/axolotl/publish-skin-site.mjs | 226 +++++++++++++++++++++ scripts/axolotl/publish-skin-site.test.mjs | 172 ++++++++++++++++ 4 files changed, 464 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/publish-skin-site.yml create mode 100644 scripts/axolotl/publish-skin-site.mjs create mode 100644 scripts/axolotl/publish-skin-site.test.mjs diff --git a/.github/workflows/build-on-tag.yml b/.github/workflows/build-on-tag.yml index 2e79483..a10e884 100644 --- a/.github/workflows/build-on-tag.yml +++ b/.github/workflows/build-on-tag.yml @@ -51,6 +51,7 @@ jobs: node scripts/axolotl/check-migrations.mjs --release node scripts/axolotl/brand-guard.mjs node scripts/axolotl/i18n-check.mjs + node --test scripts/axolotl/publish-skin-site.test.mjs cargo fmt --all --check - name: Check updater signing configuration @@ -286,6 +287,9 @@ jobs: name: Verify and publish needs: build runs-on: ubuntu-24.04 + concurrency: + group: skin-site-${{ github.ref_name }} + cancel-in-progress: false env: UPDATE_SERVER_URL: ${{ secrets.UPDATE_SERVER_URL }} UPDATE_SERVER_WEBHOOK_SECRET: ${{ secrets.UPDATE_SERVER_WEBHOOK_SECRET }} @@ -298,6 +302,11 @@ jobs: fetch-depth: 0 ref: ${{ github.ref }} + - name: Set up Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version-file: .nvmrc + - name: Download updater signatures env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -375,42 +384,14 @@ jobs: gh release upload "${GITHUB_REF_NAME}" "release-check/${bundle_name}" --clobber - name: Publish to skin site - continue-on-error: true env: SKIN_SITE_TOKEN: ${{ secrets.SKIN_SITE_CI_UPLOAD_TOKEN }} SKIN_SITE_URL: ${{ vars.SKIN_SITE_URL || 'https://skin.starlight.cool' }} run: | set -euo pipefail - if [ -z "${SKIN_SITE_TOKEN:-}" ]; then - echo "::warning::SKIN_SITE_CI_UPLOAD_TOKEN not configured, skipping skin site publish" - exit 0 - fi - version="${GITHUB_REF_NAME#v}" bundle="release-check/Starlight_Launcher_${version}_skin-site-update.zip" - if [ ! -f "${bundle}" ]; then - echo "::warning::Bundle ${bundle} not found, skipping skin site publish" - exit 0 - fi - - upload_response=$(curl -fsS -X POST \ - -H "Authorization: Bearer ${SKIN_SITE_TOKEN}" \ - -F "archive=@${bundle};type=application/zip" \ - "${SKIN_SITE_URL}/starlight/launcher/admin/upload?forceUpdate=false") - - release_id=$(printf '%s' "${upload_response}" | jq -r '.payload.id // empty') - if [ -z "${release_id}" ]; then - echo "::warning::Skin site upload did not return a release id: ${upload_response}" - exit 0 - fi - - echo "Uploaded release id: ${release_id}" - - curl -fsS -X POST \ - -H "Authorization: Bearer ${SKIN_SITE_TOKEN}" \ - "${SKIN_SITE_URL}/starlight/launcher/admin/${release_id}/activate" - - echo "Skin site release ${version} published" + node scripts/axolotl/publish-skin-site.mjs "${bundle}" "${GITHUB_REF_NAME}" - name: Notify Update Server to publish release assets if: ${{ env.UPDATE_SERVER_URL != '' && env.UPDATE_SERVER_WEBHOOK_SECRET != '' }} diff --git a/.github/workflows/publish-skin-site.yml b/.github/workflows/publish-skin-site.yml new file mode 100644 index 0000000..96665c8 --- /dev/null +++ b/.github/workflows/publish-skin-site.yml @@ -0,0 +1,56 @@ +name: Sync release to skin site + +on: + workflow_dispatch: + inputs: + tag: + description: Existing GitHub release tag to sync (for example v1.0.2-beta10) + required: true + type: string + +permissions: + contents: read + +concurrency: + group: skin-site-${{ inputs.tag }} + cancel-in-progress: false + +jobs: + publish: + name: Upload and activate existing release + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - name: Check out publisher + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version-file: .nvmrc + + - name: Download complete update package + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + if [[ ! "${RELEASE_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo '::error::Expected a semantic version tag beginning with v' + exit 1 + fi + version="${RELEASE_TAG#v}" + gh release download "${RELEASE_TAG}" \ + --pattern "Starlight_Launcher_${version}_skin-site-update.zip" \ + --dir release-check + + - name: Publish to skin site + env: + RELEASE_TAG: ${{ inputs.tag }} + SKIN_SITE_TOKEN: ${{ secrets.SKIN_SITE_CI_UPLOAD_TOKEN }} + SKIN_SITE_URL: ${{ vars.SKIN_SITE_URL || 'https://skin.starlight.cool' }} + run: | + set -euo pipefail + version="${RELEASE_TAG#v}" + node scripts/axolotl/publish-skin-site.mjs \ + "release-check/Starlight_Launcher_${version}_skin-site-update.zip" "${RELEASE_TAG}" diff --git a/scripts/axolotl/publish-skin-site.mjs b/scripts/axolotl/publish-skin-site.mjs new file mode 100644 index 0000000..7287bfa --- /dev/null +++ b/scripts/axolotl/publish-skin-site.mjs @@ -0,0 +1,226 @@ +import { randomUUID } from 'node:crypto' +import { open, stat } from 'node:fs/promises' +import path from 'node:path' +import { pathToFileURL } from 'node:url' + +export async function publishSkinSite({ + archivePath, + tag, + token, + baseUrl = 'https://skin.starlight.cool', + fetchImpl = fetch, + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + now = Date.now, + timeoutMs = 30 * 60 * 1000, + pollIntervalMs = 2000, + log = console.log, +}) { + if (!token) throw new Error('SKIN_SITE_CI_UPLOAD_TOKEN is required to publish to the skin site') + if (!archivePath || !/^v?\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(tag ?? '')) { + throw new Error('Expected a complete update ZIP and a semantic version tag') + } + const version = tag.replace(/^v/, '') + const url = new URL(baseUrl) + if (url.protocol !== 'https:' || url.username || url.password) { + throw new Error('SKIN_SITE_URL must be an HTTPS URL without credentials') + } + const { size } = await stat(archivePath) + if (!archivePath.toLowerCase().endsWith('.zip') || size < 1 || size > 4 * 1024 ** 3) { + throw new Error('Expected a nonempty complete update ZIP no larger than 4 GiB') + } + const deadline = now() + timeoutMs + function remainingTime() { + const remaining = deadline - now() + if (remaining <= 0) throw new Error('Skin site publication timed out') + return remaining + } + async function request(route, { method = 'GET', body, chunk = false } = {}) { + const attempts = chunk ? 3 : 1 + for (let attempt = 0; attempt < attempts; attempt++) { + const requestTimeout = Math.min(120_000, remainingTime()) + let response + try { + response = await fetchImpl(`${baseUrl.replace(/\/$/, '')}/starlight/${route}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + ...(body && !chunk ? { 'Content-Type': 'application/json' } : {}), + }, + body: body && !chunk ? JSON.stringify(body) : body, + signal: AbortSignal.timeout(requestTimeout), + redirect: 'error', + }) + } catch { + if (attempt + 1 === attempts) + throw new Error(`Skin site ${route}: network request failed or timed out`) + await sleep(500 * (attempt + 1)) + continue + } + if (!response.ok) { + await response.body?.cancel() + if ((response.status === 429 || response.status >= 500) && attempt + 1 < attempts) { + await sleep(500 * (attempt + 1)) + continue + } + throw new Error(`Skin site ${route}: HTTP ${response.status}`) + } + let result + try { + result = await response.json() + } catch { + throw new Error(`Skin site ${route}: invalid JSON response`) + } + if (result?.success !== true || !result.payload) { + throw new Error( + `Skin site ${route}: ${result?.errorMessage || 'request was rejected or payload is missing'}`, + ) + } + return result.payload + } + } + + let status = await request('launcher/admin/uploads', { + method: 'POST', + body: { + uploadId: randomUUID(), + filename: path.basename(archivePath), + size, + forceUpdate: false, + }, + }) + if (typeof status.id !== 'string' || !status.id) + throw new Error('Skin site returned an invalid upload ID') + const uploadRoute = `launcher/admin/uploads/${encodeURIComponent(status.id)}` + log(`Skin site upload ${status.id}: ${size} bytes`) + const file = await open(archivePath, 'r') + try { + for (;;) { + remainingTime() + if (status.state === 'ready') { + const release = status.release + if (!Number.isSafeInteger(release?.id) || release.id <= 0 || release.version !== version) { + throw new Error( + `Skin site returned an invalid release or a version other than ${version}`, + ) + } + const activated = await request(`launcher/admin/${release.id}/activate`, { method: 'POST' }) + if ( + activated.id !== release.id || + activated.version !== version || + activated.active !== true + ) { + throw new Error(`Skin site did not confirm activation of ${version}`) + } + log(`Skin site release ${version} published (id ${release.id})`) + return activated + } + if (status.state === 'failed' || status.state === 'cancelled') { + throw new Error( + `Skin site archive processing ${status.state}: ${status.errorMessage || 'no error details'}`, + ) + } + if (status.state === 'processing') { + await sleep(Math.min(pollIntervalMs, remainingTime())) + status = await request(uploadRoute) + continue + } + if ( + status.state !== 'uploading' || + status.size !== size || + !Number.isSafeInteger(status.chunkSize) || + status.chunkSize < 1 || + status.chunkSize > 64 * 1024 ** 2 || + !Number.isSafeInteger(status.received) || + status.received < 0 || + status.received > size || + !Array.isArray(status.receivedChunks) + ) { + throw new Error('Skin site returned invalid upload status or chunk size') + } + const count = Math.ceil(size / status.chunkSize) + if ( + status.receivedChunks.some( + (index) => !Number.isSafeInteger(index) || index < 0 || index >= count, + ) + ) { + throw new Error('Skin site returned invalid received chunk indices') + } + const received = new Set(status.receivedChunks) + const missing = Array.from({ length: count }, (_, index) => index).filter( + (index) => !received.has(index), + ) + if (!missing.length) { + if (status.received !== size) + throw new Error('Skin site received byte count does not match the archive') + status = await request(`${uploadRoute}/complete`, { method: 'POST' }) + continue + } + const chunkSize = status.chunkSize + const concurrency = Math.min( + Number.isSafeInteger(status.uploadConcurrency) && status.uploadConcurrency > 0 + ? status.uploadConcurrency + : 1, + 6, + missing.length, + ) + let cursor = 0 + let uploadError + async function worker() { + try { + while (!uploadError && cursor < missing.length) { + remainingTime() + const index = missing[cursor++] + const start = index * chunkSize + const bytes = Buffer.alloc(Math.min(chunkSize, size - start)) + let offset = 0 + while (offset < bytes.length) { + const { bytesRead } = await file.read( + bytes, + offset, + bytes.length - offset, + start + offset, + ) + if (!bytesRead) throw new Error('Update archive changed or ended during upload') + offset += bytesRead + } + const form = new FormData() + form.append('file', new Blob([bytes]), 'chunk.bin') + await request(`${uploadRoute}/chunks/${index}`, { + method: 'POST', + body: form, + chunk: true, + }) + } + } catch (error) { + uploadError ??= error + } + } + await Promise.all(Array.from({ length: concurrency }, worker)) + if (uploadError) throw uploadError + status = await request(uploadRoute) + log(`Skin site uploaded ${status.received}/${size} bytes`) + } + } finally { + await file.close() + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + try { + await publishSkinSite({ + archivePath: process.argv[2], + tag: process.argv[3], + token: process.env.SKIN_SITE_TOKEN, + baseUrl: process.env.SKIN_SITE_URL || 'https://skin.starlight.cool', + }) + } catch (error) { + const message = String(error.message).replaceAll( + process.env.SKIN_SITE_TOKEN || '\0', + '[redacted]', + ) + console.error( + `::error::${message.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A')}`, + ) + process.exitCode = 1 + } +} diff --git a/scripts/axolotl/publish-skin-site.test.mjs b/scripts/axolotl/publish-skin-site.test.mjs new file mode 100644 index 0000000..9f51f63 --- /dev/null +++ b/scripts/axolotl/publish-skin-site.test.mjs @@ -0,0 +1,172 @@ +import assert from 'node:assert/strict' +import { mkdtemp, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { spawnSync } from 'node:child_process' +import { publishSkinSite } from './publish-skin-site.mjs' + +async function fixture(t) { + const directory = await mkdtemp(path.join(tmpdir(), 'skin-site-publish-')) + t.after(() => rm(directory, { recursive: true, force: true })) + const archivePath = path.join(directory, 'Starlight_Launcher_1.0.2-beta10_skin-site-update.zip') + const bytes = Buffer.from('a small archive split into several chunks') + await writeFile(archivePath, bytes) + return { archivePath, bytes } +} + +function apiFixture(bytes, options = {}) { + const chunks = new Map() + const events = [] + const release = { id: 17, version: '1.0.2-beta10', active: false } + let state = 'uploading' + let failedOnce = false + if (options.resume) chunks.set(0, bytes.subarray(0, 8)) + const status = () => ({ + id: 'session-1', + chunkSize: 8, + size: bytes.length, + received: [...chunks.values()].reduce((total, part) => total + part.length, 0), + receivedChunks: [...chunks.keys()], + uploadConcurrency: 3, + state, + ...(state === 'ready' ? { release } : {}), + ...(state === 'failed' ? { errorMessage: 'Invalid archive manifest' } : {}), + }) + const reply = (payload) => Response.json({ success: true, payload }) + return { + chunks, + events, + fetch: async (url, init) => { + assert.equal(init.headers.Authorization, 'Bearer test-token') + const route = new URL(url).pathname.replace('/starlight/', '') + events.push(route) + if (route === 'launcher/admin/uploads') { + assert.equal(JSON.parse(init.body).size, bytes.length) + assert.equal(JSON.parse(init.body).forceUpdate, false) + return reply({ ...status(), ...(options.invalidChunkSize ? { chunkSize: 0 } : {}) }) + } + if (route.includes('/chunks/')) { + if (options.rejectChunks) return new Response('too large', { status: 413 }) + if (options.retryChunk && !failedOnce) { + failedOnce = true + return new Response('temporary failure', { status: 503 }) + } + const index = Number(route.split('/').at(-1)) + const file = init.body.get('file') + assert.equal(file.name, 'chunk.bin') + chunks.set(index, Buffer.from(await file.arrayBuffer())) + return reply(status()) + } + if (route.endsWith('/complete')) { + assert.deepEqual( + Buffer.concat([...chunks].sort(([a], [b]) => a - b).map(([, part]) => part)), + bytes, + ) + state = 'processing' + return reply(status()) + } + if (route.endsWith('/activate')) { + assert.equal(state, 'ready') + return options.activationFailure + ? Response.json({ success: false, errorMessage: 'Activation denied' }) + : reply({ ...release, active: true }) + } + if (route === 'launcher/admin/uploads/session-1') { + if (state === 'processing' && !options.stuck) + state = options.processingFailure ? 'failed' : 'ready' + return reply({ + ...status(), + ...(options.wrongVersion && state === 'ready' + ? { release: { ...release, version: '1.0.2-beta9' } } + : {}), + }) + } + throw new Error(`Unexpected route ${route}`) + }, + } +} + +function settings(archivePath, api) { + return { + archivePath, + tag: 'v1.0.2-beta10', + token: 'test-token', + baseUrl: 'https://skin.example', + fetchImpl: api.fetch, + pollIntervalMs: 1, + sleep: async () => {}, + log: () => {}, + } +} + +test('uploads exact bounded chunks, retries temporary errors, then completes and activates', async (t) => { + const { archivePath, bytes } = await fixture(t) + const api = apiFixture(bytes, { retryChunk: true }) + const result = await publishSkinSite(settings(archivePath, api)) + assert.equal(result.active, true) + assert.equal(api.chunks.size, Math.ceil(bytes.length / 8)) + assert.ok([...api.chunks.values()].every((part) => part.length <= 8)) + assert.equal(api.events.at(-1), 'launcher/admin/17/activate') +}) + +for (const [name, options, error] of [ + ['HTTP upload rejection', { rejectChunks: true }, /413/], + ['archive processing failure', { processingFailure: true }, /Invalid archive manifest/], + ['application-level activation failure', { activationFailure: true }, /Activation denied/], + ['invalid server chunk size', { invalidChunkSize: true }, /invalid upload status/], + ['mismatched release version', { wrongVersion: true }, /version other than/], +]) { + test(`fails explicitly on ${name}`, async (t) => { + const { archivePath, bytes } = await fixture(t) + const api = apiFixture(bytes, options) + await assert.rejects(publishSkinSite(settings(archivePath, api)), error) + if (!options.activationFailure) + assert.ok(!api.events.some((event) => event.endsWith('/activate'))) + }) +} + +test('fails instead of hanging when processing never finishes', async (t) => { + const { archivePath, bytes } = await fixture(t) + const api = apiFixture(bytes, { stuck: true }) + let now = 0 + await assert.rejects( + publishSkinSite({ + ...settings(archivePath, api), + timeoutMs: 10, + now: () => now, + sleep: async () => { + now += 5 + }, + }), + /timed out/i, + ) +}) + +test('missing credentials fail before any network request', async (t) => { + const { archivePath } = await fixture(t) + await assert.rejects( + publishSkinSite({ + ...settings(archivePath, { fetch: () => assert.fail('Unexpected network request') }), + token: '', + }), + /SKIN_SITE_CI_UPLOAD_TOKEN/, + ) +}) + +test('resumes server-acknowledged chunks without uploading them again', async (t) => { + const { archivePath, bytes } = await fixture(t) + const api = apiFixture(bytes, { resume: true }) + await publishSkinSite(settings(archivePath, api)) + assert.ok(!api.events.some((event) => event.endsWith('/chunks/0'))) +}) + +test('CLI emits a GitHub error and exits nonzero when credentials are missing', () => { + const result = spawnSync( + process.execPath, + ['scripts/axolotl/publish-skin-site.mjs', 'missing.zip', 'v1.0.2-beta10'], + { encoding: 'utf8', env: { ...process.env, SKIN_SITE_TOKEN: '' } }, + ) + assert.equal(result.status, 1) + assert.match(result.stderr, /::error::.*SKIN_SITE_CI_UPLOAD_TOKEN/) +})