import { execFileSync } from 'node:child_process' import fs from 'node:fs/promises' import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs' import { stripTypeScriptTypes } from 'node:module' import { dirname, extname, join, relative, resolve, sep } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import vm from 'node:vm' const REPOSITORY_ROOT = fileURLToPath(new URL('../../', import.meta.url)) const CATALOG_PATH = resolve(REPOSITORY_ROOT, 'packages/app-lib/src/api/lobehub_text_models.json') const RUST_API_PATH = resolve(REPOSITORY_ROOT, 'packages/app-lib/src/api/ai.rs') const SOURCE_PATTERN = /^LobeHub ([0-9a-f]{40}) model-bank chat models$/ const RUST_SOURCE_PATTERN = /const CATALOG_SOURCE: &str = "LobeHub ([0-9a-f]{40})";/g // Sponsored providers are paid partner listings that must survive every // synchronization, even if a future rewrite of ai.rs drops upstream providers. // Keep this list in sync with SPONSORED_PROVIDERS in packages/app-lib/src/api/ai.rs. const SPONSORED_PROVIDER_IDS = ['codeflow'] function assertSponsoredProvidersPreserved(rustSource) { for (const providerId of SPONSORED_PROVIDER_IDS) { const pattern = new RegExp(`provider!\\s*\\(\\s*"${providerId}"`) if (!pattern.test(rustSource)) { throw new Error( `Sponsored provider ${providerId} is missing from ai.rs; refusing to rewrite it`, ) } } } function usage() { console.log(`Usage: node --experimental-vm-modules scripts/axolotl/sync-lobehub-models.mjs \\ --upstream [--commit <40-character SHA>] [--check]`) } function parseArguments(arguments_) { const options = { check: false, commit: undefined, upstream: undefined } for (let index = 0; index < arguments_.length; index++) { const argument = arguments_[index] switch (argument) { case '--check': options.check = true break case '--commit': case '--upstream': { const value = arguments_[++index] if (!value) throw new Error(`${argument} requires a value`) options[argument.slice(2)] = value break } case '--help': case '-h': usage() process.exit(0) default: throw new Error(`Unknown argument: ${argument}`) } } if (!options.upstream) throw new Error('--upstream is required') return options } function requireObject(value, label) { if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error(`${label} must be an object`) } return value } function countModels(providers) { return Object.values(providers).reduce((total, models) => total + models.length, 0) } function resolveCommit(upstreamRoot, requestedCommit) { const commit = requestedCommit ?? execFileSync('git', ['-C', upstreamRoot, 'rev-parse', 'HEAD'], { encoding: 'utf8', }).trim() if (!/^[0-9a-f]{40}$/.test(commit)) { throw new Error(`Invalid LobeHub commit SHA: ${commit}`) } return commit } function createModelBankLoader(upstreamRoot) { if (typeof vm.SourceTextModule !== 'function') { throw new Error('Node.js must be run with --experimental-vm-modules') } const sourceRoot = realpathSync(resolve(upstreamRoot, 'packages/model-bank/src')) const context = vm.createContext(Object.create(null), { codeGeneration: { strings: false, wasm: false }, name: 'lobehub-model-bank', }) const moduleCache = new Map() const schema = vm.runInContext( `(() => { let schema schema = new Proxy(function () { return schema }, { get() { return schema } }) return schema })()`, context, ) const zodModule = new vm.SyntheticModule( ['z'], function () { this.setExport('z', schema) }, { context, identifier: 'sandbox:zod' }, ) const typeFestModule = new vm.SyntheticModule([], function () {}, { context, identifier: 'sandbox:type-fest', }) function resolveLocalImport(specifier, parentPath) { const basePath = resolve(dirname(parentPath), specifier) const candidates = extname(basePath) ? [basePath] : [`${basePath}.ts`, join(basePath, 'index.ts')] for (const candidate of candidates) { if (!existsSync(candidate) || !statSync(candidate).isFile()) continue const canonicalPath = realpathSync(candidate) const relativePath = relative(sourceRoot, canonicalPath) if (relativePath === '..' || relativePath.startsWith(`..${sep}`)) { throw new Error(`Import escapes model-bank/src: ${specifier}`) } return canonicalPath } throw new Error(`Unable to resolve ${specifier} from ${parentPath}`) } function getModule(filePath) { const cached = moduleCache.get(filePath) if (cached) return cached const code = stripTypeScriptTypes(readFileSync(filePath, 'utf8'), { mode: 'transform', sourceUrl: pathToFileURL(filePath).href, }) const module = new vm.SourceTextModule(code, { context, identifier: pathToFileURL(filePath).href, importModuleDynamically() { throw new Error('Dynamic imports are not allowed in the LobeHub model bank') }, }) moduleCache.set(filePath, module) return module } function linker(specifier, referencingModule) { if (specifier === 'zod') return zodModule if (specifier === 'type-fest') return typeFestModule if (!specifier.startsWith('.')) { throw new Error(`External import is not allowed in the model bank: ${specifier}`) } return getModule(resolveLocalImport(specifier, fileURLToPath(referencingModule.identifier))) } return async function loadModels() { const entryPath = realpathSync(join(sourceRoot, 'aiModels/index.ts')) const entry = getModule(entryPath) await entry.link(linker) await entry.evaluate({ timeout: 20_000 }) return entry.namespace.LOBE_DEFAULT_MODEL_LIST } } function validateCurrentCatalog(catalog, rustSource) { requireObject(catalog, 'Current model catalog') const providers = requireObject(catalog.providers, 'Current model catalog providers') const sourceMatch = SOURCE_PATTERN.exec(catalog.source) if (!sourceMatch) throw new Error('Current model catalog has an invalid source') const rustMatches = [...rustSource.matchAll(RUST_SOURCE_PATTERN)] if (rustMatches.length !== 1) { throw new Error('Expected exactly one CATALOG_SOURCE constant in ai.rs') } if (rustMatches[0][1] !== sourceMatch[1]) { throw new Error('The JSON and Rust catalog source commits do not match') } for (const [providerId, models] of Object.entries(providers)) { if (!/^[a-z0-9]+$/.test(providerId) || !Array.isArray(models)) { throw new Error(`Invalid current catalog provider: ${providerId}`) } } return providers } function buildCatalog(models, currentProviders) { if (!Array.isArray(models)) { throw new Error('LOBE_DEFAULT_MODEL_LIST must be an array') } const providers = Object.fromEntries( Object.keys(currentProviders).map((providerId) => [providerId, []]), ) const seen = new Set() let upstreamChatCount = 0 for (const model of models) { requireObject(model, 'LobeHub model') if (model.type !== 'chat') continue upstreamChatCount++ if (typeof model.providerId !== 'string' || !model.providerId) { throw new Error('LobeHub chat model has no providerId') } if (typeof model.id !== 'string' || !model.id.trim()) { throw new Error(`LobeHub ${model.providerId} chat model has no id`) } if (typeof model.enabled !== 'boolean') { throw new Error(`LobeHub model ${model.providerId}/${model.id} has invalid enabled state`) } if (!Object.hasOwn(providers, model.providerId)) continue const key = `${model.providerId}\0${model.id}` if (seen.has(key)) throw new Error(`Duplicate LobeHub model: ${model.providerId}/${model.id}`) seen.add(key) const displayName = model.displayName || model.id if (typeof displayName !== 'string' || !displayName.trim()) { throw new Error(`LobeHub model ${model.providerId}/${model.id} has no display name`) } providers[model.providerId].push({ id: model.id, name: displayName, enabled: model.enabled, }) } for (const providerId of SPONSORED_PROVIDER_IDS) { const sponsorModels = currentProviders[providerId] if (!sponsorModels?.length) continue providers[providerId] = [...sponsorModels] console.log( `Preserved ${sponsorModels.length} sponsored model(s) for ${providerId}`, ) } const previousCount = countModels(currentProviders) const synchronizedCount = countModels(providers) const minimumCount = Math.max(100, Math.floor(previousCount * 0.7)) const maximumCount = Math.max(500, Math.ceil(previousCount * 2)) if (synchronizedCount < minimumCount || synchronizedCount > maximumCount) { throw new Error( `Refusing suspicious model count change: ${previousCount} -> ${synchronizedCount}`, ) } for (const [providerId, previousModels] of Object.entries(currentProviders)) { if (previousModels.length > 0 && providers[providerId].length === 0) { throw new Error(`Refusing to empty existing provider ${providerId}`) } } return { providers, synchronizedCount, upstreamChatCount } } const options = parseArguments(process.argv.slice(2)) const upstreamRoot = realpathSync(resolve(options.upstream)) const commit = resolveCommit(upstreamRoot, options.commit) const [catalogText, rustSource] = await Promise.all([ fs.readFile(CATALOG_PATH, 'utf8'), fs.readFile(RUST_API_PATH, 'utf8'), ]) assertSponsoredProvidersPreserved(rustSource) const currentCatalog = JSON.parse(catalogText) const currentProviders = validateCurrentCatalog(currentCatalog, rustSource) const loadModels = createModelBankLoader(upstreamRoot) const { providers, synchronizedCount, upstreamChatCount } = buildCatalog( await loadModels(), currentProviders, ) const providersChanged = JSON.stringify(providers) !== JSON.stringify(currentProviders) if (!providersChanged) { console.log( `LobeHub ${commit} has no model changes for ${Object.keys(providers).length} supported providers (${synchronizedCount}/${upstreamChatCount} chat models).`, ) process.exit(0) } const nextCatalogText = `${JSON.stringify( { source: `LobeHub ${commit} model-bank chat models`, providers, }, null, '\t', )}\n` const nextRustSource = rustSource.replace( RUST_SOURCE_PATTERN, `const CATALOG_SOURCE: &str = "LobeHub ${commit}";`, ) assertSponsoredProvidersPreserved(nextRustSource) if (options.check) { console.error( `Bundled model catalog is stale: LobeHub ${commit} has ${synchronizedCount}/${upstreamChatCount} applicable chat models.`, ) process.exit(1) } await Promise.all([ fs.writeFile(CATALOG_PATH, nextCatalogText), fs.writeFile(RUST_API_PATH, nextRustSource), ]) console.log( `Synchronized ${synchronizedCount}/${upstreamChatCount} LobeHub chat models from ${commit} across ${Object.keys(providers).length} supported providers.`, )