311 lines
10 KiB
JavaScript
311 lines
10 KiB
JavaScript
import { execFileSync } from 'node:child_process'
|
|
import fs from 'node:fs/promises'
|
|
import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs'
|
|
import { stripTypeScriptTypes } from 'node:module'
|
|
import { dirname, extname, join, relative, resolve, sep } from 'node:path'
|
|
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
import vm from 'node:vm'
|
|
|
|
const REPOSITORY_ROOT = fileURLToPath(new URL('../../', import.meta.url))
|
|
const CATALOG_PATH = resolve(REPOSITORY_ROOT, 'packages/app-lib/src/api/lobehub_text_models.json')
|
|
const RUST_API_PATH = resolve(REPOSITORY_ROOT, 'packages/app-lib/src/api/ai.rs')
|
|
const SOURCE_PATTERN = /^LobeHub ([0-9a-f]{40}) model-bank chat models$/
|
|
const RUST_SOURCE_PATTERN = /const CATALOG_SOURCE: &str = "LobeHub ([0-9a-f]{40})";/g
|
|
|
|
// Sponsored providers are paid partner listings that must survive every
|
|
// synchronization, even if a future rewrite of ai.rs drops upstream providers.
|
|
// Keep this list in sync with SPONSORED_PROVIDERS in packages/app-lib/src/api/ai.rs.
|
|
const SPONSORED_PROVIDER_IDS = ['codeflow']
|
|
|
|
function assertSponsoredProvidersPreserved(rustSource) {
|
|
for (const providerId of SPONSORED_PROVIDER_IDS) {
|
|
const pattern = new RegExp(`provider!\\s*\\(\\s*"${providerId}"`)
|
|
if (!pattern.test(rustSource)) {
|
|
throw new Error(
|
|
`Sponsored provider ${providerId} is missing from ai.rs; refusing to rewrite it`,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
function usage() {
|
|
console.log(`Usage: node --experimental-vm-modules scripts/axolotl/sync-lobehub-models.mjs \\
|
|
--upstream <lobehub checkout> [--commit <40-character SHA>] [--check]`)
|
|
}
|
|
|
|
function parseArguments(arguments_) {
|
|
const options = { check: false, commit: undefined, upstream: undefined }
|
|
for (let index = 0; index < arguments_.length; index++) {
|
|
const argument = arguments_[index]
|
|
switch (argument) {
|
|
case '--check':
|
|
options.check = true
|
|
break
|
|
case '--commit':
|
|
case '--upstream': {
|
|
const value = arguments_[++index]
|
|
if (!value) throw new Error(`${argument} requires a value`)
|
|
options[argument.slice(2)] = value
|
|
break
|
|
}
|
|
case '--help':
|
|
case '-h':
|
|
usage()
|
|
process.exit(0)
|
|
default:
|
|
throw new Error(`Unknown argument: ${argument}`)
|
|
}
|
|
}
|
|
if (!options.upstream) throw new Error('--upstream is required')
|
|
return options
|
|
}
|
|
|
|
function requireObject(value, label) {
|
|
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
throw new Error(`${label} must be an object`)
|
|
}
|
|
return value
|
|
}
|
|
|
|
function countModels(providers) {
|
|
return Object.values(providers).reduce((total, models) => total + models.length, 0)
|
|
}
|
|
|
|
function resolveCommit(upstreamRoot, requestedCommit) {
|
|
const commit =
|
|
requestedCommit ??
|
|
execFileSync('git', ['-C', upstreamRoot, 'rev-parse', 'HEAD'], {
|
|
encoding: 'utf8',
|
|
}).trim()
|
|
if (!/^[0-9a-f]{40}$/.test(commit)) {
|
|
throw new Error(`Invalid LobeHub commit SHA: ${commit}`)
|
|
}
|
|
return commit
|
|
}
|
|
|
|
function createModelBankLoader(upstreamRoot) {
|
|
if (typeof vm.SourceTextModule !== 'function') {
|
|
throw new Error('Node.js must be run with --experimental-vm-modules')
|
|
}
|
|
|
|
const sourceRoot = realpathSync(resolve(upstreamRoot, 'packages/model-bank/src'))
|
|
const context = vm.createContext(Object.create(null), {
|
|
codeGeneration: { strings: false, wasm: false },
|
|
name: 'lobehub-model-bank',
|
|
})
|
|
const moduleCache = new Map()
|
|
const schema = vm.runInContext(
|
|
`(() => {
|
|
let schema
|
|
schema = new Proxy(function () { return schema }, { get() { return schema } })
|
|
return schema
|
|
})()`,
|
|
context,
|
|
)
|
|
const zodModule = new vm.SyntheticModule(
|
|
['z'],
|
|
function () {
|
|
this.setExport('z', schema)
|
|
},
|
|
{ context, identifier: 'sandbox:zod' },
|
|
)
|
|
const typeFestModule = new vm.SyntheticModule([], function () {}, {
|
|
context,
|
|
identifier: 'sandbox:type-fest',
|
|
})
|
|
|
|
function resolveLocalImport(specifier, parentPath) {
|
|
const basePath = resolve(dirname(parentPath), specifier)
|
|
const candidates = extname(basePath)
|
|
? [basePath]
|
|
: [`${basePath}.ts`, join(basePath, 'index.ts')]
|
|
for (const candidate of candidates) {
|
|
if (!existsSync(candidate) || !statSync(candidate).isFile()) continue
|
|
const canonicalPath = realpathSync(candidate)
|
|
const relativePath = relative(sourceRoot, canonicalPath)
|
|
if (relativePath === '..' || relativePath.startsWith(`..${sep}`)) {
|
|
throw new Error(`Import escapes model-bank/src: ${specifier}`)
|
|
}
|
|
return canonicalPath
|
|
}
|
|
throw new Error(`Unable to resolve ${specifier} from ${parentPath}`)
|
|
}
|
|
|
|
function getModule(filePath) {
|
|
const cached = moduleCache.get(filePath)
|
|
if (cached) return cached
|
|
const code = stripTypeScriptTypes(readFileSync(filePath, 'utf8'), {
|
|
mode: 'transform',
|
|
sourceUrl: pathToFileURL(filePath).href,
|
|
})
|
|
const module = new vm.SourceTextModule(code, {
|
|
context,
|
|
identifier: pathToFileURL(filePath).href,
|
|
importModuleDynamically() {
|
|
throw new Error('Dynamic imports are not allowed in the LobeHub model bank')
|
|
},
|
|
})
|
|
moduleCache.set(filePath, module)
|
|
return module
|
|
}
|
|
|
|
function linker(specifier, referencingModule) {
|
|
if (specifier === 'zod') return zodModule
|
|
if (specifier === 'type-fest') return typeFestModule
|
|
if (!specifier.startsWith('.')) {
|
|
throw new Error(`External import is not allowed in the model bank: ${specifier}`)
|
|
}
|
|
return getModule(resolveLocalImport(specifier, fileURLToPath(referencingModule.identifier)))
|
|
}
|
|
|
|
return async function loadModels() {
|
|
const entryPath = realpathSync(join(sourceRoot, 'aiModels/index.ts'))
|
|
const entry = getModule(entryPath)
|
|
await entry.link(linker)
|
|
await entry.evaluate({ timeout: 20_000 })
|
|
return entry.namespace.LOBE_DEFAULT_MODEL_LIST
|
|
}
|
|
}
|
|
|
|
function validateCurrentCatalog(catalog, rustSource) {
|
|
requireObject(catalog, 'Current model catalog')
|
|
const providers = requireObject(catalog.providers, 'Current model catalog providers')
|
|
const sourceMatch = SOURCE_PATTERN.exec(catalog.source)
|
|
if (!sourceMatch) throw new Error('Current model catalog has an invalid source')
|
|
|
|
const rustMatches = [...rustSource.matchAll(RUST_SOURCE_PATTERN)]
|
|
if (rustMatches.length !== 1) {
|
|
throw new Error('Expected exactly one CATALOG_SOURCE constant in ai.rs')
|
|
}
|
|
if (rustMatches[0][1] !== sourceMatch[1]) {
|
|
throw new Error('The JSON and Rust catalog source commits do not match')
|
|
}
|
|
|
|
for (const [providerId, models] of Object.entries(providers)) {
|
|
if (!/^[a-z0-9]+$/.test(providerId) || !Array.isArray(models)) {
|
|
throw new Error(`Invalid current catalog provider: ${providerId}`)
|
|
}
|
|
}
|
|
return providers
|
|
}
|
|
|
|
function buildCatalog(models, currentProviders) {
|
|
if (!Array.isArray(models)) {
|
|
throw new Error('LOBE_DEFAULT_MODEL_LIST must be an array')
|
|
}
|
|
|
|
const providers = Object.fromEntries(
|
|
Object.keys(currentProviders).map((providerId) => [providerId, []]),
|
|
)
|
|
const seen = new Set()
|
|
let upstreamChatCount = 0
|
|
|
|
for (const model of models) {
|
|
requireObject(model, 'LobeHub model')
|
|
if (model.type !== 'chat') continue
|
|
upstreamChatCount++
|
|
if (typeof model.providerId !== 'string' || !model.providerId) {
|
|
throw new Error('LobeHub chat model has no providerId')
|
|
}
|
|
if (typeof model.id !== 'string' || !model.id.trim()) {
|
|
throw new Error(`LobeHub ${model.providerId} chat model has no id`)
|
|
}
|
|
if (typeof model.enabled !== 'boolean') {
|
|
throw new Error(`LobeHub model ${model.providerId}/${model.id} has invalid enabled state`)
|
|
}
|
|
if (!Object.hasOwn(providers, model.providerId)) continue
|
|
|
|
const key = `${model.providerId}\0${model.id}`
|
|
if (seen.has(key)) throw new Error(`Duplicate LobeHub model: ${model.providerId}/${model.id}`)
|
|
seen.add(key)
|
|
const displayName = model.displayName || model.id
|
|
if (typeof displayName !== 'string' || !displayName.trim()) {
|
|
throw new Error(`LobeHub model ${model.providerId}/${model.id} has no display name`)
|
|
}
|
|
providers[model.providerId].push({
|
|
id: model.id,
|
|
name: displayName,
|
|
enabled: model.enabled,
|
|
})
|
|
}
|
|
|
|
for (const providerId of SPONSORED_PROVIDER_IDS) {
|
|
const sponsorModels = currentProviders[providerId]
|
|
if (!sponsorModels?.length) continue
|
|
providers[providerId] = [...sponsorModels]
|
|
console.log(
|
|
`Preserved ${sponsorModels.length} sponsored model(s) for ${providerId}`,
|
|
)
|
|
}
|
|
|
|
const previousCount = countModels(currentProviders)
|
|
const synchronizedCount = countModels(providers)
|
|
const minimumCount = Math.max(100, Math.floor(previousCount * 0.7))
|
|
const maximumCount = Math.max(500, Math.ceil(previousCount * 2))
|
|
if (synchronizedCount < minimumCount || synchronizedCount > maximumCount) {
|
|
throw new Error(
|
|
`Refusing suspicious model count change: ${previousCount} -> ${synchronizedCount}`,
|
|
)
|
|
}
|
|
for (const [providerId, previousModels] of Object.entries(currentProviders)) {
|
|
if (previousModels.length > 0 && providers[providerId].length === 0) {
|
|
throw new Error(`Refusing to empty existing provider ${providerId}`)
|
|
}
|
|
}
|
|
|
|
return { providers, synchronizedCount, upstreamChatCount }
|
|
}
|
|
|
|
const options = parseArguments(process.argv.slice(2))
|
|
const upstreamRoot = realpathSync(resolve(options.upstream))
|
|
const commit = resolveCommit(upstreamRoot, options.commit)
|
|
const [catalogText, rustSource] = await Promise.all([
|
|
fs.readFile(CATALOG_PATH, 'utf8'),
|
|
fs.readFile(RUST_API_PATH, 'utf8'),
|
|
])
|
|
assertSponsoredProvidersPreserved(rustSource)
|
|
const currentCatalog = JSON.parse(catalogText)
|
|
const currentProviders = validateCurrentCatalog(currentCatalog, rustSource)
|
|
const loadModels = createModelBankLoader(upstreamRoot)
|
|
const { providers, synchronizedCount, upstreamChatCount } = buildCatalog(
|
|
await loadModels(),
|
|
currentProviders,
|
|
)
|
|
const providersChanged = JSON.stringify(providers) !== JSON.stringify(currentProviders)
|
|
|
|
if (!providersChanged) {
|
|
console.log(
|
|
`LobeHub ${commit} has no model changes for ${Object.keys(providers).length} supported providers (${synchronizedCount}/${upstreamChatCount} chat models).`,
|
|
)
|
|
process.exit(0)
|
|
}
|
|
|
|
const nextCatalogText = `${JSON.stringify(
|
|
{
|
|
source: `LobeHub ${commit} model-bank chat models`,
|
|
providers,
|
|
},
|
|
null,
|
|
'\t',
|
|
)}\n`
|
|
const nextRustSource = rustSource.replace(
|
|
RUST_SOURCE_PATTERN,
|
|
`const CATALOG_SOURCE: &str = "LobeHub ${commit}";`,
|
|
)
|
|
assertSponsoredProvidersPreserved(nextRustSource)
|
|
|
|
if (options.check) {
|
|
console.error(
|
|
`Bundled model catalog is stale: LobeHub ${commit} has ${synchronizedCount}/${upstreamChatCount} applicable chat models.`,
|
|
)
|
|
process.exit(1)
|
|
}
|
|
|
|
await Promise.all([
|
|
fs.writeFile(CATALOG_PATH, nextCatalogText),
|
|
fs.writeFile(RUST_API_PATH, nextRustSource),
|
|
])
|
|
console.log(
|
|
`Synchronized ${synchronizedCount}/${upstreamChatCount} LobeHub chat models from ${commit} across ${Object.keys(providers).length} supported providers.`,
|
|
)
|