From 0c1e62eb100ce685e167f59d45c6d5e0b9fbb6da Mon Sep 17 00:00:00 2001 From: Sami Fouad Date: Wed, 14 Oct 2026 19:27:27 -0600 Subject: [PATCH] Imports: wake the index before the finishing work (waitUntil cut it off, so imported repos stayed Pending); sources over 500 MB import from the owner's machine instead of the container. 0.7.10 --- package-lock.json | 7 +++++-- package.json | 2 +- src/server/api/imports.ts | 18 ++++++++++++++++-- src/server/import/dispatch.test.ts | 21 +++++++++++++++++++++ src/server/import/dispatch.ts | 25 +++++++++++++++++++++++++ src/server/import/do.ts | 5 +++++ src/server/import/service.ts | 2 +- 7 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 src/server/import/dispatch.test.ts diff --git a/package-lock.json b/package-lock.json index 1e99593..8e40db7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gildforge/kit", - "version": "0.7.9", + "version": "0.7.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@gildforge/kit", - "version": "0.7.9", + "version": "0.7.10", "license": "Apache-2.0", "dependencies": { "diff": "8", @@ -18,6 +18,9 @@ "yaml": "^2.9.1", "zod": "^4.6.5" }, + "bin": { + "gild-install": "scripts/install.mjs" + }, "devDependencies": { "@cloudflare/workers-types": "^5.20261006.1", "@types/markdown-it": "14", diff --git a/package.json b/package.json index 44eba1f..b896fcf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gildforge/kit", - "version": "0.7.9", + "version": "0.7.10", "description": "The shared engine, views and server behind gild: the git forge for agents on Cloudflare.", "license": "Apache-2.0", "type": "module", diff --git a/src/server/api/imports.ts b/src/server/api/imports.ts index c7fc23d..d0a8b57 100644 --- a/src/server/api/imports.ts +++ b/src/server/api/imports.ts @@ -2,7 +2,7 @@ import { getCloudflareContext } from '../context.js' import { destinationPrivate } from '../../engine/import/fence.js' import { sourceURL } from '../../engine/import/source.js' import { importJob } from '../import/service.js' -import { startImport } from '../import/dispatch.js' +import { startImport,sourceBytes,SERVER_IMPORT_LIMIT,tooLargeMessage } from '../import/dispatch.js' import { store } from '../backend.js' import { mayWrite } from '../access.js' import { ApiError } from '../../engine/api/errors.js' @@ -22,6 +22,14 @@ export async function authorizeImport(c:Context) { if(c.principal.name.includes('/') || !(await mayWrite(owner,c.principal.name))) throw new ApiError(403,'Repository owner or organization admin required') if(c.principal.repos!==null && !c.params.repo) throw new ApiError(403,'Repository creation requires an unrestricted token') } +/** False (and the job marked for the CLI) when the source is too large for + * the import container. An unknown size goes to the container as before. */ +async function serverSized(job:Awaited>,repository:string,url:string,forge?:Parameters[1],token?:string) { + const bytes=await sourceBytes(url,forge,token) + if(bytes===null||bytes<=SERVER_IMPORT_LIMIT) return true + await job.local(tooLargeMessage(bytes,repository)) + return false +} export async function apiImports(c:Context):Promise { const b=c.body,id=c.route.id,st=await store(),{env}=await getCloudflareContext(),token=c.request.headers.get('authorization')?.replace(/^Bearer /i,'')??'' if(id==='importCreate'||id==='importOrgCreate') { @@ -38,6 +46,7 @@ export async function apiImports(c:Context):Promise { // Straight from the source into Artifacts, server-side (services/import). // `x-gild-import-runner: local` keeps the job for the caller's own machine. if(c.request.headers.get('x-gild-import-runner')==='local') return {data:created} + if(!await serverSized(job,`${owner}/${name}`,source.url,source.forge,b.source_token)) return {data:await job.status()} return {data:await startImport(env as never,new URL(c.request.url).origin,()=>job.claim())?await job.status():created} } if(id==='importUserPoll'||id==='importOrgPoll') { @@ -65,7 +74,9 @@ export async function apiImports(c:Context):Promise { const {owner,repo}=c.params,job=await importJob(owner,repo) if(id==='importStatus') return {data:await job.status()} if(id==='importClaim') return {data:await job.claim()} - if(id==='importRetry') {const queued=await job.retry(b.source_token);return {data:await startImport(env as never,new URL(c.request.url).origin,()=>job.claim())?await job.status():queued}} + if(id==='importRetry') {const queued=await job.retry(b.source_token) + if(!await serverSized(job,`${owner}/${repo}`,queued.source,undefined,b.source_token)) return {data:await job.status()} + return {data:await startImport(env as never,new URL(c.request.url).origin,()=>job.claim())?await job.status():queued}} if(id==='importCutover') { const result=await job.cutover() const {writeMeta,writeJson,DIR}=await import('../metastore.js') @@ -103,6 +114,9 @@ export async function apiImports(c:Context):Promise { // re-indexing take tens of seconds, longer than an importer waits for an // answer, so they run after it (a failure is logged; the data stays). if(!b.error) { + // Wake the index first: it is durable (a Durable Object alarm), while the + // waitUntil below may be cut off after ~30s and never reach its wake. + await scheduleIndex(owner,repo) const work=(async()=>{ const { finishImportRefs } = await import('../import/finish.js') await finishImportRefs(owner,repo,active.internalPrefix??'import/pr',{url:active.source,mirror:active.mirror,readOnly:false,warnings:b.warnings,summary:b.summary}) diff --git a/src/server/import/dispatch.test.ts b/src/server/import/dispatch.test.ts new file mode 100644 index 0000000..2a4ce67 --- /dev/null +++ b/src/server/import/dispatch.test.ts @@ -0,0 +1,21 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { sourceBytes,SERVER_IMPORT_LIMIT,tooLargeMessage } from '../../../dist/server/import/dispatch.js' + +const answer=(body:unknown,ok=true)=>(async(url:string,init?:RequestInit)=>{seen.push({url:String(url),headers:init?.headers as Record});return new Response(JSON.stringify(body),{status:ok?200:404})}) as typeof fetch +let seen:{url:string;headers:Record}[]=[] + +test('a large GitHub source is measured from its API size in kilobytes', async () => { + seen=[] + const bytes=await sourceBytes('https://github.com/torvalds/linux',undefined,'t0k',answer({size:5_000_000})) + assert.equal(bytes,5_000_000*1024) + assert.ok(bytes!>SERVER_IMPORT_LIMIT) + assert.equal(seen[0].url,'https://api.github.com/repos/torvalds/linux') + assert.equal(seen[0].headers.authorization,'Bearer t0k') + assert.match(tooLargeMessage(bytes!,'sami/linux'),/4\.8 GB.*gild repo resume sami\/linux/) +}) +test('GitLab statistics are bytes; plain git and failures are unknown', async () => { + assert.equal(await sourceBytes('https://gitlab.com/a/b',undefined,undefined,answer({statistics:{repository_size:42}})),42) + assert.equal(await sourceBytes('https://github.com/a/b',undefined,undefined,answer({},false)),null) + assert.equal(await sourceBytes('https://git.example.com/a/b.git',undefined,undefined,answer({size:1})),null) +}) diff --git a/src/server/import/dispatch.ts b/src/server/import/dispatch.ts index 615d7c9..04c8908 100644 --- a/src/server/import/dispatch.ts +++ b/src/server/import/dispatch.ts @@ -1,4 +1,29 @@ import type { ImportJob } from './job.js' +import { sourceURL,type Forge } from '../../engine/import/source.js' + +/** gild's import container has 1 GiB of memory and 4 GB of disk. A larger + * source imports from the owner's machine instead (`gild repo import`). */ +export const SERVER_IMPORT_LIMIT=500*1024*1024 + +/** The source's size in bytes as its forge reports it, or null when the forge + * does not say (plain git, a private GitLab project without statistics). */ +export async function sourceBytes(url:string,forge?:Forge,token?:string,fetcher:typeof fetch=fetch):Promise { + try { + const source=sourceURL(url,forge) + if(source.forge==='git')return null + const headers:Record={'user-agent':'gild-import',accept:'application/json'} + if(token)headers[source.forge==='github'?'authorization':'private-token']=source.forge==='github'?'Bearer '+token:token + const res=await fetcher(source.forge==='github'?source.api:source.api+'?statistics=true',{headers,signal:AbortSignal.timeout(5000)}) + if(!res.ok)return null + const body=await res.json() as {size?:number;statistics?:{repository_size?:number}} + // GitHub reports kilobytes; GitLab reports bytes. + if(source.forge==='github')return typeof body.size==='number'?body.size*1024:null + return typeof body.statistics?.repository_size==='number'?body.statistics.repository_size:null + } catch {return null} +} +export function tooLargeMessage(bytes:number,repository:string) { + return `Too large to import on gild's servers (${(bytes/1024**3).toFixed(1)} GB). Import it from your machine: gild repo resume ${repository}` +} /** gild-import (services/import): runs a claimed import in a Container. */ export interface Importer { start(server:string,job:NonNullable>>):Promise<{started:string}> } diff --git a/src/server/import/do.ts b/src/server/import/do.ts index f37581d..58a2a19 100644 --- a/src/server/import/do.ts +++ b/src/server/import/do.ts @@ -46,6 +46,11 @@ export class RepoImport extends DurableObject<{DB:D1Database;ACTIONS_SECRET_KEY? await this.ctx.storage.put('dispatches',tries+1) await startImport(this.env,this.env.PUBLIC_ORIGIN,()=>this.claim()) } + /** Keep this import off the container: it waits for the CLI or an owner runner. */ + async local(message:string) { + await this.ctx.storage.put('dispatches',DISPATCHES) + await this.mutate(async()=>{const j=await this.ctx.storage.get('job');if(j?.state==='queued')await this.ctx.storage.put('job',{...j,progress:{...j.progress,message}})}) + } cutover() {return this.mutate(()=>this.job.cutover())} async alarm() { await this.mutate(()=>this.job.alarm()) diff --git a/src/server/import/service.ts b/src/server/import/service.ts index 0a82618..970b230 100644 --- a/src/server/import/service.ts +++ b/src/server/import/service.ts @@ -1,7 +1,7 @@ import { getCloudflareContext } from '../context.js' import { store } from '../backend.js' import type { RepoImport } from './do.js' -export type ImportStub={ [K in 'address'|'create'|'status'|'claim'|'authorize'|'heartbeat'|'beginBatch'|'endBatch'|'finish'|'retry'|'cutover']: (...args:Parameters)=>Promise>> } +export type ImportStub={ [K in 'address'|'create'|'status'|'claim'|'authorize'|'heartbeat'|'beginBatch'|'endBatch'|'finish'|'retry'|'cutover'|'local']: (...args:Parameters)=>Promise>> } export async function importJob(owner:string,repo:string):Promise { const {env}=await getCloudflareContext(),ns=(env as unknown as {REPO_IMPORT:{idFromName(s:string):unknown;get(id:unknown):ImportStub}}).REPO_IMPORT const state=await (await store()).settings.state(`${owner}/${repo}`)