From 0c6344c71f2fe8d53a2bb434524c3ba05fdc3d16 Mon Sep 17 00:00:00 2001 From: Sami Fouad Date: Wed, 14 Oct 2026 20:56:17 -0600 Subject: [PATCH] Imports too large for the container are marked local and point to gild clone; importing the same source again picks up the waiting job. 0.7.12 --- package-lock.json | 4 ++-- package.json | 2 +- src/engine/api/import-contract.ts | 2 +- src/server/api/imports.ts | 4 ++++ src/server/import/dispatch.test.ts | 2 +- src/server/import/dispatch.ts | 2 +- src/server/import/do.ts | 2 +- src/server/import/job.ts | 4 ++-- 8 files changed, 13 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index c03fd43..34267d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gildforge/kit", - "version": "0.7.11", + "version": "0.7.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@gildforge/kit", - "version": "0.7.11", + "version": "0.7.12", "license": "Apache-2.0", "dependencies": { "diff": "8", diff --git a/package.json b/package.json index fc15dc7..3dfda92 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gildforge/kit", - "version": "0.7.11", + "version": "0.7.12", "description": "The shared engine, views and server behind gild: the git forge for agents on Cloudflare.", "license": "Apache-2.0", "type": "module", diff --git a/src/engine/api/import-contract.ts b/src/engine/api/import-contract.ts index d8a9883..988d079 100644 --- a/src/engine/api/import-contract.ts +++ b/src/engine/api/import-contract.ts @@ -15,7 +15,7 @@ export type ImportRecord = z.infer export const importInput = z.object({ url:z.string().min(1).max(2048), name:z.string().regex(/^[a-z0-9][a-z0-9._-]{0,62}$/i).optional(), private:z.boolean().optional(), mirror:z.boolean().default(false), source_token:z.string().max(16384).optional(), forge:z.enum(['github','gitlab','git']).optional() }) export const importProgress = z.object({ phase:z.enum(['git','metadata','workflows']), completed:z.number().int().nonnegative(), message:z.string().max(200), checkpoint:z.string().max(2048).optional(),default_branch:z.string().max(300).optional(),internal_prefix:z.string().regex(/^import_*\/pr$/).optional() }) export const importSummary=z.object({branches:z.number().int().nonnegative(),tags:z.number().int().nonnegative(),commits:z.number().int().nonnegative(),records:z.number().int().nonnegative(),elapsed_ms:z.number().nonnegative()}) -export const importStatus = z.object({ repository:z.string(), source:z.string(), mirror:z.boolean(), state:z.enum(['queued','running','complete','mirroring','failed','cutover']), progress:importProgress, warnings:z.array(z.string()), error:z.string().nullable(), updated_at:z.string(), next_sync:z.string().nullable(),summary:importSummary.optional() }) +export const importStatus = z.object({ repository:z.string(), source:z.string(), mirror:z.boolean(), state:z.enum(['queued','running','complete','mirroring','failed','cutover']), progress:importProgress, warnings:z.array(z.string()), error:z.string().nullable(), updated_at:z.string(), next_sync:z.string().nullable(),summary:importSummary.optional(),/** Too large for gild's servers: `gild clone` imports it locally. */local:z.boolean().optional() }) export const importAssignment = z.object({ repository:z.string(), source:z.string(), forge:z.enum(['github','gitlab','git']), token:z.string(), source_token:z.string().optional(), mirror:z.boolean(), checkpoint:z.string() }) export const importCredentials = z.object({ url:z.string(), token:z.string() }) export const importBatch = z.object({ records:z.array(importRecord).max(40), progress:importProgress }) diff --git a/src/server/api/imports.ts b/src/server/api/imports.ts index d0a8b57..287cd97 100644 --- a/src/server/api/imports.ts +++ b/src/server/api/imports.ts @@ -39,6 +39,10 @@ export async function apiImports(c:Context):Promise { if(b.source_token && new URL(source.url).protocol==='http:') throw new ApiError(422,'Use HTTPS or SSH with a source token') // Validate encryption before provisioning anything. Plaintext never enters durable storage. if(b.source_token) await encrypt(env as {ACTIONS_SECRET_KEY?:string},`import:${owner}/${name}`,b.source_token) + // The same source already waiting for a local run (too large for the + // container): `gild repo import` picks that job up instead of failing. + const prior=await (async()=>{try {return await (await importJob(owner,name)).status()} catch {return null}})() + if(prior?.state==='queued' && prior.source===source.url) return {data:prior} const isPrivate=destinationPrivate(b) await createRepository(c.principal.name,owner,name,undefined,isPrivate,false) const job=await importJob(owner,name) diff --git a/src/server/import/dispatch.test.ts b/src/server/import/dispatch.test.ts index 2a4ce67..81a496a 100644 --- a/src/server/import/dispatch.test.ts +++ b/src/server/import/dispatch.test.ts @@ -12,7 +12,7 @@ test('a large GitHub source is measured from its API size in kilobytes', async ( assert.ok(bytes!>SERVER_IMPORT_LIMIT) assert.equal(seen[0].url,'https://api.github.com/repos/torvalds/linux') assert.equal(seen[0].headers.authorization,'Bearer t0k') - assert.match(tooLargeMessage(bytes!,'sami/linux'),/4\.8 GB.*gild repo resume sami\/linux/) + assert.match(tooLargeMessage(bytes!,'sami/linux'),/4\.8 GB.*gild clone sami\/linux$/) }) test('GitLab statistics are bytes; plain git and failures are unknown', async () => { assert.equal(await sourceBytes('https://gitlab.com/a/b',undefined,undefined,answer({statistics:{repository_size:42}})),42) diff --git a/src/server/import/dispatch.ts b/src/server/import/dispatch.ts index 04c8908..cdc368d 100644 --- a/src/server/import/dispatch.ts +++ b/src/server/import/dispatch.ts @@ -22,7 +22,7 @@ export async function sourceBytes(url:string,forge?:Forge,token?:string,fetcher: } catch {return null} } export function tooLargeMessage(bytes:number,repository:string) { - return `Too large to import on gild's servers (${(bytes/1024**3).toFixed(1)} GB). Import it from your machine: gild repo resume ${repository}` + return `Too large to import on gild's servers (${(bytes/1024**3).toFixed(1)} GB). Import it from your machine: gild clone ${repository}` } /** gild-import (services/import): runs a claimed import in a Container. */ export interface Importer { start(server:string,job:NonNullable>>):Promise<{started:string}> } diff --git a/src/server/import/do.ts b/src/server/import/do.ts index 58a2a19..acb7579 100644 --- a/src/server/import/do.ts +++ b/src/server/import/do.ts @@ -49,7 +49,7 @@ export class RepoImport extends DurableObject<{DB:D1Database;ACTIONS_SECRET_KEY? /** Keep this import off the container: it waits for the CLI or an owner runner. */ async local(message:string) { await this.ctx.storage.put('dispatches',DISPATCHES) - await this.mutate(async()=>{const j=await this.ctx.storage.get('job');if(j?.state==='queued')await this.ctx.storage.put('job',{...j,progress:{...j.progress,message}})}) + await this.mutate(async()=>{const j=await this.ctx.storage.get('job');if(j?.state==='queued')await this.ctx.storage.put('job',{...j,local:true,progress:{...j.progress,message}})}) } cutover() {return this.mutate(()=>this.job.cutover())} async alarm() { diff --git a/src/server/import/job.ts b/src/server/import/job.ts index d405884..82f4b5d 100644 --- a/src/server/import/job.ts +++ b/src/server/import/job.ts @@ -9,7 +9,7 @@ export interface Job { state:'queued'|'running'|'complete'|'mirroring'|'failed'|'cutover'; progress:Progress; warnings:string[]; error:string|null; updated:number; next:number|null; cipher:string|null; - runner?:{owner:string;id:string;lease:string}; leaseHash:string|null; expires:number; checkpoint:string; busy:boolean; + runner?:{owner:string;id:string;lease:string}; leaseHash:string|null; expires:number; checkpoint:string; busy:boolean; local?:boolean; } export const hash=async(s:string)=>Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(s))),b=>b.toString(16).padStart(2,'0')).join('') export class ImportJob { @@ -29,7 +29,7 @@ export class ImportJob { } async status() { const j=await this.read() - return {repository:j.repository,source:j.source,mirror:j.mirror,state:j.state,progress:j.progress,warnings:j.warnings,error:j.error,updated_at:new Date(j.updated).toISOString(),next_sync:j.next?new Date(j.next).toISOString():null,summary:j.summary} + return {repository:j.repository,source:j.source,mirror:j.mirror,state:j.state,progress:j.progress,warnings:j.warnings,error:j.error,updated_at:new Date(j.updated).toISOString(),next_sync:j.next?new Date(j.next).toISOString():null,summary:j.summary,...(j.local?{local:true}:{})} } async claim(runner?:Job["runner"]) { const j=await this.read()