Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@gildforge/kit",
"version": "0.7.9",
"version": "0.7.10",
"description": "The shared engine, views and server behind gild: the git forge for agents on Cloudflare.",
"license": "Apache-2.0",
"type": "module",
Expand Down
18 changes: 16 additions & 2 deletions src/server/api/imports.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { getCloudflareContext } from '../context.js'
import { destinationPrivate } from '../../engine/import/fence.js'
import { sourceURL } from '../../engine/import/source.js'
import { importJob } from '../import/service.js'
import { startImport } from '../import/dispatch.js'
import { startImport,sourceBytes,SERVER_IMPORT_LIMIT,tooLargeMessage } from '../import/dispatch.js'
import { store } from '../backend.js'
import { mayWrite } from '../access.js'
import { ApiError } from '../../engine/api/errors.js'
Expand All @@ -22,6 +22,14 @@ export async function authorizeImport(c:Context) {
if(c.principal.name.includes('/') || !(await mayWrite(owner,c.principal.name))) throw new ApiError(403,'Repository owner or organization admin required')
if(c.principal.repos!==null && !c.params.repo) throw new ApiError(403,'Repository creation requires an unrestricted token')
}
/** False (and the job marked for the CLI) when the source is too large for
* the import container. An unknown size goes to the container as before. */
async function serverSized(job:Awaited<ReturnType<typeof importJob>>,repository:string,url:string,forge?:Parameters<typeof sourceBytes>[1],token?:string) {
const bytes=await sourceBytes(url,forge,token)
if(bytes===null||bytes<=SERVER_IMPORT_LIMIT) return true
await job.local(tooLargeMessage(bytes,repository))
return false
}
export async function apiImports(c:Context):Promise<Result> {
const b=c.body,id=c.route.id,st=await store(),{env}=await getCloudflareContext(),token=c.request.headers.get('authorization')?.replace(/^Bearer /i,'')??''
if(id==='importCreate'||id==='importOrgCreate') {
Expand All @@ -38,6 +46,7 @@ export async function apiImports(c:Context):Promise<Result> {
// Straight from the source into Artifacts, server-side (services/import).
// `x-gild-import-runner: local` keeps the job for the caller's own machine.
if(c.request.headers.get('x-gild-import-runner')==='local') return {data:created}
if(!await serverSized(job,`${owner}/${name}`,source.url,source.forge,b.source_token)) return {data:await job.status()}
return {data:await startImport(env as never,new URL(c.request.url).origin,()=>job.claim())?await job.status():created}
}
if(id==='importUserPoll'||id==='importOrgPoll') {
Expand Down Expand Up @@ -65,7 +74,9 @@ export async function apiImports(c:Context):Promise<Result> {
const {owner,repo}=c.params,job=await importJob(owner,repo)
if(id==='importStatus') return {data:await job.status()}
if(id==='importClaim') return {data:await job.claim()}
if(id==='importRetry') {const queued=await job.retry(b.source_token);return {data:await startImport(env as never,new URL(c.request.url).origin,()=>job.claim())?await job.status():queued}}
if(id==='importRetry') {const queued=await job.retry(b.source_token)
if(!await serverSized(job,`${owner}/${repo}`,queued.source,undefined,b.source_token)) return {data:await job.status()}
return {data:await startImport(env as never,new URL(c.request.url).origin,()=>job.claim())?await job.status():queued}}
if(id==='importCutover') {
const result=await job.cutover()
const {writeMeta,writeJson,DIR}=await import('../metastore.js')
Expand Down Expand Up @@ -103,6 +114,9 @@ export async function apiImports(c:Context):Promise<Result> {
// re-indexing take tens of seconds, longer than an importer waits for an
// answer, so they run after it (a failure is logged; the data stays).
if(!b.error) {
// Wake the index first: it is durable (a Durable Object alarm), while the
// waitUntil below may be cut off after ~30s and never reach its wake.
await scheduleIndex(owner,repo)
const work=(async()=>{
const { finishImportRefs } = await import('../import/finish.js')
await finishImportRefs(owner,repo,active.internalPrefix??'import/pr',{url:active.source,mirror:active.mirror,readOnly:false,warnings:b.warnings,summary:b.summary})
Expand Down
21 changes: 21 additions & 0 deletions src/server/import/dispatch.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { test } from 'node:test'
import assert from 'node:assert/strict'
import { sourceBytes,SERVER_IMPORT_LIMIT,tooLargeMessage } from '../../../dist/server/import/dispatch.js'

const answer=(body:unknown,ok=true)=>(async(url:string,init?:RequestInit)=>{seen.push({url:String(url),headers:init?.headers as Record<string,string>});return new Response(JSON.stringify(body),{status:ok?200:404})}) as typeof fetch
let seen:{url:string;headers:Record<string,string>}[]=[]

test('a large GitHub source is measured from its API size in kilobytes', async () => {
seen=[]
const bytes=await sourceBytes('https://github.com/torvalds/linux',undefined,'t0k',answer({size:5_000_000}))
assert.equal(bytes,5_000_000*1024)
assert.ok(bytes!>SERVER_IMPORT_LIMIT)
assert.equal(seen[0].url,'https://api.github.com/repos/torvalds/linux')
assert.equal(seen[0].headers.authorization,'Bearer t0k')
assert.match(tooLargeMessage(bytes!,'sami/linux'),/4\.8 GB.*gild repo resume sami\/linux/)
})
test('GitLab statistics are bytes; plain git and failures are unknown', async () => {
assert.equal(await sourceBytes('https://gitlab.com/a/b',undefined,undefined,answer({statistics:{repository_size:42}})),42)
assert.equal(await sourceBytes('https://github.com/a/b',undefined,undefined,answer({},false)),null)
assert.equal(await sourceBytes('https://git.example.com/a/b.git',undefined,undefined,answer({size:1})),null)
})
25 changes: 25 additions & 0 deletions src/server/import/dispatch.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,29 @@
import type { ImportJob } from './job.js'
import { sourceURL,type Forge } from '../../engine/import/source.js'

/** gild's import container has 1 GiB of memory and 4 GB of disk. A larger
* source imports from the owner's machine instead (`gild repo import`). */
export const SERVER_IMPORT_LIMIT=500*1024*1024

/** The source's size in bytes as its forge reports it, or null when the forge
* does not say (plain git, a private GitLab project without statistics). */
export async function sourceBytes(url:string,forge?:Forge,token?:string,fetcher:typeof fetch=fetch):Promise<number|null> {
try {
const source=sourceURL(url,forge)
if(source.forge==='git')return null
const headers:Record<string,string>={'user-agent':'gild-import',accept:'application/json'}
if(token)headers[source.forge==='github'?'authorization':'private-token']=source.forge==='github'?'Bearer '+token:token
const res=await fetcher(source.forge==='github'?source.api:source.api+'?statistics=true',{headers,signal:AbortSignal.timeout(5000)})
if(!res.ok)return null
const body=await res.json() as {size?:number;statistics?:{repository_size?:number}}
// GitHub reports kilobytes; GitLab reports bytes.
if(source.forge==='github')return typeof body.size==='number'?body.size*1024:null
return typeof body.statistics?.repository_size==='number'?body.statistics.repository_size:null
} catch {return null}
}
export function tooLargeMessage(bytes:number,repository:string) {
return `Too large to import on gild's servers (${(bytes/1024**3).toFixed(1)} GB). Import it from your machine: gild repo resume ${repository}`
}
/** gild-import (services/import): runs a claimed import in a Container. */
export interface Importer { start(server:string,job:NonNullable<Awaited<ReturnType<ImportJob['claim']>>>):Promise<{started:string}> }

Expand Down
5 changes: 5 additions & 0 deletions src/server/import/do.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ export class RepoImport extends DurableObject<{DB:D1Database;ACTIONS_SECRET_KEY?
await this.ctx.storage.put('dispatches',tries+1)
await startImport(this.env,this.env.PUBLIC_ORIGIN,()=>this.claim())
}
/** Keep this import off the container: it waits for the CLI or an owner runner. */
async local(message:string) {
await this.ctx.storage.put('dispatches',DISPATCHES)
await this.mutate(async()=>{const j=await this.ctx.storage.get<import('./job.js').Job>('job');if(j?.state==='queued')await this.ctx.storage.put('job',{...j,progress:{...j.progress,message}})})
}
cutover() {return this.mutate(()=>this.job.cutover())}
async alarm() {
await this.mutate(()=>this.job.alarm())
Expand Down
2 changes: 1 addition & 1 deletion src/server/import/service.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { getCloudflareContext } from '../context.js'
import { store } from '../backend.js'
import type { RepoImport } from './do.js'
export type ImportStub={ [K in 'address'|'create'|'status'|'claim'|'authorize'|'heartbeat'|'beginBatch'|'endBatch'|'finish'|'retry'|'cutover']: (...args:Parameters<RepoImport[K]>)=>Promise<Awaited<ReturnType<RepoImport[K]>>> }
export type ImportStub={ [K in 'address'|'create'|'status'|'claim'|'authorize'|'heartbeat'|'beginBatch'|'endBatch'|'finish'|'retry'|'cutover'|'local']: (...args:Parameters<RepoImport[K]>)=>Promise<Awaited<ReturnType<RepoImport[K]>>> }
export async function importJob(owner:string,repo:string):Promise<ImportStub> {
const {env}=await getCloudflareContext(),ns=(env as unknown as {REPO_IMPORT:{idFromName(s:string):unknown;get(id:unknown):ImportStub}}).REPO_IMPORT
const state=await (await store()).settings.state(`${owner}/${repo}`)
Expand Down
Loading