diff --git a/.changeset/integration-manifest-limit.md b/.changeset/integration-manifest-limit.md new file mode 100644 index 000000000..ccb369467 --- /dev/null +++ b/.changeset/integration-manifest-limit.md @@ -0,0 +1,5 @@ +--- +"@openmapx/extension-cli": patch +--- + +`openmapx-ext package` refuses a `manifest.json` larger than 128 KiB, the most the OpenMapX operations agent reads. An artifact with a larger manifest could not be installed. diff --git a/docs/docs/administration/community-extensions.md b/docs/docs/administration/community-extensions.md index d5599f2e7..b611e562d 100644 --- a/docs/docs/administration/community-extensions.md +++ b/docs/docs/administration/community-extensions.md @@ -201,6 +201,18 @@ entry points are also rejected; executable behavior must move into an isolated service. These reduce the blast radius of a malformed archive. They cannot establish that the author's code is safe. +The operations agent reads the configuration schema of every installed +integration, and loads one only from files its own account owns and nobody +else can write. A single integration it refuses stops it from authorizing any +configuration change. The installer therefore gives installed files fixed +modes (`0755` for directories, `0644` for files), whatever the archive or the +umask declares. It also refuses an integration the agent would not load: + +- a `manifest.json` that is a symbolic link, hard-linked to another file, + larger than 128 KiB or not UTF-8; +- a configuration schema the agent does not accept; +- an id longer than 64 characters. + ### Preview, admission and failures Extension previews have an independent limit of one request per 30 seconds per diff --git a/docs/docs/developer/building-an-external-extension.md b/docs/docs/developer/building-an-external-extension.md index a5ca7de10..67d4bc784 100644 --- a/docs/docs/developer/building-an-external-extension.md +++ b/docs/docs/developer/building-an-external-extension.md @@ -272,7 +272,7 @@ cannot reach a release: | Included | Limit | | ------------------------------------------------------------------ | ----------------------- | -| `manifest.json` (after schema validation) | : | +| `manifest.json` (after schema validation) | 128 KiB | | `openmapx-artifact.json` (generated) | : | | `dist/licenses.json` (generated when dependency notices are found) | : | | `strings/.json` matching `[a-z0-9-]{2,35}` | 100 files, 256 KiB each | diff --git a/packages/cli/__tests__/cli-integrations.test.ts b/packages/cli/__tests__/cli-integrations.test.ts index ac6b84744..3522ca6e0 100644 --- a/packages/cli/__tests__/cli-integrations.test.ts +++ b/packages/cli/__tests__/cli-integrations.test.ts @@ -1,7 +1,9 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { + chmodSync, existsSync, + linkSync, lstatSync, mkdirSync, mkdtempSync, @@ -15,6 +17,7 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { gzipSync } from "node:zlib"; +import { services } from "@openmapx/core/server"; import { backupInstalledIntegration, packageIntegration, @@ -62,7 +65,8 @@ function writeTarGz(sourceDir: string, artifactPath: string): void { interface TarFixtureEntry { name: string; - type?: "0" | "2" | "7" | "g" | "x"; + type?: "0" | "1" | "2" | "5" | "7" | "g" | "x"; + mode?: number; data?: Buffer | string; linkName?: string; claimedSize?: number; @@ -79,7 +83,7 @@ function tarFixtureEntry(entry: TarFixtureEntry): Buffer { const claimedSize = entry.claimedSize ?? payload.length; const header = Buffer.alloc(512); header.write(entry.name, 0, 100, "utf-8"); - writeTarOctal(header, 100, 8, 0o644); + writeTarOctal(header, 100, 8, entry.mode ?? 0o644); writeTarOctal(header, 108, 8, 0); writeTarOctal(header, 116, 8, 0); writeTarOctal(header, 124, 12, claimedSize); @@ -923,6 +927,192 @@ describe("packageIntegration", () => { }); }); +describe("installIntegration (what the operations agent loads)", () => { + const schemaManifest = (id: string, extra: Record = {}) => + JSON.stringify({ + ...baseManifest, + id, + configSchema: { type: "object", properties: { region: { enum: ["eu", "us"] } } }, + ...extra, + }); + const install = (source: string, sourceKind: "artifact" | "source" = "artifact") => + installIntegration({ source, sourceKind, rootDir: tmp }); + /** The integration schemas the agent's loader returns, or its error. */ + const agentLoads = () => [...services.loadTrustedIntegrationSchemas(tmp).keys()]; + const mode = (path: string) => lstatSync(path).mode & 0o777; + + /** A Debian or Ubuntu login shell's umask while `run` runs. */ + async function underUmask002(run: () => Promise): Promise { + const previous = process.umask(0o002); + try { + return await run(); + } finally { + process.umask(previous); + } + } + + async function installKept(): Promise { + const artifact = join(tmp, "kept.tar.gz"); + writeTarFixture(artifact, [{ name: "manifest.json", data: schemaManifest("kept") }]); + await install(artifact); + } + + it("installs an archive declaring 0666 and 0664 modes with the modes the agent loads", async () => { + const artifact = join(tmp, "loose-modes.tar.gz"); + writeTarFixture(artifact, [ + { name: "assets/", type: "5", mode: 0o777 }, + { name: "manifest.json", data: schemaManifest("loose-modes"), mode: 0o666 }, + { name: "strings/en.json", data: "{}", mode: 0o664 }, + { name: "assets/data.json", data: "{}", mode: 0o000 }, + ]); + await underUmask002(() => install(artifact)); + + const installed = join(tmp, "custom_integrations", "loose-modes"); + expect(mode(join(tmp, "custom_integrations"))).toBe(0o755); + expect(mode(installed)).toBe(0o755); + expect(mode(join(installed, "manifest.json"))).toBe(0o644); + expect(mode(join(installed, "strings"))).toBe(0o755); + expect(mode(join(installed, "strings", "en.json"))).toBe(0o644); + expect(mode(join(installed, "assets"))).toBe(0o755); + expect(mode(join(installed, "assets", "data.json"))).toBe(0o644); + expect(agentLoads()).toEqual(["loose-modes"]); + }); + + it("installs a local source written under umask 002 with the modes the agent loads", async () => { + const source = join(tmp, "local-loose"); + mkdirSync(join(source, "strings"), { recursive: true }); + writeFileSync(join(source, "manifest.json"), schemaManifest("local-loose")); + writeFileSync(join(source, "strings", "en.json"), "{}"); + chmodSync(join(source, "manifest.json"), 0o664); + chmodSync(join(source, "strings"), 0o775); + await underUmask002(() => install(source, "source")); + + const installed = join(tmp, "custom_integrations", "local-loose"); + expect(mode(installed)).toBe(0o755); + expect(mode(join(installed, "manifest.json"))).toBe(0o644); + expect(mode(join(installed, "strings"))).toBe(0o755); + expect(agentLoads()).toEqual(["local-loose"]); + }); + + it("refuses a hard-linked manifest.json and keeps the installed integrations loadable", async () => { + await installKept(); + const linkedTo = join(tmp, "manifest-linked-to.tar.gz"); + writeTarFixture(linkedTo, [ + { name: "copy.json", data: schemaManifest("linked-to") }, + { name: "manifest.json", type: "1", linkName: "copy.json" }, + ]); + await expectArtifactRejection(linkedTo, "linked-to", /must not be hard-linked/); + const linkedFrom = join(tmp, "manifest-linked-from.tar.gz"); + writeTarFixture(linkedFrom, [ + { name: "manifest.json", data: schemaManifest("linked-from") }, + { name: "alias.json", type: "1", linkName: "manifest.json" }, + ]); + await expectArtifactRejection(linkedFrom, "linked-from", /must not be hard-linked/); + expect(agentLoads()).toEqual(["kept"]); + }); + + it("refuses a symlinked manifest.json from an archive or a local source", async () => { + await installKept(); + const artifact = join(tmp, "manifest-symlink.tar.gz"); + writeTarFixture(artifact, [ + { name: "real.json", data: schemaManifest("manifest-symlink") }, + { name: "manifest.json", type: "2", linkName: "real.json" }, + ]); + await expectArtifactRejection(artifact, "manifest-symlink", /not a symbolic link/); + + const source = join(tmp, "local-symlink"); + mkdirSync(source); + writeFileSync(join(source, "real.json"), schemaManifest("local-symlink")); + symlinkSync("real.json", join(source, "manifest.json")); + await expect(install(source, "source")).rejects.toThrow(/not a symbolic link/); + expect(existsSync(join(tmp, "custom_integrations", "local-symlink"))).toBe(false); + expect(agentLoads()).toEqual(["kept"]); + }); + + it("installs a local source's hard-linked manifest.json as a file of its own", async () => { + // A local source is copied file by file, so no link survives the copy. + const source = join(tmp, "local-hardlink"); + mkdirSync(source); + writeFileSync(join(source, "manifest.json"), schemaManifest("local-hardlink")); + linkSync(join(source, "manifest.json"), join(source, "alias.json")); + await install(source, "source"); + const installed = join(tmp, "custom_integrations", "local-hardlink", "manifest.json"); + expect(lstatSync(installed).nlink).toBe(1); + expect(agentLoads()).toEqual(["local-hardlink"]); + }); + + /** A manifest whose description holds a byte that is not UTF-8. */ + function invalidUtf8Manifest(id: string): Buffer { + const [head, tail] = schemaManifest(id, { description: "X" }).split('"X"'); + return Buffer.concat([Buffer.from(`${head}"`), Buffer.from([0xff]), Buffer.from(`"${tail}`)]); + } + + it("refuses manifests the agent's loader would reject", async () => { + await installKept(); + const cases: Array<[string, Buffer | string, RegExp]> = [ + [ + "oversized", + `${schemaManifest("oversized")}${" ".repeat(services.MAX_INTEGRATION_MANIFEST_BYTES)}`, + /exceeds the 128 KiB limit/, + ], + [ + "unaccepted-schema", + schemaManifest("unaccepted-schema", { + configSchema: { + type: "object", + properties: { region: { type: "string", examples: ["eu"] } }, + }, + }), + /configSchema uses a construct/, + ], + ["invalid-utf8", invalidUtf8Manifest("invalid-utf8"), /not valid UTF-8/], + [`a${"b".repeat(64)}`, schemaManifest(`a${"b".repeat(64)}`), /longer than the 64 characters/], + ]; + for (const [id, data, message] of cases) { + const artifact = join(tmp, "refused.tar.gz"); + writeTarFixture(artifact, [{ name: "manifest.json", data }]); + await expectArtifactRejection(artifact, id, message); + } + expect(agentLoads()).toEqual(["kept"]); + }); + + it("takes over a group-writable custom_integrations it owns", async () => { + mkdirSync(join(tmp, "custom_integrations")); + chmodSync(join(tmp, "custom_integrations"), 0o775); + await installKept(); + expect(mode(join(tmp, "custom_integrations"))).toBe(0o755); + expect(agentLoads()).toEqual(["kept"]); + }); + + it("restores a rollback backup with the modes the agent loads", async () => { + await installKept(); + const target = join(tmp, "custom_integrations", "kept"); + const backup = backupInstalledIntegration(tmp, "kept") as NonNullable< + ReturnType + >; + chmodSync(join(backup.backupDirectory, "manifest.json"), 0o666); + chmodSync(backup.backupDirectory, 0o777); + restoreInstalledIntegration(tmp, backup); + expect(mode(target)).toBe(0o755); + expect(mode(join(target, "manifest.json"))).toBe(0o644); + expect(agentLoads()).toEqual(["kept"]); + }); +}); + +describe("packageIntegration (what the operations agent loads)", () => { + it("refuses a manifest larger than the agent reads", async () => { + const source = join(tmp, "oversized-source"); + mkdirSync(source); + writeFileSync( + join(source, "manifest.json"), + `${JSON.stringify({ ...baseManifest, id: "oversized-source" })}${" ".repeat(services.MAX_INTEGRATION_MANIFEST_BYTES)}`, + ); + await expect( + packageIntegration({ rootDir: tmp, source, outFile: join(tmp, "out.tar.gz") }), + ).rejects.toThrow(/manifest\.json exceeds its 131072-byte limit/); + }); +}); + describe("integration rollback backups", () => { it("restores the exact pre-update files", () => { writeIntegration("custom_integrations", "weather-radar", { diff --git a/packages/core/package.json b/packages/core/package.json index 2710aba96..2276b2dbc 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -122,6 +122,14 @@ "types": "./src/services/secret-key.ts", "default": "./src/services/secret-key.ts" }, + "./services/configuration-schema": { + "types": "./src/services/configuration-schema.ts", + "default": "./src/services/configuration-schema.ts" + }, + "./services/trusted-integration-schemas": { + "types": "./src/services/trusted-integration-schemas.ts", + "default": "./src/services/trusted-integration-schemas.ts" + }, "./utils/brandFilter": { "types": "./src/utils/brandFilter.ts", "default": "./src/utils/brandFilter.ts" diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 647b39891..376aff61a 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -220,7 +220,9 @@ export { type TrustedAuthorityReadHooks, } from "./trusted-authority-filesystem"; export { + assertTrustedIntegrationDirectory, type LoadTrustedIntegrationSchemasOptions, loadTrustedIntegrationSchemas, + MAX_INTEGRATION_MANIFEST_BYTES, } from "./trusted-integration-schemas"; export * from "./types"; diff --git a/packages/core/src/services/trusted-integration-schemas.ts b/packages/core/src/services/trusted-integration-schemas.ts index cf80d76bb..44bbcc086 100644 --- a/packages/core/src/services/trusted-integration-schemas.ts +++ b/packages/core/src/services/trusted-integration-schemas.ts @@ -14,7 +14,8 @@ import { const FAILED = "Trusted integration registry rejected"; const MAX_MANIFESTS = 256; -const MAX_MANIFEST_BYTES = 128 * 1024; +/** The largest integration manifest the operations agent reads. */ +export const MAX_INTEGRATION_MANIFEST_BYTES = 128 * 1024; const MAX_TOTAL_BYTES = 4 * 1024 * 1024; const ID = /^[a-z0-9][a-z0-9-]{0,63}$/; @@ -34,6 +35,36 @@ function parseSchema( return { id: record.id, schema: schema as Record }; } +function readIntegrationManifest(filesystem: TrustedAuthorityFilesystem, directory: string) { + return filesystem.readManifest( + join(directory, "manifest.json"), + "integration", + MAX_INTEGRATION_MANIFEST_BYTES, + ); +} + +/** + * Throws unless the operations agent would load `directory` as the installed + * integration `id`: the directory and its `manifest.json` owned by `uid` and + * not group- or other-writable, the manifest a single-link regular file of at + * most 128 KiB, in UTF-8, declaring `id` and a configuration schema the agent + * accepts. The installer runs it on what it is about to install, so an + * installed integration cannot stop the agent from loading the others. + */ +export function assertTrustedIntegrationDirectory( + directory: string, + id: string, + uid: number = process.geteuid?.() ?? 0, +): void { + const filesystem = new TrustedAuthorityFilesystem(directory, uid); + try { + parseSchema(readIntegrationManifest(filesystem, directory), id); + filesystem.recheck(); + } finally { + filesystem.close(); + } +} + export interface LoadTrustedIntegrationSchemasOptions { /** * The built-in integration manifests. Defaults to the release's copies in @@ -52,8 +83,9 @@ export interface LoadTrustedIntegrationSchemasOptions { * Built-in manifests come from the release when bundled, so the checkout's * `integrations/` and its file permissions do not decide them. Installed * integrations under `custom_integrations/` are not part of any release: - * they keep the owner and mode checks, anchored at that directory, until - * their installer controls the modes it writes. + * they keep the owner and mode checks, anchored at that directory. The + * installer writes fixed modes and refuses anything these checks would + * reject (`assertTrustedIntegrationDirectory`). */ export function loadTrustedIntegrationSchemas( rootDir: string, @@ -76,7 +108,7 @@ export function loadTrustedIntegrationSchemas( checkoutBuiltInIntegrationManifests(rootDir); for (const id of builtIns.ids) { if (!ID.test(id)) throw new Error(); - add(id, readRegularFileBytes(builtIns.manifestPath(id), MAX_MANIFEST_BYTES)); + add(id, readRegularFileBytes(builtIns.manifestPath(id), MAX_INTEGRATION_MANIFEST_BYTES)); } const custom = join(rootDir, "custom_integrations"); @@ -99,11 +131,7 @@ export function loadTrustedIntegrationSchemas( filesystem.openDirectory(directory); let bytes: Uint8Array; try { - bytes = filesystem.readManifest( - join(directory, "manifest.json"), - "integration", - MAX_MANIFEST_BYTES, - ); + bytes = readIntegrationManifest(filesystem, directory); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; throw error; diff --git a/packages/integration-framework/src/installer.ts b/packages/integration-framework/src/installer.ts index a2273c9bf..fb187a13d 100644 --- a/packages/integration-framework/src/installer.ts +++ b/packages/integration-framework/src/installer.ts @@ -28,6 +28,11 @@ import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep import { gunzipSync } from "node:zlib"; import { scanLicenses } from "@openmapx/core/licenses"; import { assertAllowedGitUrl, gitShallowClone, safeDownload } from "@openmapx/core/server"; +import { assertTrustedConfigurationSchema } from "@openmapx/core/services/configuration-schema"; +import { + assertTrustedIntegrationDirectory, + MAX_INTEGRATION_MANIFEST_BYTES, +} from "@openmapx/core/services/trusted-integration-schemas"; import { type ArtifactMetadata, validateArtifactMetadata as validatePortableArtifactMetadata, @@ -48,6 +53,102 @@ function customDir(rootDir: string): string { return join(rootDir, "custom_integrations"); } +// The operations agent loads installed integrations only from directories and +// manifests its own account owns and nobody else can write; one it would +// reject stops it from authorizing any configuration change. Installed files +// therefore get these modes whatever the source, the archive or the umask +// declared. +const INSTALLED_DIRECTORY_MODE = 0o755; +const INSTALLED_FILE_MODE = 0o644; +const MAX_AGENT_INTEGRATION_ID_LENGTH = 64; + +/** + * Create `custom_integrations/`, or take it over as it is, in the state the + * agent trusts: a directory owned by the installing account that neither its + * group nor others can write. + */ +function prepareCustomDir(rootDir: string): void { + const directory = customDir(rootDir); + mkdirSync(directory, { recursive: true, mode: INSTALLED_DIRECTORY_MODE }); + const stats = lstatSync(directory); + if (!stats.isDirectory()) { + throw new Error(`${directory} must be a directory, not a symbolic link or a file`); + } + const uid = process.geteuid?.(); + if (uid !== undefined && stats.uid !== uid) { + throw new Error( + `${directory} is owned by uid ${stats.uid}, not by the installing account (uid ${uid}). ` + + "The operations agent only loads integrations from a directory its own account owns.", + ); + } + if ((stats.mode & 0o022) !== 0) chmodSync(directory, stats.mode & 0o755); +} + +/** Give a staged tree the installed modes. Symbolic links are left as they are. */ +function applyInstalledModes(directory: string): void { + chmodSync(directory, INSTALLED_DIRECTORY_MODE); + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) applyInstalledModes(path); + else if (entry.isFile()) chmodSync(path, INSTALLED_FILE_MODE); + } +} + +/** + * Refuse a `manifest.json` the agent would not read: a symbolic link, a file + * hard-linked to another, or one larger than the agent reads. Checked before + * the manifest is parsed, since parsing would follow a link. + */ +function assertManifestFile(directory: string): void { + let stats: ReturnType; + try { + stats = lstatSync(join(directory, "manifest.json")); + } catch { + return; // Missing: reported when the manifest is read. + } + if (stats.isSymbolicLink()) { + throw new Error("manifest.json must be a regular file, not a symbolic link"); + } + if (!stats.isFile()) throw new Error("manifest.json must be a regular file"); + if (stats.nlink !== 1) { + throw new Error("manifest.json must not be hard-linked to another file"); + } + if (stats.size > MAX_INTEGRATION_MANIFEST_BYTES) { + throw new Error(`manifest.json exceeds the ${MAX_INTEGRATION_MANIFEST_BYTES / 1024} KiB limit`); + } +} + +/** + * The checks the operations agent's loader applies, run on the staged tree + * after its modes are set: anything it would reject is refused here, before + * it replaces an installed integration. + */ +function assertAgentLoads(directory: string, id: string): void { + if (id.length > MAX_AGENT_INTEGRATION_ID_LENGTH) { + throw new Error( + `Integration id "${id}" is longer than the ${MAX_AGENT_INTEGRATION_ID_LENGTH} characters the operations agent loads`, + ); + } + try { + new TextDecoder("utf-8", { fatal: true }).decode( + readFileSync(join(directory, "manifest.json")), + ); + } catch { + throw new Error("manifest.json is not valid UTF-8"); + } + const manifest = readManifest(directory); + try { + assertTrustedConfigurationSchema(manifest?.configSchema as Record | undefined); + } catch { + throw new Error("configSchema uses a construct the operations agent does not accept"); + } + try { + assertTrustedIntegrationDirectory(directory, id); + } catch { + throw new Error("The operations agent would not load the installed files"); + } +} + const STAGING_DIR_NAME = ".staging"; const DEFAULT_MAX_ARTIFACT_BYTES = 200 * 1024 * 1024; const MAX_LAYER_SELECTOR_PREVIEW_BYTES = 64 * 1024; @@ -297,7 +398,7 @@ function resolveGitUrl(source: string): string | null { } export async function installIntegration(opts: InstallOptions): Promise { - mkdirSync(customDir(opts.rootDir), { recursive: true }); + prepareCustomDir(opts.rootDir); const allowLocal = opts.allowLocalSources ?? true; const sourceKind = opts.sourceKind ?? "source"; @@ -352,6 +453,7 @@ export async function installIntegration(opts: InstallOptions): Promise data.length - payloadOffset) { @@ -578,7 +681,6 @@ function validateTarDescriptors(data: Buffer, destDir: string): TarDescriptor[] path: safePath, payloadOffset, payloadLength: size, - mode, }); continue; } @@ -590,7 +692,6 @@ function validateTarDescriptors(data: Buffer, destDir: string): TarDescriptor[] linkTarget: linkTarget.raw, payloadOffset, payloadLength: size, - mode, }); continue; } @@ -602,7 +703,6 @@ function validateTarDescriptors(data: Buffer, destDir: string): TarDescriptor[] linkTarget, payloadOffset, payloadLength: size, - mode, }); continue; } @@ -615,7 +715,6 @@ function validateTarDescriptors(data: Buffer, destDir: string): TarDescriptor[] path: safePath, payloadOffset, payloadLength: size, - mode, }); } if (!reachedEndBlock && offset !== data.length) { @@ -664,14 +763,8 @@ function materializeTarDescriptors( writeFileSync( target, data.subarray(descriptor.payloadOffset, descriptor.payloadOffset + descriptor.payloadLength), + { mode: INSTALLED_FILE_MODE }, ); - if (descriptor.mode > 0) { - try { - chmodSync(target, descriptor.mode & 0o777); - } catch { - // Non-fatal on filesystems that do not support chmod. - } - } } } @@ -924,6 +1017,7 @@ export function restoreInstalledIntegration( const restoreStage = createStagePath(rootDir, `restore-${backup.id}`); try { cpSync(backupDirectory, restoreStage, { recursive: true, errorOnExist: true, force: false }); + applyInstalledModes(restoreStage); if (existsSync(target)) rmSync(target, { recursive: true, force: true }); renameSync(restoreStage, target); } catch (error) { @@ -1041,7 +1135,14 @@ function collectArtifactFiles( generated: { metadata: string; licenses: string | null }, ): StagedFile[] { const files: StagedFile[] = []; - const manifestFile = collectStagedFile(directory, "manifest.json", MAX_LEGAL_BYTES, true); + // The operations agent reads no larger manifest; an artifact with one would + // install nowhere. + const manifestFile = collectStagedFile( + directory, + "manifest.json", + MAX_INTEGRATION_MANIFEST_BYTES, + true, + ); if (manifestFile) files.push(manifestFile); files.push({ diff --git a/release-notes/release-community-and-integration-trust.md b/release-notes/release-community-and-integration-trust.md index 8e6bda0a0..1186f21ea 100644 --- a/release-notes/release-community-and-integration-trust.md +++ b/release-notes/release-community-and-integration-trust.md @@ -26,7 +26,10 @@ clone and pull with umask `022`. `infra/docker`, which holds the current configuration pointer, when its own user owns it, instead of refusing to render. - Installed integrations under `custom_integrations/` keep their owner and mode - checks until their installer controls the file modes it writes. + checks. Their installer gives installed files fixed modes and refuses an + integration those checks, or the agent's schema check, would reject, so one + installed integration can no longer stop the agent from authorizing + configuration changes. It also fixes trusted configuration requests (applying settings, the service selection or secrets from the admin panel): the agent bound each request to one @@ -44,3 +47,9 @@ None. If the agent refuses to start with "Trusted configuration authority rejected" because of a repository installed by an earlier release, delete that repository's directory and its `.repository-.json` under `services/.community/` and install the extension again. + +If it rejects its authority with "Trusted integration registry rejected" +because of an integration installed earlier, for example with +`pnpm openmapx integrations install` under a umask of `002`, install that +integration again, from the admin panel or with the same command. The installer +replaces its files with the modes the agent loads. diff --git a/scripts/ops-agent-checkout-trust.test.ts b/scripts/ops-agent-checkout-trust.test.ts index 6cdd0a91d..7500c06d7 100644 --- a/scripts/ops-agent-checkout-trust.test.ts +++ b/scripts/ops-agent-checkout-trust.test.ts @@ -1,10 +1,19 @@ import { type ChildProcess, execFile, spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import { join, relative, resolve } from "node:path"; import { promisify } from "node:util"; +import { gzipSync } from "node:zlib"; import { OpsClient } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -16,6 +25,7 @@ import { consumePublishedTrustedConfiguration, publishTrustedConfigurationSnapshot, } from "../apps/api/src/services/trusted-config-publisher"; +import { installIntegration } from "../packages/integration-framework/src/installer"; import { buildRuntimeBundle, RUNTIME_BUNDLES, @@ -112,6 +122,106 @@ async function freePort(): Promise { }); } +interface RunningAgent { + port: number; + output: () => string; + stop: () => void; +} + +/** Starts the bundled agent on a checkout and waits until it reports healthy. */ +async function startAgent(checkout: string): Promise { + const infra = join(checkout, "infra", "docker"); + const port = await freePort(); + const agent = spawn(process.execPath, [join(outdir, "agent.mjs")], { + cwd: checkout, + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + OPENMAPX_ROOT_DIR: checkout, + OPS_AGENT_API_TOKEN_FILE: join(infra, "secrets", "ops-agent-api-token"), + OPS_AGENT_DATA_MANAGER_TOKEN_FILE: join(infra, "secrets", "ops-agent-data-manager-token"), + OPS_TRUSTED_CONFIG_DIR: join(infra, "state", "ops-agent", "trusted-config"), + HOST: "127.0.0.1", + PORT: String(port), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + agents.push(agent); + let output = ""; + agent.stdout?.on("data", (chunk) => { + output += chunk; + }); + agent.stderr?.on("data", (chunk) => { + output += chunk; + }); + const deadline = Date.now() + 30_000; + for (;;) { + if (agent.exitCode !== null) throw new Error(`The agent exited: ${output}`); + const healthy = await fetch(`http://127.0.0.1:${port}/health`) + .then((response) => response.ok) + .catch(() => false); + if (healthy) break; + if (Date.now() > deadline) throw new Error(`The agent never became healthy: ${output}`); + await new Promise((resolveWait) => setTimeout(resolveWait, 100)); + } + return { port, output: () => output, stop: () => agent.kill("SIGTERM") }; +} + +/** + * Publishes a trusted-configuration request as app-api does, asks the agent + * to render it, and expects a committed generation. + */ +async function expectTrustedConfigurationCommitted( + checkout: string, + agent: RunningAgent, + name: string, + payload: Partial[0]["payload"]>, +): Promise { + const infra = join(checkout, "infra", "docker"); + const trustedConfig = join(infra, "state", "ops-agent", "trusted-config"); + const tokenFile = join(infra, "secrets", "ops-agent-api-token"); + const token = readFileSync(tokenFile, "utf8").trim(); + const directory = statSync(trustedConfig); + const operationKey = createDurableOpsKey("stack.render", name); + const before = services.currentConfigurationGeneration(infra); + const sealed = await publishTrustedConfigurationSnapshot({ + directory: trustedConfig, + token, + ownerUid: directory.uid, + ownerGid: directory.gid, + operationKey, + operationForRevision: (revisionId) => ({ + kind: "stack.render", + revisionId, + expectedCurrentRevisionId: before, + }), + payload: { + domain: `${name}.example.test`, + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + ...payload, + }, + }); + const client = new OpsClient({ baseUrl: `http://127.0.0.1:${agent.port}`, tokenFile }); + const consumed = await consumePublishedTrustedConfiguration(sealed, () => + executeTrustedConfigurationAndWait( + client, + sealed.operation as Parameters[1], + operationKey, + ), + ); + expect(consumed.submission.status, agent.output()).toBe("fulfilled"); + const result = consumed.submission.status === "fulfilled" ? consumed.submission.value : undefined; + expect(result, `${JSON.stringify(result)}\n${agent.output()}`).toMatchObject({ + outcome: "committed", + }); + expect(services.currentConfigurationGeneration(infra)).not.toBe(before); + expect( + readFileSync(join(infra, ".trusted-config-current", "docker-compose.generated.yml"), "utf8"), + ).toContain(`${name}.example.test`); +} + describe("a checkout written under umask 002", () => { it("gets the bundled agent through startup and a trusted-configuration request", async () => { const checkout = await umask002Checkout(); @@ -127,82 +237,106 @@ describe("a checkout written under umask 002", () => { const rendered = await cli(checkout, "compose", "render"); expect(rendered.stdout, rendered.stderr).toContain("Rendered"); - const infra = join(checkout, "infra", "docker"); - const trustedConfig = join(infra, "state", "ops-agent", "trusted-config"); - const port = await freePort(); - const agent = spawn(process.execPath, [join(outdir, "agent.mjs")], { - cwd: checkout, - env: { - PATH: process.env.PATH, - HOME: process.env.HOME, - OPENMAPX_ROOT_DIR: checkout, - OPS_AGENT_API_TOKEN_FILE: join(infra, "secrets", "ops-agent-api-token"), - OPS_AGENT_DATA_MANAGER_TOKEN_FILE: join(infra, "secrets", "ops-agent-data-manager-token"), - OPS_TRUSTED_CONFIG_DIR: trustedConfig, - HOST: "127.0.0.1", - PORT: String(port), - }, - stdio: ["ignore", "pipe", "pipe"], - }); - agents.push(agent); - let output = ""; - agent.stdout?.on("data", (chunk) => { - output += chunk; - }); - agent.stderr?.on("data", (chunk) => { - output += chunk; - }); - const deadline = Date.now() + 30_000; - for (;;) { - if (agent.exitCode !== null) throw new Error(`The agent exited: ${output}`); - const healthy = await fetch(`http://127.0.0.1:${port}/health`) - .then((response) => response.ok) - .catch(() => false); - if (healthy) break; - if (Date.now() > deadline) throw new Error(`The agent never became healthy: ${output}`); - await new Promise((resolveWait) => setTimeout(resolveWait, 100)); - } + const agent = await startAgent(checkout); + await expectTrustedConfigurationCommitted(checkout, agent, "umask-002-checkout", {}); + agent.stop(); + }, 120_000); +}); - const tokenFile = join(infra, "secrets", "ops-agent-api-token"); - const token = readFileSync(tokenFile, "utf8").trim(); - const directory = statSync(trustedConfig); - const operationKey = createDurableOpsKey("stack.render", "umask-002-checkout"); - const before = services.currentConfigurationGeneration(infra); - const sealed = await publishTrustedConfigurationSnapshot({ - directory: trustedConfig, - token, - ownerUid: directory.uid, - ownerGid: directory.gid, - operationKey, - operationForRevision: (revisionId) => ({ - kind: "stack.render", - revisionId, - expectedCurrentRevisionId: before, - }), - payload: { - domain: "umask.example.test", - serviceConfigs: [], - integrationConfigs: [], - serviceSecrets: [], - }, - }); - const client = new OpsClient({ baseUrl: `http://127.0.0.1:${port}`, tokenFile }); - const consumed = await consumePublishedTrustedConfiguration(sealed, () => - executeTrustedConfigurationAndWait( - client, - sealed.operation as Parameters[1], - operationKey, - ), +/** A one-entry-per-file ustar archive; `mode` and links as an archive may declare them. */ +function tarGz( + entries: Array<{ + name: string; + data?: string; + mode?: number; + type?: "0" | "1" | "2"; + link?: string; + }>, +): Buffer { + const blocks = entries.flatMap((entry) => { + const payload = Buffer.from(entry.data ?? "", "utf8"); + const header = Buffer.alloc(512); + const octal = (offset: number, length: number, value: number) => + header.write(`${value.toString(8).padStart(length - 1, "0")}\0`, offset, length, "ascii"); + header.write(entry.name, 0, 100, "utf8"); + octal(100, 8, entry.mode ?? 0o644); + octal(108, 8, 0); + octal(116, 8, 0); + octal(124, 12, payload.length); + octal(136, 12, 0); + header.fill(0x20, 148, 156); + header.write(entry.type ?? "0", 156, 1, "ascii"); + if (entry.link) header.write(entry.link, 157, 100, "utf8"); + header.write("ustar\0", 257, 6, "ascii"); + header.write("00", 263, 2, "ascii"); + octal( + 148, + 8, + header.reduce((sum, byte) => sum + byte, 0), ); - expect(consumed.submission.status, output).toBe("fulfilled"); - const result = - consumed.submission.status === "fulfilled" ? consumed.submission.value : undefined; - expect(result, `${JSON.stringify(result)}\n${output}`).toMatchObject({ outcome: "committed" }); - expect(services.currentConfigurationGeneration(infra)).not.toBe(before); - expect( - readFileSync(join(infra, ".trusted-config-current", "docker-compose.generated.yml"), "utf8"), - ).toContain("umask.example.test"); - agent.kill("SIGTERM"); + return [header, payload, Buffer.alloc((512 - (payload.length % 512)) % 512)]; + }); + return gzipSync(Buffer.concat([...blocks, Buffer.alloc(1024)])); +} + +describe("an integration installed into a umask-002 checkout", () => { + it("installs or is refused without breaking trusted-configuration requests", async () => { + const checkout = await umask002Checkout(); + const manifest = (id: string) => + JSON.stringify({ + id, + name: id, + version: "1.0.0", + domains: ["map-overlay"], + quality: "community", + platform: "1.0", + configSchema: { type: "object", properties: { region: { enum: ["eu", "us"] } } }, + }); + const artifacts = join(checkout, "..", "artifacts"); + mkdirSync(artifacts); + const install = async (name: string, entries: Parameters[0]) => { + const source = join(artifacts, `${name}.tar.gz`); + writeFileSync(source, tarGz(entries)); + const previous = process.umask(0o002); + try { + return await installIntegration({ rootDir: checkout, source, sourceKind: "artifact" }); + } finally { + process.umask(previous); + } + }; + + // As app-api's Store installs them: an archive declaring loose modes, + // and two whose manifest.json the agent would refuse to read. + await expect( + install("loose-modes", [ + { name: "manifest.json", data: manifest("loose-modes"), mode: 0o666 }, + { name: "strings/en.json", data: "{}", mode: 0o664 }, + ]), + ).resolves.toMatchObject({ id: "loose-modes" }); + await expect( + install("hard-linked", [ + { name: "copy.json", data: manifest("hard-linked") }, + { name: "manifest.json", type: "1", link: "copy.json" }, + ]), + ).rejects.toThrow("must not be hard-linked"); + await expect( + install("symlinked", [ + { name: "real.json", data: manifest("symlinked") }, + { name: "manifest.json", type: "2", link: "real.json" }, + ]), + ).rejects.toThrow("not a symbolic link"); + expect(readdirSync(join(checkout, "custom_integrations")).sort()).toEqual([ + ".staging", + "loose-modes", + ]); + + const rendered = await cli(checkout, "compose", "render"); + expect(rendered.stdout, rendered.stderr).toContain("Rendered"); + const agent = await startAgent(checkout); + await expectTrustedConfigurationCommitted(checkout, agent, "installed-integration", { + integrationConfigs: [{ integrationId: "loose-modes", values: { region: "eu" } }], + }); + agent.stop(); }, 120_000); });