diff --git a/.github/workflows/helm-template-check.yml b/.github/workflows/helm-template-check.yml index bc5b0b1..9e02eda 100644 --- a/.github/workflows/helm-template-check.yml +++ b/.github/workflows/helm-template-check.yml @@ -113,6 +113,58 @@ jobs: --set imageCredentials.username=test \ --set imageCredentials.password=test \ --set gateway=null + # Read-only root filesystem: an emptyDir on each writable directory by + # default; the previous layout for ro=false, an image older than the first + # one that supports it, and --reuse-values from a chart before 0.2.5. + ai_ro() { + helm template test ./charts/ai \ + --set-file ctaiPropertiesFile=./charts/ai/ctai-configuration.properties.template \ + --set service.fqdn=https://example.codetogether.io \ + --set imageCredentials.username=test \ + --set imageCredentials.password=test \ + --show-only templates/deployment.yaml "$@" + } + ro_min="$(sed -n '/define "ctai.readOnlyRootFilesystemMinImage"/{n;p}' ./charts/ai/templates/_helpers.tpl)" + if ! [[ "$ro_min" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Set ctai.readOnlyRootFilesystemMinImage to the first release that runs read-only (got $ro_min)" >&2; exit 1 + fi + app_version="$(sed -n 's/^appVersion: "\(.*\)"$/\1/p' ./charts/ai/Chart.yaml)" + if [ "$(printf '%s\n%s\n' "$ro_min" "$app_version" | sort -V | head -1)" != "$ro_min" ]; then + echo "appVersion $app_version is older than ctai.readOnlyRootFilesystemMinImage $ro_min" >&2; exit 1 + fi + out="$(ai_ro)" + grep -q 'readOnlyRootFilesystem: true' <<<"$out" + [ "$(grep -c 'name: writable-' <<<"$out")" = 10 ] + for args in "--set securityContext.readOnlyRootFilesystem=false" \ + "--set image.tag=1.0.0" \ + "--set image.tag=1.0.0-hotfix" \ + "--set securityContext.readOnlyRootFilesystem=null --set writableDirectories=null"; do + # shellcheck disable=SC2086 + out="$(ai_ro $args)" + grep -q 'readOnlyRootFilesystem: false' <<<"$out" + if grep -q 'name: writable-' <<<"$out"; then echo "writable volumes with $args" >&2; exit 1; fi + done + # A customer volume instead of one writable directory, and the refusals. + out="$(ai_ro --set writableDirectories.logs.enabled=false \ + --set 'extraVolumes[0].name=logs' --set 'extraVolumes[0].persistentVolumeClaim.claimName=logs' \ + --set 'extraVolumeMounts[0].name=logs' --set 'extraVolumeMounts[0].mountPath=/opt/ctai/logs/')" + grep -q 'claimName: logs' <<<"$out" + if grep -q 'name: writable-logs' <<<"$out"; then echo "writable-logs mounted while disabled" >&2; exit 1; fi + [ "$(grep -c 'name: writable-' <<<"$out")" = 8 ] + ai_ro_fails() { + local pattern="$1"; shift + if ai_ro "$@" > /tmp/ro.out 2>&1; then echo "rendered: $*" >&2; exit 1; fi + grep -q "$pattern" /tmp/ro.out || { cat /tmp/ro.out >&2; exit 1; } + } + ai_ro_fails 'which writableDirectories.logs already mounts' \ + --set 'extraVolumes[0].name=logs' --set 'extraVolumes[0].emptyDir.medium=Memory' \ + --set 'extraVolumeMounts[0].name=logs' --set 'extraVolumeMounts[0].mountPath=/opt/ctai/logs/' + ai_ro_fails 'writableDirectories.systemTmp is disabled but nothing is mounted on /tmp' \ + --set writableDirectories.systemTmp.enabled=false + ai_ro_fails 'both mount /opt/ctai/www' --set writableDirectories.runtime.mountPath=/opt/ctai/www + ai_ro_fails 'must be an absolute path' --set writableDirectories.www.mountPath=opt/ctai/www + ai_ro_fails 'must be true or false' --set-string securityContext.readOnlyRootFilesystem=false + ai_ro_fails 'writableDirectories.logs must be a map' --set writableDirectories.logs=false # Rollout strategy: the default, what --reuse-values from a chart # before 0.2.4 leaves (no deploymentStrategy), and a Recreate override. ai_strategy() { diff --git a/charts/ai/Chart.yaml b/charts/ai/Chart.yaml index 12db068..f020e32 100644 --- a/charts/ai/Chart.yaml +++ b/charts/ai/Chart.yaml @@ -3,7 +3,7 @@ name: codetogether-ai description: CodeTogether AI on-premises Kubernetes deployment type: application -version: 0.2.4 +version: 0.2.5 appVersion: "1.2.8616" icon: https://www.codetogether.com/wp-content/uploads/2020/02/codetogether-circle-128.png diff --git a/charts/ai/README.md b/charts/ai/README.md index 1e1ebed..b142ca9 100644 --- a/charts/ai/README.md +++ b/charts/ai/README.md @@ -311,6 +311,9 @@ The following table lists the primary configurable parameters and defaults. | `readinessProbe.path` | Readiness endpoint on port 1080 | `/healthz` | | `livenessProbe.path` | Liveness endpoint on port 8080 | `/actuator/health` | | `startupProbe.enabled` | Enable startup probe | `true` | +| `securityContext.readOnlyRootFilesystem` | Run with a read-only root filesystem; the container writes only to `writableDirectories`. See [Read-only root filesystem](#read-only-root-filesystem) | `true` | +| `writableDirectories..enabled` | Mount an `emptyDir` on that directory (`runtime`, `www`, `logs`, `tmp`, `systemTmp`); `false` to mount your own volume there | `true` | +| `writableDirectories..sizeLimit` | Optional `emptyDir` size limit (for example `2Gi`); the pod is evicted when it is exceeded | `""` | `image.tag` controls which container image the Deployment pulls at install or upgrade time. `appVersion` in `Chart.yaml` is chart metadata only; it does not @@ -426,6 +429,67 @@ An ephemeral container stays in the pod's spec until the pod is replaced, and each one needs a new name (`-c`). The process environment and the mounted properties file contain secrets: do not paste them into tickets or chat. +### Read-only root filesystem + +From chart 0.2.5 the container runs with `readOnlyRootFilesystem: true`: the +image's binaries, scripts and configuration cannot be changed from inside the +pod. It writes only to these directories, each an `emptyDir` that lives as long +as the pod (`writableDirectories`): + +| Name | Path | What | +|---|---|---| +| `runtime` | `/opt/ctai/runtime` | Artifacts unpacked at start with the server URL injected into the IDE plugins (about 200 MB), and the YAML generated from the properties file (`config/ctai-configuration.yml`) | +| `www` | `/opt/ctai/www` | nginx's document root: the portal and metric lineage | +| `logs` | `/opt/ctai/logs` | `server.log`, the nginx logs, heap dumps (the newest is kept) | +| `tmp` | `/opt/ctai/tmp` | nginx's pid file and temp files | +| `systemTmp` | `/tmp` | JVM and Tomcat temp files | + +A container restart keeps them, and reuses the unpacked artifacts; a new pod +starts empty. To keep logs on a volume of your own, turn that directory over to +it: + +```yaml +writableDirectories: + logs: + enabled: false +extraVolumes: + - name: ctai-logs + persistentVolumeClaim: + claimName: ctai-logs +extraVolumeMounts: + - name: ctai-logs + mountPath: /opt/ctai/logs +``` + +The chart refuses: +- a volume in `extraVolumeMounts` on a path a writable directory already mounts; +- a disabled writable directory with no volume of yours on its path, which would be read-only; +- two writable directories on one path; +- `readOnlyRootFilesystem` as a string (`"true"` or `"false"`): it must be a boolean. + +Paths are compared after cleaning, so `/opt/ctai/logs/` is `/opt/ctai/logs`. + +**When the previous layout is used.** With the previous layout there are no +`writableDirectories`, and the container writes to its own layer. The chart +uses it: +- when `securityContext.readOnlyRootFilesystem` is `false`. Set that if a + pre-start hook (`/opt/ctai/hooks/pre-start.d`) writes outside these + directories. +- when `image.tag` (or `appVersion`, when the tag is empty) names an image + older than the first one that supports a read-only root filesystem. Older + images crash-loop under it. `helm install` prints a warning when this + happens. A tag that is not a version, such as `latest`, or an `image.digest`, + is taken as supporting it. +- after `helm upgrade --reuse-values` from a chart before 0.2.5, whose values + have no `readOnlyRootFilesystem`. Add + `--set securityContext.readOnlyRootFilesystem=true` to turn it on. + +**Upgrading from 0.2.4 with a volume of your own on one of these paths** (for +example a PVC on `/opt/ctai/logs` in `extraVolumeMounts`): `helm upgrade` stops +with "extraVolumeMounts mounts /opt/ctai/logs, which writableDirectories.logs +already mounts". Add `--set writableDirectories.logs.enabled=false` (or the +same in your values file) and run it again; your volume keeps that path. + ## Upgrade ```bash diff --git a/charts/ai/templates/NOTES.txt b/charts/ai/templates/NOTES.txt index 92bc7b7..9c7336f 100644 --- a/charts/ai/templates/NOTES.txt +++ b/charts/ai/templates/NOTES.txt @@ -67,3 +67,12 @@ Gateway API: serve this host. Set ingress.enabled=false once the Gateway path works. {{- end }} {{- end }} +{{- if and (eq (include "ctai.readOnlyRootFilesystemRequested" .) "true") (eq (include "ctai.imageTooOldForReadOnlyRoot" .) "true") }} + +WARNING: image {{ .Values.image.tag | default .Chart.AppVersion }} is older than {{ include "ctai.readOnlyRootFilesystemMinImage" . }}, the first image that +runs with a read-only root filesystem, so this release runs with the previous +layout (a writable root filesystem). Use a newer image to get it. +{{- else if not (eq (include "ctai.readOnlyRootFilesystem" .) "true") }} + +Read-only root filesystem: off (securityContext.readOnlyRootFilesystem is not true). +{{- end }} diff --git a/charts/ai/templates/_helpers.tpl b/charts/ai/templates/_helpers.tpl index bdac465..c4a9122 100644 --- a/charts/ai/templates/_helpers.tpl +++ b/charts/ai/templates/_helpers.tpl @@ -216,6 +216,198 @@ Fail helm template when properties contain unfilled values or URL {{- end -}} {{- end -}} {{- $_ := include "ctai.serviceFqdn" . -}} +{{- $_ = include "ctai.validateWritableDirectories" . -}} +{{- end }} + +{{/* +The first image version whose start-ctai runs on a read-only root filesystem. +Older images remove /opt/ctai/www (a mount point here) and write next to the +properties file, so they crash-loop under it. +*/}} +{{- define "ctai.readOnlyRootFilesystemMinImage" -}} +1.2.0-SET-AT-RELEASE +{{- end }} + +{{/* +"true" when the container runs with a read-only root filesystem: +securityContext.readOnlyRootFilesystem is true and the image is not older than +ctai.readOnlyRootFilesystemMinImage. The image's version is image.tag, or the +chart's appVersion when the tag is empty; a tag that is not a version (latest) or +a digest is taken as supporting it. --reuse-values from a chart before 0.2.5 has +no readOnlyRootFilesystem, so it keeps the previous layout. +*/}} +{{- define "ctai.readOnlyRootFilesystem" -}} +{{- if and (eq (include "ctai.readOnlyRootFilesystemRequested" .) "true") (eq (include "ctai.imageTooOldForReadOnlyRoot" .) "false") -}} +true +{{- else -}} +false +{{- end -}} +{{- end }} + +{{/* +"true" when securityContext.readOnlyRootFilesystem is true (validateWritableDirectories +refuses anything but a boolean). +*/}} +{{- define "ctai.readOnlyRootFilesystemRequested" -}} +{{- if eq ((.Values.securityContext | default dict).readOnlyRootFilesystem) true -}} +true +{{- else -}} +false +{{- end -}} +{{- end }} + +{{/* +"true" when the image is a version older than ctai.readOnlyRootFilesystemMinImage. +The version is the tag's leading x.y.z, so 1.2.8000-hotfix counts as 1.2.8000. +*/}} +{{- define "ctai.imageTooOldForReadOnlyRoot" -}} +{{- $image := .Values.image | default dict -}} +{{- $tag := toString ($image.tag | default .Chart.AppVersion) -}} +{{- $version := regexFind "^[0-9]+\\.[0-9]+\\.[0-9]+" (trimPrefix "v" $tag) -}} +{{- /* Nested ifs, not and: Helm before 3.10 evaluates every argument of and, + and semverCompare fails on an empty version. */ -}} +{{- $tooOld := false -}} +{{- if and (not $image.digest) $version -}} +{{- if semverCompare (printf "< %s" (include "ctai.readOnlyRootFilesystemMinImage" .)) $version -}} +{{- $tooOld = true -}} +{{- end -}} +{{- end -}} +{{- $tooOld -}} +{{- end }} + +{{/* +writableDirectories with the chart defaults underneath, as YAML: --reuse-values +from a chart before 0.2.5 has none. +*/}} +{{- define "ctai.writableDirectories" -}} +{{- $defaults := dict + "runtime" (dict "enabled" true "mountPath" "/opt/ctai/runtime" "sizeLimit" "") + "www" (dict "enabled" true "mountPath" "/opt/ctai/www" "sizeLimit" "") + "logs" (dict "enabled" true "mountPath" "/opt/ctai/logs" "sizeLimit" "") + "tmp" (dict "enabled" true "mountPath" "/opt/ctai/tmp" "sizeLimit" "") + "systemTmp" (dict "enabled" true "mountPath" "/tmp" "sizeLimit" "") -}} +{{- toYaml (mergeOverwrite $defaults (deepCopy (.Values.writableDirectories | default dict))) -}} +{{- end }} + +{{/* +writableDirectories: every enabled entry needs an absolute mountPath, no two +volumes may be mounted on the same path (Kubernetes rejects the pod, or compares +the paths as strings and lets one volume hide the other, and neither error names +the chart value nor the fix), and a disabled entry needs a volume of the +customer's in extraVolumeMounts, since the path is read-only otherwise. Paths are +compared after path.Clean, so /opt/ctai/logs/ is /opt/ctai/logs. +*/}} +{{- define "ctai.validateWritableDirectories" -}} +{{- $ro := (.Values.securityContext | default dict).readOnlyRootFilesystem -}} +{{- if and (not (kindIs "invalid" $ro)) (not (kindIs "bool" $ro)) -}} +{{- fail (printf "securityContext.readOnlyRootFilesystem must be true or false, got %q (a string): set it without quotes, or with --set rather than --set-string" (toString $ro)) -}} +{{- end -}} +{{- if eq (include "ctai.readOnlyRootFilesystem" .) "true" -}} +{{- $seen := dict -}} +{{- $extra := dict -}} +{{- range .Values.extraVolumeMounts -}} +{{- $_ := set $extra (clean (toString .mountPath)) true -}} +{{- end -}} +{{- $volumeNames := dict -}} +{{- range $name, $dir := (include "ctai.writableDirectories" . | fromYaml) -}} +{{- if not (kindIs "map" $dir) -}} +{{- fail (printf "writableDirectories.%s must be a map (enabled, mountPath, sizeLimit), got %v: to mount your own volume there, set writableDirectories.%s.enabled=false" $name $dir $name) -}} +{{- end -}} +{{- $volumeName := printf "writable-%s" ($name | kebabcase) -}} +{{- if hasKey $volumeNames $volumeName -}} +{{- fail (printf "writableDirectories.%s and writableDirectories.%s both make the volume name %s: rename one" (get $volumeNames $volumeName) $name $volumeName) -}} +{{- end -}} +{{- $_ := set $volumeNames $volumeName $name -}} +{{- $path := toString $dir.mountPath -}} +{{- if not (hasPrefix "/" $path) -}} +{{- fail (printf "writableDirectories.%s.mountPath must be an absolute path, got %q" $name $path) -}} +{{- end -}} +{{- $path = clean $path -}} +{{- if include "ctai.writableDirectoryEnabled" $dir -}} +{{- if hasKey $seen $path -}} +{{- fail (printf "writableDirectories.%s and writableDirectories.%s both mount %s" (get $seen $path) $name $path) -}} +{{- end -}} +{{- $_ := set $seen $path $name -}} +{{- else if not (hasKey $extra $path) -}} +{{- fail (printf "writableDirectories.%s is disabled but nothing is mounted on %s, which is read-only then: mount your own volume there with extraVolumes and extraVolumeMounts, or set securityContext.readOnlyRootFilesystem=false" $name $path) -}} +{{- end -}} +{{- end -}} +{{- range .Values.extraVolumeMounts -}} +{{- $path := clean (toString .mountPath) -}} +{{- if hasKey $seen $path -}} +{{- fail (printf "extraVolumeMounts mounts %s, which writableDirectories.%s already mounts: set writableDirectories.%s.enabled=false to use your own volume there" .mountPath (get $seen $path) (get $seen $path)) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +"true" (or empty) for a writableDirectories entry: enabled, or no enabled key. +*/}} +{{- define "ctai.writableDirectoryEnabled" -}} +{{- if or (not (hasKey . "enabled")) (eq .enabled true) -}} +true +{{- end -}} +{{- end }} + +{{/* +The container's securityContext, with readOnlyRootFilesystem as +ctai.readOnlyRootFilesystem resolves it. +*/}} +{{- define "ctai.containerSecurityContext" -}} +{{- $ctx := deepCopy (.Values.securityContext | default dict) -}} +{{- $_ := set $ctx "readOnlyRootFilesystem" (eq (include "ctai.readOnlyRootFilesystem" .) "true") -}} +{{- toYaml $ctx -}} +{{- end }} + +{{/* +The container's volumeMounts: the properties file, the writableDirectories (only +with a read-only root filesystem), then extraVolumeMounts. Empty when there are none. +*/}} +{{- define "ctai.volumeMounts" -}} +{{- if eq (include "ctai.propertiesEnabled" .) "true" }} +- name: ctai-properties + mountPath: {{ include "ctai.propertiesFilePath" . }} + subPath: {{ .Values.properties.fileName }} + readOnly: true +{{- end }} +{{- if eq (include "ctai.readOnlyRootFilesystem" .) "true" }} +{{- range $name, $dir := (include "ctai.writableDirectories" . | fromYaml) }} +{{- if include "ctai.writableDirectoryEnabled" $dir }} +- name: writable-{{ $name | kebabcase }} + mountPath: {{ $dir.mountPath }} +{{- end }} +{{- end }} +{{- end }} +{{- with .Values.extraVolumeMounts }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +The pod's volumes, matching ctai.volumeMounts. Empty when there are none. +*/}} +{{- define "ctai.volumes" -}} +{{- if eq (include "ctai.propertiesEnabled" .) "true" }} +- name: ctai-properties + secret: + secretName: {{ include "ctai.propertiesSecretName" . }} +{{- end }} +{{- if eq (include "ctai.readOnlyRootFilesystem" .) "true" }} +{{- range $name, $dir := (include "ctai.writableDirectories" . | fromYaml) }} +{{- if include "ctai.writableDirectoryEnabled" $dir }} +- name: writable-{{ $name | kebabcase }} + emptyDir: + {{- if $dir.sizeLimit }} + sizeLimit: {{ $dir.sizeLimit }} + {{- else }} {} + {{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- with .Values.extraVolumes }} +{{ toYaml . }} +{{- end }} {{- end }} {{/* diff --git a/charts/ai/templates/deployment.yaml b/charts/ai/templates/deployment.yaml index 464e5a6..c340b22 100644 --- a/charts/ai/templates/deployment.yaml +++ b/charts/ai/templates/deployment.yaml @@ -48,7 +48,7 @@ spec: containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} + {{- include "ctai.containerSecurityContext" . | nindent 12 }} image: "{{ .Values.image.repository }}{{ if .Values.image.digest }}@{{ .Values.image.digest }}{{ else }}:{{ .Values.image.tag | default .Chart.AppVersion }}{{ end }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: @@ -83,18 +83,9 @@ spec: {{- with .Values.extraEnvVars }} {{- toYaml . | nindent 12 }} {{- end }} - {{- if eq (include "ctai.propertiesEnabled" .) "true" }} + {{- with (include "ctai.volumeMounts" . | trim) }} volumeMounts: - - name: ctai-properties - mountPath: {{ include "ctai.propertiesFilePath" . }} - subPath: {{ .Values.properties.fileName }} - readOnly: true - {{- with .Values.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- else if .Values.extraVolumeMounts }} - volumeMounts: - {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- . | nindent 12 }} {{- end }} ports: - name: http @@ -138,17 +129,9 @@ spec: {{- with .Values.extraContainers }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if eq (include "ctai.propertiesEnabled" .) "true" }} - volumes: - - name: ctai-properties - secret: - secretName: {{ include "ctai.propertiesSecretName" . }} - {{- with .Values.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- else if .Values.extraVolumes }} + {{- with (include "ctai.volumes" . | trim) }} volumes: - {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- . | nindent 8 }} {{- end }} {{- with .Values.nodeSelector }} nodeSelector: diff --git a/charts/ai/values.yaml b/charts/ai/values.yaml index dbca263..a63033b 100644 --- a/charts/ai/values.yaml +++ b/charts/ai/values.yaml @@ -166,6 +166,52 @@ securityContext: runAsUser: 2200 runAsGroup: 2200 allowPrivilegeEscalation: false + # The image's own files are read-only; the directories it writes are the + # writableDirectories below. false restores the previous layout (no + # writableDirectories, the container writes to its own layer), for a pre-start + # hook that writes elsewhere. Images older than the first one that supports it + # (see "Read-only root filesystem" in README.md) always get the previous layout + # when image.tag names a version. + readOnlyRootFilesystem: true + +# The directories the container writes, each an emptyDir mounted over the image's +# empty directory, so the root filesystem can be read-only. Mounted only when +# securityContext.readOnlyRootFilesystem is in effect. They live as long as the pod: +# a container restart keeps them (and reuses the unpacked artifacts), a new pod +# starts empty. Set enabled: false on one to mount your own volume there instead +# (a PVC on logs, for example) through extraVolumes and extraVolumeMounts; the +# chart refuses two volumes on the same path, and a disabled directory with no +# volume of yours on it. sizeLimit (for example 2Gi) caps one: the kubelet evicts +# the pod when it is exceeded, so leave it empty unless the node's ephemeral +# storage needs protecting. +writableDirectories: + # Artifacts unpacked from artifacts.zip at start, with the server URL injected + # into the IDE plugins (about 200 MB), and the YAML generated from the + # properties file. + runtime: + enabled: true + mountPath: /opt/ctai/runtime + sizeLimit: "" + # nginx's document root: the portal and metric lineage, unpacked at start. + www: + enabled: true + mountPath: /opt/ctai/www + sizeLimit: "" + # server.log, the nginx logs and heap dumps (only the newest is kept). + logs: + enabled: true + mountPath: /opt/ctai/logs + sizeLimit: "" + # nginx's pid file and request-body and proxy temp files. + tmp: + enabled: true + mountPath: /opt/ctai/tmp + sizeLimit: "" + # The JVM's and Tomcat's temp files. + systemTmp: + enabled: true + mountPath: /tmp + sizeLimit: "" resources: {}