Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions .github/workflows/helm-template-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,58 @@ jobs:
--set imageCredentials.username=test \
--set imageCredentials.password=test \
--set gateway=null
# Read-only root filesystem: an emptyDir on each writable directory by
# default; the previous layout for ro=false, an image older than the first
# one that supports it, and --reuse-values from a chart before 0.2.5.
ai_ro() {
helm template test ./charts/ai \
--set-file ctaiPropertiesFile=./charts/ai/ctai-configuration.properties.template \
--set service.fqdn=https://example.codetogether.io \
--set imageCredentials.username=test \
--set imageCredentials.password=test \
--show-only templates/deployment.yaml "$@"
}
ro_min="$(sed -n '/define "ctai.readOnlyRootFilesystemMinImage"/{n;p}' ./charts/ai/templates/_helpers.tpl)"
if ! [[ "$ro_min" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Set ctai.readOnlyRootFilesystemMinImage to the first release that runs read-only (got $ro_min)" >&2; exit 1
fi
app_version="$(sed -n 's/^appVersion: "\(.*\)"$/\1/p' ./charts/ai/Chart.yaml)"
if [ "$(printf '%s\n%s\n' "$ro_min" "$app_version" | sort -V | head -1)" != "$ro_min" ]; then
echo "appVersion $app_version is older than ctai.readOnlyRootFilesystemMinImage $ro_min" >&2; exit 1
fi
out="$(ai_ro)"
grep -q 'readOnlyRootFilesystem: true' <<<"$out"
[ "$(grep -c 'name: writable-' <<<"$out")" = 10 ]
for args in "--set securityContext.readOnlyRootFilesystem=false" \
"--set image.tag=1.0.0" \
"--set image.tag=1.0.0-hotfix" \
"--set securityContext.readOnlyRootFilesystem=null --set writableDirectories=null"; do
# shellcheck disable=SC2086
out="$(ai_ro $args)"
grep -q 'readOnlyRootFilesystem: false' <<<"$out"
if grep -q 'name: writable-' <<<"$out"; then echo "writable volumes with $args" >&2; exit 1; fi
done
# A customer volume instead of one writable directory, and the refusals.
out="$(ai_ro --set writableDirectories.logs.enabled=false \
--set 'extraVolumes[0].name=logs' --set 'extraVolumes[0].persistentVolumeClaim.claimName=logs' \
--set 'extraVolumeMounts[0].name=logs' --set 'extraVolumeMounts[0].mountPath=/opt/ctai/logs/')"
grep -q 'claimName: logs' <<<"$out"
if grep -q 'name: writable-logs' <<<"$out"; then echo "writable-logs mounted while disabled" >&2; exit 1; fi
[ "$(grep -c 'name: writable-' <<<"$out")" = 8 ]
ai_ro_fails() {
local pattern="$1"; shift
if ai_ro "$@" > /tmp/ro.out 2>&1; then echo "rendered: $*" >&2; exit 1; fi
grep -q "$pattern" /tmp/ro.out || { cat /tmp/ro.out >&2; exit 1; }
}
ai_ro_fails 'which writableDirectories.logs already mounts' \
--set 'extraVolumes[0].name=logs' --set 'extraVolumes[0].emptyDir.medium=Memory' \
--set 'extraVolumeMounts[0].name=logs' --set 'extraVolumeMounts[0].mountPath=/opt/ctai/logs/'
ai_ro_fails 'writableDirectories.systemTmp is disabled but nothing is mounted on /tmp' \
--set writableDirectories.systemTmp.enabled=false
ai_ro_fails 'both mount /opt/ctai/www' --set writableDirectories.runtime.mountPath=/opt/ctai/www
ai_ro_fails 'must be an absolute path' --set writableDirectories.www.mountPath=opt/ctai/www
ai_ro_fails 'must be true or false' --set-string securityContext.readOnlyRootFilesystem=false
ai_ro_fails 'writableDirectories.logs must be a map' --set writableDirectories.logs=false
# Rollout strategy: the default, what --reuse-values from a chart
# before 0.2.4 leaves (no deploymentStrategy), and a Recreate override.
ai_strategy() {
Expand Down
2 changes: 1 addition & 1 deletion charts/ai/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: codetogether-ai
description: CodeTogether AI on-premises Kubernetes deployment

type: application
version: 0.2.4
version: 0.2.5
appVersion: "1.2.8616"

icon: https://www.codetogether.com/wp-content/uploads/2020/02/codetogether-circle-128.png
Expand Down
64 changes: 64 additions & 0 deletions charts/ai/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,9 @@ The following table lists the primary configurable parameters and defaults.
| `readinessProbe.path` | Readiness endpoint on port 1080 | `/healthz` |
| `livenessProbe.path` | Liveness endpoint on port 8080 | `/actuator/health` |
| `startupProbe.enabled` | Enable startup probe | `true` |
| `securityContext.readOnlyRootFilesystem` | Run with a read-only root filesystem; the container writes only to `writableDirectories`. See [Read-only root filesystem](#read-only-root-filesystem) | `true` |
| `writableDirectories.<name>.enabled` | Mount an `emptyDir` on that directory (`runtime`, `www`, `logs`, `tmp`, `systemTmp`); `false` to mount your own volume there | `true` |
| `writableDirectories.<name>.sizeLimit` | Optional `emptyDir` size limit (for example `2Gi`); the pod is evicted when it is exceeded | `""` |

`image.tag` controls which container image the Deployment pulls at install or
upgrade time. `appVersion` in `Chart.yaml` is chart metadata only; it does not
Expand Down Expand Up @@ -426,6 +429,67 @@ An ephemeral container stays in the pod's spec until the pod is replaced, and
each one needs a new name (`-c`). The process environment and the mounted
properties file contain secrets: do not paste them into tickets or chat.

### Read-only root filesystem

From chart 0.2.5 the container runs with `readOnlyRootFilesystem: true`: the
image's binaries, scripts and configuration cannot be changed from inside the
pod. It writes only to these directories, each an `emptyDir` that lives as long
as the pod (`writableDirectories`):

| Name | Path | What |
|---|---|---|
| `runtime` | `/opt/ctai/runtime` | Artifacts unpacked at start with the server URL injected into the IDE plugins (about 200 MB), and the YAML generated from the properties file (`config/ctai-configuration.yml`) |
| `www` | `/opt/ctai/www` | nginx's document root: the portal and metric lineage |
| `logs` | `/opt/ctai/logs` | `server.log`, the nginx logs, heap dumps (the newest is kept) |
| `tmp` | `/opt/ctai/tmp` | nginx's pid file and temp files |
| `systemTmp` | `/tmp` | JVM and Tomcat temp files |

A container restart keeps them, and reuses the unpacked artifacts; a new pod
starts empty. To keep logs on a volume of your own, turn that directory over to
it:

```yaml
writableDirectories:
logs:
enabled: false
extraVolumes:
- name: ctai-logs
persistentVolumeClaim:
claimName: ctai-logs
extraVolumeMounts:
- name: ctai-logs
mountPath: /opt/ctai/logs
```

The chart refuses:
- a volume in `extraVolumeMounts` on a path a writable directory already mounts;
- a disabled writable directory with no volume of yours on its path, which would be read-only;
- two writable directories on one path;
- `readOnlyRootFilesystem` as a string (`"true"` or `"false"`): it must be a boolean.

Paths are compared after cleaning, so `/opt/ctai/logs/` is `/opt/ctai/logs`.

**When the previous layout is used.** With the previous layout there are no
`writableDirectories`, and the container writes to its own layer. The chart
uses it:
- when `securityContext.readOnlyRootFilesystem` is `false`. Set that if a
pre-start hook (`/opt/ctai/hooks/pre-start.d`) writes outside these
directories.
- when `image.tag` (or `appVersion`, when the tag is empty) names an image
older than the first one that supports a read-only root filesystem. Older
images crash-loop under it. `helm install` prints a warning when this
happens. A tag that is not a version, such as `latest`, or an `image.digest`,
is taken as supporting it.
- after `helm upgrade --reuse-values` from a chart before 0.2.5, whose values
have no `readOnlyRootFilesystem`. Add
`--set securityContext.readOnlyRootFilesystem=true` to turn it on.

**Upgrading from 0.2.4 with a volume of your own on one of these paths** (for
example a PVC on `/opt/ctai/logs` in `extraVolumeMounts`): `helm upgrade` stops
with "extraVolumeMounts mounts /opt/ctai/logs, which writableDirectories.logs
already mounts". Add `--set writableDirectories.logs.enabled=false` (or the
same in your values file) and run it again; your volume keeps that path.

## Upgrade

```bash
Expand Down
9 changes: 9 additions & 0 deletions charts/ai/templates/NOTES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,12 @@ Gateway API:
serve this host. Set ingress.enabled=false once the Gateway path works.
{{- end }}
{{- end }}
{{- if and (eq (include "ctai.readOnlyRootFilesystemRequested" .) "true") (eq (include "ctai.imageTooOldForReadOnlyRoot" .) "true") }}

WARNING: image {{ .Values.image.tag | default .Chart.AppVersion }} is older than {{ include "ctai.readOnlyRootFilesystemMinImage" . }}, the first image that
runs with a read-only root filesystem, so this release runs with the previous
layout (a writable root filesystem). Use a newer image to get it.
{{- else if not (eq (include "ctai.readOnlyRootFilesystem" .) "true") }}

Read-only root filesystem: off (securityContext.readOnlyRootFilesystem is not true).
{{- end }}
192 changes: 192 additions & 0 deletions charts/ai/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,198 @@ Fail helm template when properties contain unfilled <placeholder> values or URL
{{- end -}}
{{- end -}}
{{- $_ := include "ctai.serviceFqdn" . -}}
{{- $_ = include "ctai.validateWritableDirectories" . -}}
{{- end }}

{{/*
The first image version whose start-ctai runs on a read-only root filesystem.
Older images remove /opt/ctai/www (a mount point here) and write next to the
properties file, so they crash-loop under it.
*/}}
{{- define "ctai.readOnlyRootFilesystemMinImage" -}}
1.2.0-SET-AT-RELEASE
{{- end }}

{{/*
"true" when the container runs with a read-only root filesystem:
securityContext.readOnlyRootFilesystem is true and the image is not older than
ctai.readOnlyRootFilesystemMinImage. The image's version is image.tag, or the
chart's appVersion when the tag is empty; a tag that is not a version (latest) or
a digest is taken as supporting it. --reuse-values from a chart before 0.2.5 has
no readOnlyRootFilesystem, so it keeps the previous layout.
*/}}
{{- define "ctai.readOnlyRootFilesystem" -}}
{{- if and (eq (include "ctai.readOnlyRootFilesystemRequested" .) "true") (eq (include "ctai.imageTooOldForReadOnlyRoot" .) "false") -}}
true
{{- else -}}
false
{{- end -}}
{{- end }}

{{/*
"true" when securityContext.readOnlyRootFilesystem is true (validateWritableDirectories
refuses anything but a boolean).
*/}}
{{- define "ctai.readOnlyRootFilesystemRequested" -}}
{{- if eq ((.Values.securityContext | default dict).readOnlyRootFilesystem) true -}}
true
{{- else -}}
false
{{- end -}}
{{- end }}

{{/*
"true" when the image is a version older than ctai.readOnlyRootFilesystemMinImage.
The version is the tag's leading x.y.z, so 1.2.8000-hotfix counts as 1.2.8000.
*/}}
{{- define "ctai.imageTooOldForReadOnlyRoot" -}}
{{- $image := .Values.image | default dict -}}
{{- $tag := toString ($image.tag | default .Chart.AppVersion) -}}
{{- $version := regexFind "^[0-9]+\\.[0-9]+\\.[0-9]+" (trimPrefix "v" $tag) -}}
{{- /* Nested ifs, not and: Helm before 3.10 evaluates every argument of and,
and semverCompare fails on an empty version. */ -}}
{{- $tooOld := false -}}
{{- if and (not $image.digest) $version -}}
{{- if semverCompare (printf "< %s" (include "ctai.readOnlyRootFilesystemMinImage" .)) $version -}}
{{- $tooOld = true -}}
{{- end -}}
{{- end -}}
{{- $tooOld -}}
{{- end }}

{{/*
writableDirectories with the chart defaults underneath, as YAML: --reuse-values
from a chart before 0.2.5 has none.
*/}}
{{- define "ctai.writableDirectories" -}}
{{- $defaults := dict
"runtime" (dict "enabled" true "mountPath" "/opt/ctai/runtime" "sizeLimit" "")
"www" (dict "enabled" true "mountPath" "/opt/ctai/www" "sizeLimit" "")
"logs" (dict "enabled" true "mountPath" "/opt/ctai/logs" "sizeLimit" "")
"tmp" (dict "enabled" true "mountPath" "/opt/ctai/tmp" "sizeLimit" "")
"systemTmp" (dict "enabled" true "mountPath" "/tmp" "sizeLimit" "") -}}
{{- toYaml (mergeOverwrite $defaults (deepCopy (.Values.writableDirectories | default dict))) -}}
{{- end }}

{{/*
writableDirectories: every enabled entry needs an absolute mountPath, no two
volumes may be mounted on the same path (Kubernetes rejects the pod, or compares
the paths as strings and lets one volume hide the other, and neither error names
the chart value nor the fix), and a disabled entry needs a volume of the
customer's in extraVolumeMounts, since the path is read-only otherwise. Paths are
compared after path.Clean, so /opt/ctai/logs/ is /opt/ctai/logs.
*/}}
{{- define "ctai.validateWritableDirectories" -}}
{{- $ro := (.Values.securityContext | default dict).readOnlyRootFilesystem -}}
{{- if and (not (kindIs "invalid" $ro)) (not (kindIs "bool" $ro)) -}}
{{- fail (printf "securityContext.readOnlyRootFilesystem must be true or false, got %q (a string): set it without quotes, or with --set rather than --set-string" (toString $ro)) -}}
{{- end -}}
{{- if eq (include "ctai.readOnlyRootFilesystem" .) "true" -}}
{{- $seen := dict -}}
{{- $extra := dict -}}
{{- range .Values.extraVolumeMounts -}}
{{- $_ := set $extra (clean (toString .mountPath)) true -}}
{{- end -}}
{{- $volumeNames := dict -}}
{{- range $name, $dir := (include "ctai.writableDirectories" . | fromYaml) -}}
{{- if not (kindIs "map" $dir) -}}
{{- fail (printf "writableDirectories.%s must be a map (enabled, mountPath, sizeLimit), got %v: to mount your own volume there, set writableDirectories.%s.enabled=false" $name $dir $name) -}}
{{- end -}}
{{- $volumeName := printf "writable-%s" ($name | kebabcase) -}}
{{- if hasKey $volumeNames $volumeName -}}
{{- fail (printf "writableDirectories.%s and writableDirectories.%s both make the volume name %s: rename one" (get $volumeNames $volumeName) $name $volumeName) -}}
{{- end -}}
{{- $_ := set $volumeNames $volumeName $name -}}
{{- $path := toString $dir.mountPath -}}
{{- if not (hasPrefix "/" $path) -}}
{{- fail (printf "writableDirectories.%s.mountPath must be an absolute path, got %q" $name $path) -}}
{{- end -}}
{{- $path = clean $path -}}
{{- if include "ctai.writableDirectoryEnabled" $dir -}}
{{- if hasKey $seen $path -}}
{{- fail (printf "writableDirectories.%s and writableDirectories.%s both mount %s" (get $seen $path) $name $path) -}}
{{- end -}}
{{- $_ := set $seen $path $name -}}
{{- else if not (hasKey $extra $path) -}}
{{- fail (printf "writableDirectories.%s is disabled but nothing is mounted on %s, which is read-only then: mount your own volume there with extraVolumes and extraVolumeMounts, or set securityContext.readOnlyRootFilesystem=false" $name $path) -}}
{{- end -}}
{{- end -}}
{{- range .Values.extraVolumeMounts -}}
{{- $path := clean (toString .mountPath) -}}
{{- if hasKey $seen $path -}}
{{- fail (printf "extraVolumeMounts mounts %s, which writableDirectories.%s already mounts: set writableDirectories.%s.enabled=false to use your own volume there" .mountPath (get $seen $path) (get $seen $path)) -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end }}

{{/*
"true" (or empty) for a writableDirectories entry: enabled, or no enabled key.
*/}}
{{- define "ctai.writableDirectoryEnabled" -}}
{{- if or (not (hasKey . "enabled")) (eq .enabled true) -}}
true
{{- end -}}
{{- end }}

{{/*
The container's securityContext, with readOnlyRootFilesystem as
ctai.readOnlyRootFilesystem resolves it.
*/}}
{{- define "ctai.containerSecurityContext" -}}
{{- $ctx := deepCopy (.Values.securityContext | default dict) -}}
{{- $_ := set $ctx "readOnlyRootFilesystem" (eq (include "ctai.readOnlyRootFilesystem" .) "true") -}}
{{- toYaml $ctx -}}
{{- end }}

{{/*
The container's volumeMounts: the properties file, the writableDirectories (only
with a read-only root filesystem), then extraVolumeMounts. Empty when there are none.
*/}}
{{- define "ctai.volumeMounts" -}}
{{- if eq (include "ctai.propertiesEnabled" .) "true" }}
- name: ctai-properties
mountPath: {{ include "ctai.propertiesFilePath" . }}
subPath: {{ .Values.properties.fileName }}
readOnly: true
{{- end }}
{{- if eq (include "ctai.readOnlyRootFilesystem" .) "true" }}
{{- range $name, $dir := (include "ctai.writableDirectories" . | fromYaml) }}
{{- if include "ctai.writableDirectoryEnabled" $dir }}
- name: writable-{{ $name | kebabcase }}
mountPath: {{ $dir.mountPath }}
{{- end }}
{{- end }}
{{- end }}
{{- with .Values.extraVolumeMounts }}
{{ toYaml . }}
{{- end }}
{{- end }}

{{/*
The pod's volumes, matching ctai.volumeMounts. Empty when there are none.
*/}}
{{- define "ctai.volumes" -}}
{{- if eq (include "ctai.propertiesEnabled" .) "true" }}
- name: ctai-properties
secret:
secretName: {{ include "ctai.propertiesSecretName" . }}
{{- end }}
{{- if eq (include "ctai.readOnlyRootFilesystem" .) "true" }}
{{- range $name, $dir := (include "ctai.writableDirectories" . | fromYaml) }}
{{- if include "ctai.writableDirectoryEnabled" $dir }}
- name: writable-{{ $name | kebabcase }}
emptyDir:
{{- if $dir.sizeLimit }}
sizeLimit: {{ $dir.sizeLimit }}
{{- else }} {}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- with .Values.extraVolumes }}
{{ toYaml . }}
{{- end }}
{{- end }}

{{/*
Expand Down
Loading
Loading