Skip to content

Remove kolla entrypoint pattern from Cyborg, harden SecurityContext #1182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stuggi wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Remove kolla entrypoint pattern from Cyborg, harden SecurityContext #1182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions config/rbac/role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,14 +272,6 @@ rules:
- patch
- update
- watch
- apiGroups:
- security.openshift.io
resourceNames:
- anyuid
resources:
- securitycontextconstraints
verbs:
- use
- apiGroups:
- security.openshift.io
resourceNames:
Expand Down
4 changes: 2 additions & 2 deletions internal/controller/cyborg/cyborg_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (r *CyborgReconciler) GetLogger(ctx context.Context) logr.Logger {
// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch
// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch
// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch
// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use
// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
Expand Down Expand Up @@ -515,7 +515,7 @@ func (r *CyborgReconciler) ensureRbac(
rbacRules := []rbacv1.PolicyRule{
{
APIGroups: []string{"security.openshift.io"},
ResourceNames: []string{"anyuid"},
ResourceNames: []string{"nonroot-v2"},
Resources: []string{"securitycontextconstraints"},
Verbs: []string{"use"},
},
Expand Down
107 changes: 54 additions & 53 deletions internal/cyborg/api/statefulset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,18 @@ limitations under the License.
package api

import (
"fmt"

topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1"
"github.com/openstack-k8s-operators/lib-common/modules/common"
"github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
"github.com/openstack-k8s-operators/lib-common/modules/common/env"
"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
"github.com/openstack-k8s-operators/lib-common/modules/common/probes"
libservice "github.com/openstack-k8s-operators/lib-common/modules/common/service"
"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
"github.com/openstack-k8s-operators/lib-common/modules/common/volume"
"github.com/openstack-k8s-operators/lib-common/modules/users"

cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1"
internalcommon "github.com/openstack-k8s-operators/nova-operator/internal/common"
Expand All @@ -50,12 +55,8 @@ func StatefulSet(
labels map[string]string,
topology *topologyv1.Topology,
) (*appsv1.StatefulSet, error) {
var config0644AccessMode int32 = 0644

envVars := make(map[string]env.Setter)
envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
envVars["CONFIG_HASH"] = env.SetValue(configHash)
args := []string{"-c", internalcommon.ServiceCommand}

scheme := corev1.URISchemeHTTP
if instance.Spec.TLS.API.Enabled(libservice.EndpointPublic) {
Expand All @@ -71,50 +72,39 @@ func StatefulSet(
return nil, err
}

logVolumeMount := corev1.VolumeMount{
Name: cyborg.LogVolume,
MountPath: "/var/log/cyborg",
ReadOnly: false,
}
logVolumeMount := volume.WritableDirVolumeMount(cyborg.LogVolume, "/var/log/cyborg")

volumes := []corev1.Volume{
{
Name: cyborg.ConfigVolume,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
DefaultMode: &config0644AccessMode,
SecretName: internalcommon.GetServiceConfigSecretName(instance.Name),
},
},
},
{
Name: cyborg.LogVolume,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""},
},
},
cyborg.GetConfigVolume(internalcommon.GetServiceConfigSecretName(instance.Name)),
volume.WritableDirVolume(cyborg.LogVolume),
volume.WritableDirVolume(volume.RunHttpdVolumeName),
volume.WritableDirVolume(volume.VarLogHttpdVolumeName),
}

volumeMounts := []corev1.VolumeMount{
{
volumeMounts := cyborg.GetConfVolumeMounts(instance.Spec.CustomServiceConfig != "")
volumeMounts = append(volumeMounts,
logVolumeMount,
corev1.VolumeMount{
Name: cyborg.ConfigVolume,
MountPath: "/var/lib/config-data/default",
MountPath: "/etc/httpd/conf/httpd.conf",
SubPath: "httpd.conf",
ReadOnly: true,
},
{
corev1.VolumeMount{
Name: cyborg.ConfigVolume,
MountPath: "/var/lib/kolla/config_files/config.json",
SubPath: "cyborg-api-config.json",
MountPath: "/etc/httpd/conf.d/ssl.conf",
SubPath: "ssl.conf",
ReadOnly: true,
},
{
corev1.VolumeMount{
Name: cyborg.ConfigVolume,
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
MountPath: "/etc/httpd/conf.d/10-cyborg-wsgi-main.conf",
SubPath: "10-cyborg-wsgi-main.conf",
ReadOnly: true,
},
logVolumeMount,
}
volume.WritableDirVolumeMount(volume.RunHttpdVolumeName, volume.RunHttpdMountPath),
volume.WritableDirVolumeMount(volume.VarLogHttpdVolumeName, volume.VarLogHttpdMountPath),
)

// Add CA bundle volume if set
if instance.Spec.TLS.CaBundleSecretName != "" {
Expand All @@ -136,6 +126,15 @@ func StatefulSet(
if err != nil {
return nil, err
}
// Final paths, matching what generateServiceConfig renders into
// 10-cyborg-wsgi-main.conf's SSLCertificateFile/SSLCertificateKeyFile
// -- without this, CreateVolumeMounts defaults to lib-common's
// staging path, which nothing copies from once kolla's
// config.json is gone.
certMount := fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String())
keyMount := fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String())
svc.CertMount = &certMount
svc.KeyMount = &keyMount
volumes = append(volumes, svc.CreateVolume(endpt.String()))
volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...)
}
Expand All @@ -160,6 +159,12 @@ func StatefulSet(
Spec: corev1.PodSpec{
ServiceAccountName: instance.Spec.ServiceAccount,
AutomountServiceAccountToken: ptr.To(false),
// httpd.conf's User/Group were changed from apache to
// cyborg, matching 10-cyborg-wsgi-main.conf's
// pre-existing WSGIDaemonProcess user=cyborg
// group=cyborg (proof a dedicated "cyborg" system user
// already exists in the image).
SecurityContext: pod.RestrictivePodSecurityContext(users.CyborgUID, users.CyborgGID),
Containers: []corev1.Container{
{
Name: ComponentName + "-log",
Expand All @@ -174,29 +179,25 @@ func StatefulSet(
"-F",
cyborg.CyborgLogPath + instance.Name + ".log",
},
Image: instance.Spec.ContainerImage,
SecurityContext: &corev1.SecurityContext{
RunAsUser: ptr.To(cyborg.CyborgUserID),
},
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
VolumeMounts: []corev1.VolumeMount{logVolumeMount},
Resources: instance.Spec.Resources,
Image: instance.Spec.ContainerImage,
SecurityContext: pod.RestrictiveSecurityContext(users.CyborgUID, users.CyborgGID),
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
VolumeMounts: []corev1.VolumeMount{logVolumeMount},
Resources: instance.Spec.Resources,
},
{
Name: ComponentName,
Command: []string{
"/bin/bash",
},
Args: args,
Image: instance.Spec.ContainerImage,
SecurityContext: &corev1.SecurityContext{
RunAsUser: ptr.To(cyborg.CyborgUserID),
"/usr/sbin/httpd",
},
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
VolumeMounts: volumeMounts,
Resources: instance.Spec.Resources,
ReadinessProbe: apiProbes.Readiness,
LivenessProbe: apiProbes.Liveness,
Args: []string{"-DFOREGROUND"},
Image: instance.Spec.ContainerImage,
SecurityContext: pod.RestrictiveSecurityContext(users.CyborgUID, users.CyborgGID),
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
VolumeMounts: volumeMounts,
Resources: instance.Spec.Resources,
ReadinessProbe: apiProbes.Readiness,
LivenessProbe: apiProbes.Liveness,
},
},
Volumes: volumes,
Expand Down
57 changes: 14 additions & 43 deletions internal/cyborg/conductor/statefulset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import (
"github.com/openstack-k8s-operators/lib-common/modules/common"
"github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
"github.com/openstack-k8s-operators/lib-common/modules/common/env"
"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
"github.com/openstack-k8s-operators/lib-common/modules/common/probes"
"github.com/openstack-k8s-operators/lib-common/modules/users"

topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1"
cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1"
Expand Down Expand Up @@ -57,44 +59,14 @@ func StatefulSet(
return nil, err
}

var config0644AccessMode int32 = 0644

envVars := make(map[string]env.Setter)
envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
envVars["CONFIG_HASH"] = env.SetValue(configHash)
args := []string{"-c", internalcommon.ServiceCommand}

volumes := []corev1.Volume{
{
Name: cyborg.ConfigVolume,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
DefaultMode: &config0644AccessMode,
SecretName: internalcommon.GetServiceConfigSecretName(instance.Name),
},
},
},
cyborg.GetConfigVolume(internalcommon.GetServiceConfigSecretName(instance.Name)),
}

volumeMounts := []corev1.VolumeMount{
{
Name: cyborg.ConfigVolume,
MountPath: "/var/lib/config-data/default",
ReadOnly: true,
},
{
Name: cyborg.ConfigVolume,
MountPath: "/var/lib/kolla/config_files/config.json",
SubPath: "cyborg-conductor-config.json",
ReadOnly: true,
},
{
Name: cyborg.ConfigVolume,
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
ReadOnly: true,
},
}
volumeMounts := cyborg.GetConfVolumeMounts(instance.Spec.CustomServiceConfig != "")

if instance.Spec.TLS.CaBundleSecretName != "" {
volumes = append(volumes, instance.Spec.TLS.CreateVolume())
Expand All @@ -120,22 +92,21 @@ func StatefulSet(
Spec: corev1.PodSpec{
ServiceAccountName: instance.Spec.ServiceAccount,
AutomountServiceAccountToken: ptr.To(false),
SecurityContext: pod.RestrictivePodSecurityContext(users.CyborgUID, users.CyborgGID),
Containers: []corev1.Container{
{
Name: ComponentName,
Command: []string{
"/bin/bash",
},
Args: args,
Image: instance.Spec.ContainerImage,
SecurityContext: &corev1.SecurityContext{
RunAsUser: ptr.To(cyborg.CyborgUserID),
"cyborg-conductor",
},
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
VolumeMounts: volumeMounts,
Resources: instance.Spec.Resources,
StartupProbe: conductorProbes.Startup,
LivenessProbe: conductorProbes.Liveness,
Args: []string{"--config-dir", "/etc/cyborg/cyborg.conf.d"},
Image: instance.Spec.ContainerImage,
SecurityContext: pod.RestrictiveSecurityContext(users.CyborgUID, users.CyborgGID),
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
VolumeMounts: volumeMounts,
Resources: instance.Spec.Resources,
StartupProbe: conductorProbes.Startup,
LivenessProbe: conductorProbes.Liveness,
},
},
Volumes: volumes,
Expand Down
8 changes: 4 additions & 4 deletions internal/cyborg/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,13 @@ const (
// DefaultsConfigFileName is the file name with default configuration
DefaultsConfigFileName = "00-default.conf"

// ServiceCustomConfigFileName is the file name with the CustomServiceConfig
// override, only present in the config Secret when actually set
ServiceCustomConfigFileName = "01-service-custom.conf"

// CyborgLogPath is the default path for the cyborg service logs
CyborgLogPath = "/var/log/cyborg/"

// LogVolume is the name of the EmptyDir volume used for log streaming
LogVolume = "logs"

// CyborgUserID is the linux user ID used by Kolla for the cyborg user
// in the service containers
CyborgUserID int64 = 42485
)
Loading
Loading