Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion openstack/archer/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
version: 0.1.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
Expand Down
2 changes: 1 addition & 1 deletion openstack/archer/alerts/kubernetes.alerts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ groups:
- name: kubernetes.alerts
rules:
- alert: archer-pod-not-ready
expr: kube_pod_status_ready_normalized{condition="true", pod=~"archer.+", pod!~"archer-migration.*|archer-cp-seed.*"} == 0
expr: kube_pod_status_ready_normalized{condition="true", pod=~"archer.+", pod!~"archer-migration.*"} == 0
for: 10m
labels:
context: availability
Expand Down
52 changes: 44 additions & 8 deletions openstack/archer/alerts/openstack.alerts
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,27 @@ groups:
description: 'Archer agent for {{ $labels.cloud_sap_host }} has failed health scrape loops > 5m.'
summary: 'Archer agent for {{ $labels.cloud_sap_host }} has health scrape failures.'

# SLO: availability >= 99.90% (error budget 0.10%).
# Page when the 1h error rate burns the budget more than 2x faster than allowed,
# i.e. error rate > 2 * 0.001 = 0.002 (0.2%) sustained over 1h.
- alert: archer-slo-error-budget-burn
# SLO: availability >= 99.90% (error budget 0.10%, i.e. allowed error ratio 0.001).
# Multi-window, multi-burn-rate alerting (Google SRE Workbook): page only when a
# long AND a short window are both burning budget fast. The short window makes the
# alert clear quickly once the errors stop, so a single/transient 5xx cannot keep
# it firing for a full hour the way a lone 1h rate() window would.
#
# Fast burn: 14.4x rate over 1h, confirmed by 5m -> burns 2% of budget in 1h.
- alert: archer-slo-error-budget-burn-fast
expr: |
(
sum(rate(http_requests_total{ccloud_service="archer",code=~"5.."}[1h]))
/
sum(rate(http_requests_total{ccloud_service="archer"}[1h]))
) > (2 * 0.001)
for: 5m
) > (14.4 * 0.001)
and
(
sum(rate(http_requests_total{ccloud_service="archer",code=~"5.."}[5m]))
/
sum(rate(http_requests_total{ccloud_service="archer"}[5m]))
) > (14.4 * 0.001)
for: 2m
labels:
context: api
service: archer
Expand All @@ -60,5 +70,31 @@ groups:
support_group: containers
playbook: docs/support/playbook/archer/alerts/archer-slo-error-budget-burn
annotations:
description: 'Archer API is burning its 99.90% availability SLO error budget at more than 2x the allowed rate over the last 1h.'
summary: 'Archer SLO error budget burn rate > 2x over 1h.'
description: 'Archer API is burning its 99.90% availability SLO error budget at 14.4x the allowed rate (1h and 5m windows both hot).'
summary: 'Archer SLO fast error budget burn (14.4x).'

# Slow burn: 6x rate over 6h, confirmed by 30m -> burns 5% of budget in 6h.
- alert: archer-slo-error-budget-burn-slow
expr: |
(
sum(rate(http_requests_total{ccloud_service="archer",code=~"5.."}[6h]))
/
sum(rate(http_requests_total{ccloud_service="archer"}[6h]))
) > (6 * 0.001)
and
(
sum(rate(http_requests_total{ccloud_service="archer",code=~"5.."}[30m]))
/
sum(rate(http_requests_total{ccloud_service="archer"}[30m]))
) > (6 * 0.001)
for: 15m
labels:
context: api
service: archer
severity: warning
tier: os
support_group: containers
playbook: docs/support/playbook/archer/alerts/archer-slo-error-budget-burn
annotations:
description: 'Archer API is burning its 99.90% availability SLO error budget at 6x the allowed rate (6h and 30m windows both hot).'
summary: 'Archer SLO slow error budget burn (6x).'
4 changes: 1 addition & 3 deletions openstack/archer/ci/test-values.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
global:
region: qa-de-1
tld: example.com
registry: keppel.example.com/ccloud
dockerHubMirror: keppel.example.com/dockerhub
dockerHubMirrorAlternateRegion: registry-other.example.com/dockerhub
Expand All @@ -14,9 +15,6 @@ image:
cp_network_interface: bond1

agents:
ni:
cp001:
create_service: false
cp:
- physical_network: cp001
replicas: 1
Expand Down
21 changes: 21 additions & 0 deletions openstack/archer/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,24 @@ Create the name of the service account to use
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

{{/*
Create the name of the main Archer Secret.
*/}}
{{- define "archer.secretName" -}}
{{- printf "%s-secret" (include "archer.fullname" .) -}}
{{- end }}

{{/*
Create the name of the linuxbridge agent Secret unless explicitly overridden.
*/}}
{{- define "archer.neutronSecretName" -}}
{{- .Values.linuxbridge_agent.secretName | default (printf "%s-neutron-secret" (include "archer.fullname" .)) -}}
{{- end }}

{{/*
Create the name of the postgres user Secret generated by the postgresql chart.
*/}}
{{- define "archer.pgUserSecretName" -}}
{{- printf "%s-pguser-archer" .Release.Name -}}
{{- end }}
137 changes: 137 additions & 0 deletions openstack/archer/templates/_neutron-linuxbridge-agent.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
{{/*
Minimal linuxbridge-agent helpers vendored from the unmerged library branch so
archer can render its own agent config and pod fragments without a chart
dependency.
*/}}

{{- define "archer.neutron-linuxbridge-agent.container" -}}
{{- $registry := .root.Values.global.registry | required ".Values.global.registry is required" -}}
{{- $version := .values.imageVersion | default .root.Values.image.neutron_version | required "Set .Values.image.neutron_version or .Values.linuxbridge_agent.imageVersion" -}}
- name: neutron-linuxbridge-agent
image: {{ $registry }}/loci-neutron:{{ $version }}
imagePullPolicy: IfNotPresent
command:
- neutron-linuxbridge-agent
{{- range .values.extraConfigs }}
- --config-file
- {{ . }}
{{- end }}
{{- range .values.extraConfigDirs }}
- --config-dir
- {{ . }}
{{- end }}
securityContext:
runAsUser: 0
# Must be a superset of neutron's oslo.privsep default context + SETPCAP, or the privsep daemon fails to start.
capabilities:
add:
- NET_ADMIN
- SYS_ADMIN
- SETPCAP
- DAC_OVERRIDE
- DAC_READ_SEARCH
- SYS_PTRACE
envFrom:
- secretRef:
name: {{ required ".Values.linuxbridge_agent.secretName is required" .values.secretName }}
{{- with .values.extraEnv }}
env:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .values.resources }}
resources:
{{- toYaml . | nindent 4 }}
{{- end }}
volumeMounts:
- name: linuxbridge-agent-etc
mountPath: /etc/neutron/neutron.conf
subPath: neutron.conf
readOnly: true
- name: linuxbridge-agent-etc
mountPath: /etc/neutron/logging.conf
subPath: logging.conf
readOnly: true
- name: modules
mountPath: /lib/modules
readOnly: true
{{- with .values.extraVolumeMounts }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end -}}

{{- define "archer.neutron-linuxbridge-agent.volumes" -}}
- name: linuxbridge-agent-etc
configMap:
name: {{ .root.Release.Name }}-linuxbridge-agent-etc
- name: modules
hostPath:
path: /lib/modules
{{- with .values.extraVolumes }}
{{ toYaml . }}
{{- end }}
{{- end -}}

{{- define "archer.neutron-linuxbridge-agent.configmap-etc" -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .root.Release.Name }}-linuxbridge-agent-etc
data:
neutron.conf: |
{{- include "archer.neutron-linuxbridge-agent.neutron-conf" . | nindent 4 }}
logging.conf: |
{{- include "archer.neutron-linuxbridge-agent.logging-conf" . | nindent 4 }}
{{- end -}}

{{- define "archer.neutron-linuxbridge-agent.neutron-conf" -}}
[DEFAULT]
core_plugin = ml2
debug = {{ .values.debug | default false }}
log_config_append = /etc/neutron/logging.conf
default_log_levels = {{ join "," (.values.default_log_levels | default (list "neutron.plugins.ml2.drivers.agent._common_agent=WARN")) }}
{{- with .values.host }}
host = {{ . }}
{{- end }}

[agent]
polling_interval = {{ dig "agent" "polling_interval" 2 .values }}

[linux_bridge]
# physical_interface_mappings comes from env OS_LINUX_BRIDGE__PHYSICAL_INTERFACE_MAPPINGS

[vxlan]
enable_vxlan = {{ dig "vxlan" "enable" false .values }}

[securitygroup]
firewall_driver = {{ dig "securitygroup" "firewall_driver" "iptables" .values }}

[privsep]
thread_pool_size = {{ dig "privsep" "thread_pool_size" 3 .values }}
helper_command = {{ dig "privsep" "helper_command" "privsep-helper --config-file /etc/neutron/neutron.conf" .values }}

{{- $helper := dig "privsep" "helper_command" "privsep-helper --config-file /etc/neutron/neutron.conf" .values }}
{{- range list "privsep_dhcp_release" "privsep_ovs_vsctl" "privsep_namespace" "privsep_conntrack" "privsep_link" }}

[{{ . }}]
helper_command = {{ $helper }}
{{- end }}

[oslo_concurrency]
lock_path = /tmp
{{- end -}}

{{- define "archer.neutron-linuxbridge-agent.logging-conf" -}}
{{- $default := dict
"loggers" (dict
"root" (dict "handlers" "stdout" "level" "INFO")
)
"handlers" (dict
"stdout" (dict "class" "StreamHandler" "args" "(sys.stdout,)" "formatter" "context")
)
"formatters" (dict
"context" (dict "class" "oslo_log.formatters.ContextFormatter")
)
-}}
{{- $logging := .values.logging | default $default -}}
{{- include "loggerIni" $logging -}}
{{- end -}}
Loading
Loading