Incident Response Runbook

name: incident-runbook
desc: "On-call triage: classify severity, gather diagnostics in parallel, route by severity, file an incident record"

defaults:
  incident_dir: "./incidents"
  service: "checkout-service"

tasks:
  triage:
    desc: "Run this when the pager goes off"
    steps:
      - name: severity
        desc: "Interactive in a terminal; defaults make it CI-safe"
        func: prompt
        args:
          message: "Severity for {{ vars.service }} incident?"
          type: select
          options: [SEV1, SEV2, SEV3]
          default: "SEV2"

      - name: diagnostics
        desc: "Gather system state concurrently - seconds matter"
        parallel:
          - name: disk
            func: shell
            do: df -h / | tail -1 | awk '{print "root_disk_used="$5}'
            outputs:
              used: '{{ result.output | regexFind "root_disk_used=(.+)" }}'
          - name: load
            func: shell
            do: uptime | awk -F'load averages?:' '{print "load="$2}'
            outputs:
              load: '{{ result.output | regexFind "load=(.+)" }}'
          - name: top_procs
            func: shell
            do: ps aux | sort -rk 3 | head -4 | awk '{print $11}' | tail -3 | tr '\n' ',' 
            outputs:
              procs: "{{ result.output }}"

      - name: route
        desc: "Different severities trigger different playbooks"
        switch:
          value: "{{ steps.severity.value }}"
          cases:
            - when: SEV1
              then:
                - name: page_oncall
                  func: shell
                  do: 'echo "action=paged secondary on-call + opened war room channel"'
                  outputs: { action: '{{ result.output | regexFind "action=(.+)" }}' }
            - when: SEV2
              then:
                - name: open_ticket
                  func: shell
                  do: 'echo "action=opened priority ticket, on-call notified"'
                  outputs: { action: '{{ result.output | regexFind "action=(.+)" }}' }
          default:
            - name: log_only
              func: shell
              do: 'echo "action=logged for daily review"'
              outputs: { action: '{{ result.output | regexFind "action=(.+)" }}' }
        outputs:
          action: '{{ steps.page_oncall.action | default steps.open_ticket.action | default steps.log_only.action }}'

      - name: file_incident
        desc: "Write the incident record with everything gathered"
        func: shell
        do: |
          mkdir -p {{ vars.incident_dir }}
          ID=$(date +%Y%m%d-%H%M%S)
          cat > {{ vars.incident_dir }}/$ID.md <<R
          # Incident $ID - {{ vars.service }} ({{ steps.severity.value }})

          ## Action taken
          {{ steps.route.action }}

          ## Diagnostics at triage time
          - disk used: {{ steps.disk.used }}
          - load:      {{ steps.load.load }}
          - top procs: {{ steps.top_procs.procs }}
          R
          echo "record={{ vars.incident_dir }}/$ID.md"
        outputs:
          record: '{{ result.output | regexFind "record=(.+)" }}'

      - name: confirm
        func: assert
        args:
          conditions:
            - condition: '{{ ne steps.route.action "" }}'
              desc: "route produced an action"
            - condition: '{{ contains "incidents/" steps.file_incident.record }}'
              desc: "record path captured"

      - name: done
        func: shell
        do: 'echo "TRIAGED {{ steps.severity.value }} -> {{ steps.route.action }} ({{ steps.file_incident.record }})"'

orchstep run triage                      # interactive (defaults to SEV2)
orchstep run triage --var severity=SEV1  # pre-answer the prompt

TRIAGED SEV2 -> opened priority ticket, on-call notified (./incidents/...md)
TRIAGED SEV1 -> paged secondary on-call + opened war room channel (...)
TRIAGED SEV3 -> logged for daily review (...)

# Incident 20260611-222115 - checkout-service (SEV1)

## Action taken
paged secondary on-call + opened war room channel

## Diagnostics at triage time
- disk used: 1%
- load:      9.16 10.25 11.31
- top procs: ...