Room 55

# Deploy the crashing app
kubectl apply -f crashing-app.yaml

# Check pod status - it will be CrashLoopBackOff
kubectl get pod mystery-crash -w

# Read the crash logs
kubectl logs mystery-crash
# Output:
# Starting application...
# Loading config from /config/app.conf
# FATAL: Config file missing!

# Check previous container logs (after restart)
kubectl logs mystery-crash --previous

# Get detailed pod events
kubectl describe pod mystery-crash
kubectl get events --field-selector involvedObject.name=mystery-crash --sort-by=.lastTimestamp

# Attach an ephemeral debug container to inspect the pod
kubectl debug mystery-crash -it \
  --image=busybox:1.36 \
  --target=app \
  -- sh

# Inside the debug container:
# - Check what's in /config
ls -la /config/
# - Look at the crash log
cat /tmp/crash.log
# Output: Debug hint: debug55k
# - Check processes
ps aux
# - Check environment
env | sort
# - Exit
exit

# Create a copy of the pod with a debug-friendly image
kubectl debug mystery-crash \
  --copy-to=mystery-debug \
  --image=ubuntu:22.04 \
  --share-processes \
  -- sleep 3600

# Exec into the debug copy
kubectl exec -it mystery-debug -- bash

# Inside:
apt-get update && apt-get install -y procps net-tools curl strace
# Trace system calls
strace -p 1 -f -e trace=open,read,write 2>&1 | head -50
# Check network
netstat -tlnp
# Check DNS
cat /etc/resolv.conf
nslookup kubernetes.default.svc.cluster.local
exit

# Get the node name
NODE=$(kubectl get pod mystery-crash -o jsonpath='{.spec.nodeName}')

# Create a debug pod on that node
kubectl debug "node/$NODE" -it \
  --image=busybox:1.36 \
  -- sh

# Inside the node debug pod:
# The node's filesystem is mounted at /host
ls /host/var/log/pods/
ls /host/var/log/containers/

# Read container logs directly from the node
cat /host/var/log/pods/default_mystery-crash_*/app/*.log

# Check kubelet logs
cat /host/var/log/syslog | grep kubelet | tail -20

# Check node resource usage
cat /host/proc/meminfo | head -5
cat /host/proc/cpuinfo | grep "model name" | head -1
exit

# Deploy a network debug pod
kubectl run netdebug --image=nicolaka/netshoot --command -- sleep 3600
kubectl wait --for=condition=ready pod/netdebug

# Exec in and debug networking
kubectl exec -it netdebug -- bash

# DNS resolution
dig kubernetes.default.svc.cluster.local
dig +search api-service

# Check connectivity
curl -v http://api-service:8080 2>&1 | head -20

# Capture traffic (requires NET_ADMIN)
tcpdump -i any -c 50 port 80 -w /tmp/capture.pcap

# Trace route to another pod
traceroute <pod-ip>

# Check iptables rules (service routing)
iptables -t nat -L KUBE-SERVICES | head -20
exit

# Now fix the crashing app - create the missing config
kubectl create configmap app-config --from-literal=app.conf="mode=production"

# Patch the pod spec to use the ConfigMap
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: mystery-fixed
  labels:
    app: mystery
spec:
  containers:
    - name: app
      image: busybox:1.36
      command:
        - sh
        - -c
        - |
          echo "Starting application..."
          echo "Loading config from /config/app.conf"
          if [ ! -f /config/app.conf ]; then
            echo "FATAL: Config file missing!"
            exit 1
          fi
          echo "Config loaded: \$(cat /config/app.conf)"
          echo "Application running successfully"
          sleep 3600
      volumeMounts:
        - name: config
          mountPath: /config
  volumes:
    - name: config
      configMap:
        name: app-config
EOF

# Verify the fix
kubectl logs mystery-fixed
# Output:
# Starting application...
# Loading config from /config/app.conf
# Config loaded: mode=production
# Application running successfully