JVM Konteynerde CPU — Ortalama Düşük, İstek Neden Yavaş?

1
apiVersion: apps/v1
2
kind: Deployment
3
metadata:
4
  name: payments
5
spec:
6
  template:
7
    spec:
8
      containers:
9
        - name: payments
10
          image: registry.bank.local/payments:1.42.0
11
          resources:
12
            requests:
13
              cpu: "1"        # what the scheduler guarantees
14
              memory: 1Gi
15
            limits:
16
              cpu: "2"        # the quota per 100 ms period: 200 ms of CPU time
17
              memory: 1Gi
18
          env:
19
            - name: JAVA_TOOL_OPTIONS
20
              value: "-XX:ActiveProcessorCount=2 -XX:MaxRAMPercentage=75"

1
@Component
2
class CpuThrottleMetrics {
3

4
    private static final Path CPU_STAT = Path.of("/sys/fs/cgroup/cpu.stat"); // cgroup v2
5

6
    CpuThrottleMetrics(MeterRegistry registry) {
7
        FunctionCounter.builder("container.cpu.throttled.periods", this, m -> m.read("nr_throttled"))
8
                .description("Periods in which the container used up its CPU quota")
9
                .register(registry);
10
        FunctionCounter.builder("container.cpu.throttled.seconds", this, m -> m.read("throttled_usec") / 1_000_000.0)
11
                .description("Total time the container's threads waited for the next period")
12
                .register(registry);
13
    }
14

15
    private double read(String key) {
16
        try (Stream<String> lines = Files.lines(CPU_STAT)) {
17
            return lines.map(line -> line.split(" "))
18
                    .filter(parts -> parts[0].equals(key))
19
                    .mapToDouble(parts -> Double.parseDouble(parts[1]))
20
                    .findFirst()
21
                    .orElse(0);
22
        } catch (IOException e) {
23
            return 0; // not running in a cgroup v2 container
24
        }
25
    }
26
}

1
groups:
2
  - name: payments-cpu
3
    rules:
4
      - alert: PaymentsCpuThrottled
5
        # More than a quarter of periods throttled while p99 latency is high.
6
        expr: |
7
          rate(container_cpu_cfs_throttled_periods_total{container="payments"}[5m])
8
            / rate(container_cpu_cfs_periods_total{container="payments"}[5m]) > 0.25
9
          and histogram_quantile(0.99, rate(http_server_requests_seconds_bucket{app="payments"}[5m])) > 0.5
10
        for: 10m
11
        labels:
12
          severity: warning