İstemi Ölçmek — Yeni Sürüm Gerçekten Daha mı İyi?

1
{"text": "Kartım ATM'de kaldı", "team": "CARD"}
2
{"text": "Temassız ödeme çalışmıyor", "team": "CARD"}
3
{"text": "IBAN'ı yanlış yazdım", "team": "TRANSFER"}
4
{"text": "Kredi taksitim yanlış", "team": "LOAN"}
5
{"text": "Tanımadığım bir harcama var", "team": "FRAUD"}
6
{"text": "Biri bankayı arayıp şifremi istedi", "team": "FRAUD"}

1
public final class RoutingEval {
2

3
    public record Example(String text, Team team) {}
4

5
    public record Slice(int correct, int total) {
6
        double rate() {
7
            return total == 0 ? 1.0 : (double) correct / total;
8
        }
9
    }
10

11
    private final ComplaintRouter router;
12

13
    public RoutingEval(ComplaintRouter router) {
14
        this.router = router;
15
    }
16

17
    /** Runs every example and scores each team separately. */
18
    public Map<Team, Slice> run(List<Example> examples, String promptVersion) {
19
        Map<Team, int[]> counts = new EnumMap<>(Team.class);
20
        for (Example example : examples) {
21
            Team predicted = router.route(example.text(), promptVersion);
22
            int[] c = counts.computeIfAbsent(example.team(), t -> new int[2]);
23
            if (predicted == example.team()) {
24
                c[0]++;
25
            }
26
            c[1]++;
27
        }
28
        Map<Team, Slice> slices = new EnumMap<>(Team.class);
29
        counts.forEach((team, c) -> slices.put(team, new Slice(c[0], c[1])));
30
        return slices;
31
    }
32
}

1
@Tag("eval")
2
class PromptChangeTest {
3

4
    private final RoutingEval eval = new RoutingEval(TestRouters.real());
5
    private final List<RoutingEval.Example> examples = EvalFiles.read("eval/complaints.jsonl");
6

7
    @Test
8
    void the_new_prompt_does_not_get_worse_for_any_team() {
9
        Map<Team, RoutingEval.Slice> before = eval.run(examples, "v1");
10
        Map<Team, RoutingEval.Slice> after = eval.run(examples, "v2");
11

12
        List<String> regressions = before.keySet().stream()
13
                .filter(team -> after.get(team).rate() < before.get(team).rate())
14
                .map(team -> team + ": " + before.get(team) + " -> " + after.get(team))
15
                .toList();
16

17
        // An overall score could go up while FRAUD drops; check every slice.
18
        assertThat(regressions).isEmpty();
19
    }
20
}