{
  "generatedAt": "2026-09-21T18:31:30.860Z",
  "endpoint": "https://openrouter.ai/api/v1/systemone",
  "model": "jev-1.13",
  "servedModel": "typesafe/jev-1.13-20260917",
  "concurrency": 6,
  "methodology": "Real API calls on 26 synthetic, author-labelled EasyChamp chat inputs. The JavaScript regex baseline was ported from the Python router. This measures classification on this corpus, not production accuracy, end-to-end latency, or developer time saved.",
  "source": "0878-jev-system-one-integration/reports/jev-bench.json",
  "suite": {
    "name": "easychamp-intent",
    "title": "EasyChamp chat intent routing (26 messages)",
    "baselineName": "INTENT_PATTERNS regex router (intent_router.py, ported)",
    "items": 26,
    "stats": {
      "calls": 26,
      "failures": 0,
      "latency": {
        "p50": 258,
        "p95": 333,
        "mean": 268,
        "min": 170,
        "max": 577
      },
      "input_tokens": 17584,
      "output_tokens": 2804,
      "cost": 0.00073853,
      "costPer1kItems": 0.0284
    },
    "metrics": {
      "jev": {
        "correct": 25,
        "total": 26,
        "accuracy": 0.9615
      },
      "baseline": {
        "correct": 12,
        "total": 26,
        "accuracy": 0.4615
      }
    },
    "rows": [
      {
        "id": "i01",
        "message": "show me all my leagues",
        "expected": "get_leagues",
        "jev": "get_leagues",
        "jevConfidence": 1,
        "jevTop": "get_leagues=1, simulate_match=0, simulate_games=0",
        "regex": "get_leagues",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 326,
        "error": null
      },
      {
        "id": "i02",
        "message": "what leagues do i have?",
        "expected": "get_leagues",
        "jev": "get_leagues",
        "jevConfidence": 1,
        "jevTop": "get_leagues=1, simulate_match=0, get_standings=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 295,
        "error": null
      },
      {
        "id": "i03",
        "message": "leagues pls",
        "expected": "get_leagues",
        "jev": "get_leagues",
        "jevConfidence": 0.88,
        "jevTop": "get_leagues=0.9, create_league=0.09, complex_query=0.01",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 333,
        "error": null
      },
      {
        "id": "i04",
        "message": "покажи мои лиги",
        "expected": "get_leagues",
        "jev": "get_leagues",
        "jevConfidence": 1,
        "jevTop": "get_leagues=1, simulate_match=0, get_standings=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 314,
        "error": null
      },
      {
        "id": "i05",
        "message": "list my competitions",
        "expected": "get_competitions",
        "jev": "get_competitions",
        "jevConfidence": 1,
        "jevTop": "get_competitions=1, get_leagues=0, simulate_games=0",
        "regex": "get_competitions",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 320,
        "error": null
      },
      {
        "id": "i06",
        "message": "which tournaments am i running?",
        "expected": "get_competitions",
        "jev": "get_competitions",
        "jevConfidence": 0.99,
        "jevTop": "get_competitions=0.99, get_leagues=0.01, disambiguation=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 298,
        "error": null
      },
      {
        "id": "i07",
        "message": "muéstrame mis competiciones",
        "expected": "get_competitions",
        "jev": "get_competitions",
        "jevConfidence": 0.99,
        "jevTop": "get_competitions=0.99, get_leagues=0.01, disambiguation=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 272,
        "error": null
      },
      {
        "id": "i08",
        "message": "show teams",
        "expected": "get_teams",
        "jev": "get_teams",
        "jevConfidence": 1,
        "jevTop": "get_teams=1, simulate_match=0, get_leagues=0",
        "regex": "get_teams",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 270,
        "error": null
      },
      {
        "id": "i09",
        "message": "shwo me my teams",
        "expected": "get_teams",
        "jev": "get_teams",
        "jevConfidence": 0.99,
        "jevTop": "get_teams=0.99, get_leagues=0.01, complex_query=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 257,
        "error": null
      },
      {
        "id": "i10",
        "message": "standings for the Miami Cup",
        "expected": "get_standings",
        "jev": "get_standings",
        "jevConfidence": 1,
        "jevTop": "get_standings=1, get_competitions=0, get_leagues=0",
        "regex": "get_standings",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 269,
        "error": null
      },
      {
        "id": "i11",
        "message": "how is my table looking in the winter league?",
        "expected": "get_standings",
        "jev": "get_standings",
        "jevConfidence": 1,
        "jevTop": "get_standings=1, disambiguation=0, get_leagues=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 250,
        "error": null
      },
      {
        "id": "i12",
        "message": "tabla de posiciones de la liga de invierno",
        "expected": "get_standings",
        "jev": "get_standings",
        "jevConfidence": 1,
        "jevTop": "get_standings=1, get_leagues=0, get_competitions=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 237,
        "error": null
      },
      {
        "id": "i13",
        "message": "create a soccer league \"Sunny Isles Winter\"",
        "expected": "create_league",
        "jev": "create_league",
        "jevConfidence": 1,
        "jevTop": "create_league=1, simulate_games=0, get_teams=0",
        "regex": "create_league",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 235,
        "error": null
      },
      {
        "id": "i14",
        "message": "start a new basketball league called Ocean Drive",
        "expected": "create_league",
        "jev": "create_league",
        "jevConfidence": 1,
        "jevTop": "create_league=1, get_standings=0, get_leagues=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 184,
        "error": null
      },
      {
        "id": "i15",
        "message": "simulate all games",
        "expected": "simulate_games",
        "jev": "simulate_games",
        "jevConfidence": 1,
        "jevTop": "simulate_games=1, enter_result=0, create_league=0",
        "regex": "simulate_games",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 236,
        "error": null
      },
      {
        "id": "i16",
        "message": "simulate every remaining fixture in the cup",
        "expected": "simulate_games",
        "jev": "simulate_games",
        "jevConfidence": 1,
        "jevTop": "simulate_games=1, enter_result=0, create_league=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 262,
        "error": null
      },
      {
        "id": "i17",
        "message": "sim the match Inter Miami vs Orlando City",
        "expected": "simulate_match",
        "jev": "simulate_match",
        "jevConfidence": 1,
        "jevTop": "simulate_match=1, get_teams=0, simulate_games=0",
        "regex": "simulate_match",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 180,
        "error": null
      },
      {
        "id": "i18",
        "message": "симулируй матч Интер Майами против Орландо",
        "expected": "simulate_match",
        "jev": "simulate_match",
        "jevConfidence": 1,
        "jevTop": "simulate_match=1, get_standings=0, get_leagues=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 240,
        "error": null
      },
      {
        "id": "i19",
        "message": "simula el partido Barcelona vs Madrid",
        "expected": "simulate_match",
        "jev": "simulate_match",
        "jevConfidence": 1,
        "jevTop": "simulate_match=1, get_standings=0, get_teams=0",
        "regex": "complex_query",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 231,
        "error": null
      },
      {
        "id": "i20",
        "message": "Inter Miami 3-2 Orlando City",
        "expected": "enter_result",
        "jev": "enter_result",
        "jevConfidence": 1,
        "jevTop": "enter_result=1, get_standings=0, complex_query=0",
        "regex": "enter_result",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 178,
        "error": null
      },
      {
        "id": "i21",
        "message": "record that Barcelona beat Madrid 2-1",
        "expected": "enter_result",
        "jev": "enter_result",
        "jevConfidence": 1,
        "jevTop": "enter_result=1, get_standings=0, create_league=0",
        "regex": "enter_result",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 288,
        "error": null
      },
      {
        "id": "i22",
        "message": "Реал 2:1 Барселона",
        "expected": "enter_result",
        "jev": "enter_result",
        "jevConfidence": 1,
        "jevTop": "enter_result=1, complex_query=0, simulate_games=0",
        "regex": "enter_result",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 287,
        "error": null
      },
      {
        "id": "i23",
        "message": "Messi scored at minute 67, final score 1-1",
        "expected": "complex_query",
        "jev": "enter_result",
        "jevConfidence": 0.46,
        "jevTop": "enter_result=0.52, complex_query=0.48, get_leagues=0",
        "regex": "complex_query",
        "jevCorrect": false,
        "regexCorrect": true,
        "latencyMs": 577,
        "error": null
      },
      {
        "id": "i24",
        "message": "the second one",
        "expected": "disambiguation",
        "jev": "disambiguation",
        "jevConfidence": 1,
        "jevTop": "disambiguation=1, simulate_games=0, get_teams=0",
        "regex": "disambiguation",
        "jevCorrect": true,
        "regexCorrect": true,
        "latencyMs": 258,
        "error": null
      },
      {
        "id": "i25",
        "message": "don't show me my teams, show me the fixtures instead",
        "expected": "complex_query",
        "jev": "complex_query",
        "jevConfidence": 0.91,
        "jevTop": "complex_query=0.93, simulate_games=0.03, get_competitions=0.02",
        "regex": "get_teams",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 170,
        "error": null
      },
      {
        "id": "i26",
        "message": "explain what simulate all games does before I run it",
        "expected": "complex_query",
        "jev": "complex_query",
        "jevConfidence": 0.46,
        "jevTop": "complex_query=0.51, simulate_games=0.49, get_competitions=0",
        "regex": "simulate_games",
        "jevCorrect": true,
        "regexCorrect": false,
        "latencyMs": 194,
        "error": null
      }
    ],
    "misses": [
      {
        "id": "i23",
        "input": "Messi scored at minute 67, final score 1-1",
        "expected": "complex_query",
        "got": "enter_result",
        "probabilities": "enter_result=0.52, complex_query=0.48, get_leagues=0, get_teams=0",
        "regexSaid": "complex_query",
        "error": null
      }
    ]
  }
}