Playground

function evaluateFlightBooking(
  input: string,
  output: string,
  state: AgentState
): EvalResult {
  let score = 1.0;
  const issues: string[] = [];

  // Check if agent called searchFlights with correct params
  const searchCall = state.toolCalls.find(
    t => t.name === 'searchFlights'
  );

  if (!searchCall) {
    score -= 0.5;
    issues.push('Missing searchFlights call');
  } else {
    // Validate parameters
    if (searchCall.params.from !== 'SF') {
      score -= 0.2;
      issues.push('Incorrect departure city');
    }
    if (searchCall.params.to !== 'NYC') {
      score -= 0.2;
      issues.push('Incorrect destination city');
    }
  }

  // Check if agent filtered by time preference
  const filterCall = state.toolCalls.find(
    t => t.name === 'filterFlights'
  );

  if (!filterCall) {
    score -= 0.3;
    issues.push('Did not filter by time preference');
  }

  // Check for hallucinated information
  const mentionedPrices = extractPrices(output);
  const actualPrices = state.toolResults.map(r => r.price);

  const hallucinated = mentionedPrices.filter(
    p => !actualPrices.includes(p)
  );

  if (hallucinated.length > 0) {
    score -= 0.4;
    issues.push(`Hallucinated prices: ${hallucinated.join(', ')}`);
  }

  return {
    score: Math.max(0, score),
    issues,
    passed: score >= 0.7
  };
}