{"serverId":"mcp-registry/com.completionkit/evals","tools":[{"name":"agreements_create","fixed":{"name":"agreements_create","description":"Upsert an agreement for (run, response, metric, created_by). Verdict is one of agree, disagree, borderline. corrected_score (1..5) is required when verdict is 'disagree'.","inputSchema":{"type":"object","required":["run_id","response_id","metric_id","verdict"],"properties":{"note":{"type":"string"},"run_id":{"type":"integer"},"verdict":{"enum":["agree","disagree","borderline"],"type":"string"},"metric_id":{"type":"integer"},"created_by":{"type":"string"},"response_id":{"type":"integer"},"corrected_score":{"type":"number"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":160,"tokensAfter":160},{"name":"agreements_list","fixed":{"name":"agreements_list","description":"List agreements. Filter by run_id, response_id, metric_id, or created_by.","inputSchema":{"type":"object","required":[],"properties":{"run_id":{"type":"integer"},"metric_id":{"type":"integer"},"created_by":{"type":"string"},"response_id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":84,"tokensAfter":84},{"name":"datasets_create","fixed":{"name":"datasets_create","description":"Create a dataset with CSV data. First row is the header. Two column names are recognized specially: \"expected_output\" is each row's answer key (ground truth) given to the judge and to checks that compare against the row's expected value, and \"actual_output\" is a pre-made output to score in a prompt-less run. Both are overridable per run (expected_column / output_column). Every column is also available to the prompt as a variable.","inputSchema":{"type":"object","required":["name","csv_data"],"properties":{"name":{"type":"string"},"csv_data":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":186,"tokensAfter":186},{"name":"datasets_create_from_url","fixed":{"name":"datasets_create_from_url","description":"Create a dataset by downloading CSV from a URL instead of inlining it. Use this for large datasets: pass a public http(s) URL and the server fetches the CSV directly, so the data never has to pass through the tool-call arguments. The URL is SSRF-checked and the download is capped at 10MB. First row is the header; the \"expected_output\" (answer key) and \"actual_output\" (pre-made output) columns are recognized specially, overridable per run.","inputSchema":{"type":"object","required":["name","url"],"properties":{"url":{"type":"string","description":"Public http(s) URL of the CSV file to download."},"name":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":206,"tokensAfter":206},{"name":"datasets_delete","fixed":{"name":"datasets_delete","description":"Delete a dataset","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":true,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: true, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":42,"tokensAfter":42},{"name":"datasets_get","fixed":{"name":"datasets_get","description":"Get a dataset by ID","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":42,"tokensAfter":42},{"name":"datasets_list","fixed":{"name":"datasets_list","description":"List all datasets","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":34,"tokensAfter":34},{"name":"datasets_update","fixed":{"name":"datasets_update","description":"Update a dataset","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"name":{"type":"string"},"csv_data":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":72,"tokensAfter":72},{"name":"judges_compare","fixed":{"name":"judges_compare","description":"Compare two versions of one metric's agreement stats side by side. Requires metric_id, metric_version_a_id, and metric_version_b_id (both versions must belong to that metric). Unavailable for check metrics.","inputSchema":{"type":"object","required":["metric_id","metric_version_a_id","metric_version_b_id"],"properties":{"metric_id":{"type":"integer"},"metric_version_a_id":{"type":"integer"},"metric_version_b_id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":133,"tokensAfter":133},{"name":"metric_groups_create","fixed":{"name":"metric_groups_create","description":"Create a metric group","inputSchema":{"type":"object","required":["name"],"properties":{"name":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}},"metric_ids":{"type":"array","items":{"type":"integer"}},"description":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":85,"tokensAfter":85},{"name":"metric_groups_delete","fixed":{"name":"metric_groups_delete","description":"Delete a metric group","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":true,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: true, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":44,"tokensAfter":44},{"name":"metric_groups_get","fixed":{"name":"metric_groups_get","description":"Get a metric group by ID","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":44,"tokensAfter":44},{"name":"metric_groups_list","fixed":{"name":"metric_groups_list","description":"List all metric groups","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":37,"tokensAfter":37},{"name":"metric_groups_update","fixed":{"name":"metric_groups_update","description":"Update a metric group","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"name":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}},"metric_ids":{"type":"array","items":{"type":"integer"}},"description":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":91,"tokensAfter":91},{"name":"metric_versions_list","fixed":{"name":"metric_versions_list","description":"List every MetricVersion (drafts + published) for a metric, newest first. Each row carries version_number, state, source, current flag, and timestamps.","inputSchema":{"type":"object","required":["metric_id"],"properties":{"metric_id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":84,"tokensAfter":84},{"name":"metric_versions_publish","fixed":{"name":"metric_versions_publish","description":"Publish a MetricVersion as the live version of its metric. Works for both 'draft → published' and 'revert to an older published version → current'. Transactionally flips current, demotes peers, and writes the version's instruction + rubric_bands back onto the metric so the judge grades against it.","inputSchema":{"type":"object","required":["metric_version_id"],"properties":{"metric_version_id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":130,"tokensAfter":130},{"name":"metrics_create","fixed":{"name":"metrics_create","description":"Create a metric with evaluation criteria. For a deterministic check set metric_type:\"check\" and check_config. Per-kind required keys: value (contains/not_contains/equals/list_overlap/numeric_equals), pattern (regex), json_path+expected (json_path_equals), min and/or max (length_bounds/numeric_bounds); valid_json takes no extra keys. target_path is required when target is json_path; a target_path sent with no target sets target to json_path, and one sent with any other target is refused. target_p","inputSchema":{"type":"object","required":["name"],"properties":{"name":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}},"instruction":{"type":"string"},"metric_type":{"enum":["llm_judge","check"],"type":"string"},"check_config":{"type":"object","properties":{"max":{"type":"number"},"min":{"type":"number"},"trim":{"type":"boolean"},"value":{"type":["string","number","array"],"items":{"type":["string","number"]},"description":"Only list_overlap takes a list, as a JSON array or as comma-separated text; every other rule refuses one"},"target":{"enum":["response_text","input_data","json_path"],"type":"string"},"pattern":{"type":"string"},"expected":{},"score_by":{"enum":["recall","precision","f1","jaccard"],"type":"string"},"json_path":{"type":"string"},"multiline":{"type":"boolean"},"tolerance":{"type":["number","string"],"description":"A plain number for a fixed amount, or a percentage string such as \"2%\" for a share of the expected value"},"check_kind":{"enum":["contains","not_contains","equals","regex","valid_json","json_path_equals","length_bounds","list_overlap","numeric_bounds","numeric_equals"],"type":"string"},"compare_to":{"enum":["constant","expected"],"type":"string"},"target_path":{"type":"string"},"expected_path":{"type":"string"},"case_sensitive":{"type":"boolean","description":"A check with no case_sensitive matches case-insensitively, regex and list_overlap included; send case_sensitive: true to match case exactly. Before 0.29.0 a regex check read a missing case_sensitive as case-sensitive."}}},"rubric_bands":{"type":"array","items":{"type":"object","properties":{"stars":{"type":"integer"},"description":{"type":"string"}}}}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":622,"tokensAfter":622},{"name":"metrics_delete","fixed":{"name":"metrics_delete","description":"Delete a metric","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":true,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: true, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":41,"tokensAfter":41},{"name":"metrics_get","fixed":{"name":"metrics_get","description":"Get a metric by ID","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":41,"tokensAfter":41},{"name":"metrics_list","fixed":{"name":"metrics_list","description":"List all metrics","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":33,"tokensAfter":33},{"name":"metrics_update","fixed":{"name":"metrics_update","description":"Update a metric. For a deterministic check set metric_type:\"check\" and check_config. Per-kind required keys: value (contains/not_contains/equals/list_overlap/numeric_equals), pattern (regex), json_path+expected (json_path_equals), min and/or max (length_bounds/numeric_bounds); valid_json takes no extra keys. target_path is required when target is json_path; a target_path sent with no target sets target to json_path, and one sent with any other target is refused. target_path walks array indices, ","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"name":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}},"instruction":{"type":"string"},"metric_type":{"enum":["llm_judge","check"],"type":"string"},"check_config":{"type":"object","properties":{"max":{"type":"number"},"min":{"type":"number"},"trim":{"type":"boolean"},"value":{"type":["string","number","array"],"items":{"type":["string","number"]},"description":"Only list_overlap takes a list, as a JSON array or as comma-separated text; every other rule refuses one"},"target":{"enum":["response_text","input_data","json_path"],"type":"string"},"pattern":{"type":"string"},"expected":{},"score_by":{"enum":["recall","precision","f1","jaccard"],"type":"string"},"json_path":{"type":"string"},"multiline":{"type":"boolean"},"tolerance":{"type":["number","string"],"description":"A plain number for a fixed amount, or a percentage string such as \"2%\" for a share of the expected value"},"check_kind":{"enum":["contains","not_contains","equals","regex","valid_json","json_path_equals","length_bounds","list_overlap","numeric_bounds","numeric_equals"],"type":"string"},"compare_to":{"enum":["constant","expected"],"type":"string"},"target_path":{"type":"string"},"expected_path":{"type":"string"},"case_sensitive":{"type":"boolean","description":"A check with no case_sensitive matches case-insensitively, regex and list_overlap included; send case_sensitive: true to match case exactly. Before 0.29.0 a regex check read a missing case_sensitive as case-sensitive."}}},"rubric_bands":{"type":"array","items":{"type":"object","properties":{"stars":{"type":"integer"},"description":{"type":"string"}}}}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":628,"tokensAfter":628},{"name":"promptfoo_import","fixed":{"name":"promptfoo_import","description":"Import a promptfooconfig.yaml. Creates a prompt, a dataset from the test vars, and metrics from the assert blocks (llm-rubric/g-eval become judge metrics; contains/equals/regex/is-json become deterministic check metrics). Returns a summary of what mapped and what was skipped and why; nothing is dropped silently.","inputSchema":{"type":"object","required":["config"],"properties":{"config":{"type":"string","description":"The full promptfooconfig.yaml contents."}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":142,"tokensAfter":142},{"name":"prompts_create","fixed":{"name":"prompts_create","description":"Create a prompt","inputSchema":{"type":"object","required":["name","template","llm_model"],"properties":{"name":{"type":"string"},"template":{"type":"string"},"llm_model":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}},"description":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":89,"tokensAfter":89},{"name":"prompts_delete","fixed":{"name":"prompts_delete","description":"Delete a prompt","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":true,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: true, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":41,"tokensAfter":41},{"name":"prompts_get","fixed":{"name":"prompts_get","description":"Get a prompt by ID","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer","description":"Prompt ID"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":48,"tokensAfter":48},{"name":"prompts_list","fixed":{"name":"prompts_list","description":"List all prompts","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":33,"tokensAfter":33},{"name":"prompts_publish","fixed":{"name":"prompts_publish","description":"Publish a prompt version, making it the current version","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":52,"tokensAfter":52},{"name":"prompts_update","fixed":{"name":"prompts_update","description":"Update a prompt. If the prompt already has runs, this creates a new DRAFT version (current=false) rather than editing in place or publishing. Promote it with prompts_publish, so an agent's edits don't go live without a gate. If it has no runs, it is updated in place.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"name":{"type":"string"},"template":{"type":"string"},"llm_model":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}},"description":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":159,"tokensAfter":159},{"name":"provider_credentials_create","fixed":{"name":"provider_credentials_create","description":"Create a provider credential","inputSchema":{"type":"object","required":["provider","api_key"],"properties":{"api_key":{"type":"string"},"provider":{"enum":["openai","anthropic","ollama","openrouter","azure_foundry"],"type":"string"},"api_version":{"type":"string"},"api_endpoint":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":99,"tokensAfter":99},{"name":"provider_credentials_delete","fixed":{"name":"provider_credentials_delete","description":"Delete a provider credential","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":true,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: true, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":48,"tokensAfter":48},{"name":"provider_credentials_get","fixed":{"name":"provider_credentials_get","description":"Get a provider credential by ID (API key is not exposed)","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":55,"tokensAfter":55},{"name":"provider_credentials_list","fixed":{"name":"provider_credentials_list","description":"List all provider credentials (API keys are not exposed)","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":48,"tokensAfter":48},{"name":"provider_credentials_update","fixed":{"name":"provider_credentials_update","description":"Update a provider credential. Changing provider or api_endpoint on a credential with a stored key requires api_key in the same call, so a stored key never moves to a new provider or endpoint.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"api_key":{"type":"string","description":"Required when provider or api_endpoint changes and a key is stored"},"provider":{"type":"string"},"api_version":{"type":"string"},"api_endpoint":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":150,"tokensAfter":150},{"name":"responses_get","fixed":{"name":"responses_get","description":"Get a specific response","inputSchema":{"type":"object","required":["run_id","id"],"properties":{"id":{"type":"integer"},"run_id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":53,"tokensAfter":53},{"name":"responses_list","fixed":{"name":"responses_list","description":"List responses for a run, in row order. Returns {total, limit, offset, returned, responses}. Defaults to 50 rows because full payloads are large: use \"fields\" to drop the bodies, \"min_score\"/\"max_score\" to isolate low scorers, and sort \"score_asc\" to read the worst rows first. For per-metric averages of the whole run use runs_get instead of aggregating here.","inputSchema":{"type":"object","required":["run_id"],"properties":{"sort":{"enum":["id","score_asc","score_desc"],"type":"string","description":"Row order; defaults to \"id\"."},"limit":{"type":"integer","description":"Rows to return; defaults to 50, capped at 500."},"fields":{"type":"array","items":{"type":"string"},"description":"Only return these keys, keeping the payload small. Response keys: id, run_id, input_data, response_text, expected_output, created_at, score, reviewed, reviews, status, attempts, row_index, error. Prefix with \"reviews.\" to trim each review, e.g. [\"score\", \"reviews.metric_name\", \"reviews.ai_score\"]. id is always included."},"offset":{"type":"integer","description":"Rows to skip before returning results."},"run_id":{"type":"integer"},"status":{"type":"string","description":"Filter by row status: pending, retrying, succeeded or failed."},"max_score":{"type":"number","description":"Only rows whose average judge score is at most this. Use with sort \"score_asc\" for failure-mode analysis."},"min_score":{"type":"number","description":"Only rows whose average judge score is at least this."}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":430,"tokensAfter":430},{"name":"runs_create","fixed":{"name":"runs_create","description":"Create a run. Omit prompt_id and provide output_column to score existing outputs by grading a pre-existing dataset column instead of generating new ones.","inputSchema":{"type":"object","required":["name"],"properties":{"name":{"type":"string"},"prompt_id":{"type":"integer"},"tag_names":{"type":"array","items":{"type":"string"}},"dataset_id":{"type":"integer"},"max_tokens":{"type":"integer","description":"Cap on generated tokens per row. Leave unset to use the provider client's default, which is what silently truncates long outputs and makes the judge score malformed JSON. Set it to whatever the prompt uses in production so the eval matches."},"metric_ids":{"type":"array","items":{"type":"integer"}},"judge_model":{"type":"string"},"temperature":{"type":"number","description":"Sampling temperature for generation, 0 to 1. Leave it unset, which is the default, and no temperature is sent at all, so the model applies its own. Most current frontier models refuse the parameter outright; set it only when you are targeting a model that honours it, such as anything served locally through Ollama. A refused value is re-sent without one and the run is flagged temperature_ignored."},"output_column":{"type":"string","description":"Dataset column to grade when prompt_id is omitted; defaults to \"actual_output\"."},"expected_column":{"type":"string","description":"Dataset column holding each row's answer key / ground truth, graded by checks with compare_to \"expected\" and passed to the judge; defaults to \"expected_output\"."},"metric_group_id":{"type":"integer","description":"Attach the metrics belonging to this metric group (its current metric_ids). Ignored when metric_ids is also given."},"judge_temperature":{"type":"number","description":"Sampling temperature for the judge, 0 to 1. Defaults to 0 so re-judging the same output gives the same score. Raise it only to measure judge variance on purpose; any value above 0 makes the run's scores irreproducible."}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":561,"tokensAfter":561},{"name":"runs_delete","fixed":{"name":"runs_delete","description":"Delete a run","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":true,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: true, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":39,"tokensAfter":39},{"name":"runs_generate","fixed":{"name":"runs_generate","description":"Start a run. Required for every run, including score-only runs (no prompt): generates responses with the prompt when there is one, otherwise copies the graded dataset column and grades it.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":89,"tokensAfter":89},{"name":"runs_get","fixed":{"name":"runs_get","description":"Get a run by ID, including \"metric_averages\": a per-metric breakdown with each metric's average score (or pass rate for checks), how many rows it graded, and how many scored low. Use this to find the metric dragging a prompt down without listing responses.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":107,"tokensAfter":107},{"name":"runs_list","fixed":{"name":"runs_list","description":"List all runs","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":32,"tokensAfter":32},{"name":"runs_regrade","fixed":{"name":"runs_regrade","description":"Re-grade a run's existing responses with its currently attached metrics, without regenerating. Use after attaching or editing metrics on an already-generated run.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":81,"tokensAfter":81},{"name":"runs_rerun","fixed":{"name":"runs_rerun","description":"Create and start a fresh copy of a run with the same prompt, dataset, metrics, and settings. Use when the judge changed and you want a clean run instead of mixing versions.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":84,"tokensAfter":84},{"name":"runs_retry_failures","fixed":{"name":"runs_retry_failures","description":"Re-run only the failed responses of a run, optionally limited to specific response ids via \"only\".","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"only":{"type":"array","items":{"type":"integer"}}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":80,"tokensAfter":80},{"name":"runs_update","fixed":{"name":"runs_update","description":"Update a run","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"name":{"type":"string"},"tag_names":{"type":"array","items":{"type":"string"}},"dataset_id":{"type":"integer"},"max_tokens":{"type":"integer","description":"Cap on generated tokens per row. Leave unset to use the provider client's default, which is what silently truncates long outputs and makes the judge score malformed JSON. Set it to whatever the prompt uses in production so the eval matches."},"metric_ids":{"type":"array","items":{"type":"integer"}},"judge_model":{"type":"string"},"temperature":{"type":"number","description":"Sampling temperature for generation, 0 to 1. Leave it unset, which is the default, and no temperature is sent at all, so the model applies its own. Most current frontier models refuse the parameter outright; set it only when you are targeting a model that honours it, such as anything served locally through Ollama. A refused value is re-sent without one and the run is flagged temperature_ignored."},"output_column":{"type":"string"},"expected_column":{"type":"string"},"metric_group_id":{"type":"integer","description":"Replace the run's metrics with those belonging to this metric group. Ignored when metric_ids is also given."},"judge_temperature":{"type":"number","description":"Sampling temperature for the judge, 0 to 1. Defaults to 0 so re-judging the same output gives the same score. Raise it only to measure judge variance on purpose; any value above 0 makes the run's scores irreproducible."}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":440,"tokensAfter":440},{"name":"tags_create","fixed":{"name":"tags_create","description":"Create a tag. Color is auto-assigned.","inputSchema":{"type":"object","required":["name"],"properties":{"name":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":47,"tokensAfter":47},{"name":"tags_delete","fixed":{"name":"tags_delete","description":"Delete a tag. Removes the tag from every linked metric, prompt, run, and dataset.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":false,"destructiveHint":true,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: true, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":59,"tokensAfter":59},{"name":"tags_get","fixed":{"name":"tags_get","description":"Get a tag by ID","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"}}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":39,"tokensAfter":39},{"name":"tags_list","fixed":{"name":"tags_list","description":"List all tags","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":32,"tokensAfter":32},{"name":"tags_update","fixed":{"name":"tags_update","description":"Rename a tag.","inputSchema":{"type":"object","required":["id"],"properties":{"id":{"type":"integer"},"name":{"type":"string"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: true (inferred from the name).","verify":true}],"tokensBefore":47,"tokensAfter":47},{"name":"usage_get","fixed":{"name":"usage_get","description":"Get this organization's plan usage and limits for the current billing period: runs and prompt fetches used, their limits, how many remain, and when the period resets. Call this to pre-check quota before starting runs. Runs are hard-blocked once the run limit is reached (with a small grace band), so a run over the limit will fail with run_limit_reached.","inputSchema":{"type":"object","required":[],"properties":{}},"annotations":{"readOnlyHint":true}},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":126,"tokensAfter":126}],"toolsChanged":50,"tokensSaved":0,"todo":[{"tool":"agreements_create","params":["note","run_id","verdict","metric_id","created_by","response_id","corrected_score"]},{"tool":"agreements_list","params":["run_id","metric_id","created_by","response_id"]},{"tool":"datasets_create","params":["name","csv_data","tag_names"]},{"tool":"datasets_create_from_url","params":["name","tag_names"]},{"tool":"datasets_delete","params":["id"]},{"tool":"datasets_get","params":["id"]},{"tool":"datasets_update","params":["id","name","csv_data","tag_names"]},{"tool":"judges_compare","params":["metric_id","metric_version_a_id","metric_version_b_id"]},{"tool":"judges_replay","params":["name","metric_id","dataset_id","judge_model"]},{"tool":"metric_groups_create","params":["name","tag_names","metric_ids","description"]},{"tool":"metric_groups_delete","params":["id"]},{"tool":"metric_groups_get","params":["id"]},{"tool":"metric_groups_update","params":["id","name","tag_names","metric_ids","description"]},{"tool":"metric_versions_dismiss","params":["metric_version_id"]},{"tool":"metric_versions_list","params":["metric_id"]},{"tool":"metric_versions_publish","params":["metric_version_id"]},{"tool":"metrics_create","params":["name","tag_names","instruction","metric_type","check_config","rubric_bands"]},{"tool":"metrics_delete","params":["id"]},{"tool":"metrics_get","params":["id"]},{"tool":"metrics_suggest_variants","params":["metric_id"]},{"tool":"metrics_update","params":["id","name","tag_names","instruction","metric_type","check_config","rubric_bands"]},{"tool":"prompts_create","params":["name","template","llm_model","tag_names","description"]},{"tool":"prompts_delete","params":["id"]},{"tool":"prompts_publish","params":["id"]},{"tool":"prompts_update","params":["id","name","template","llm_model","tag_names","description"]},{"tool":"provider_credentials_create","params":["api_key","provider","api_version","api_endpoint"]},{"tool":"provider_credentials_delete","params":["id"]},{"tool":"provider_credentials_get","params":["id"]},{"tool":"provider_credentials_update","params":["id","provider","api_version","api_endpoint"]},{"tool":"responses_get","params":["id","run_id"]},{"tool":"responses_list","params":["run_id"]},{"tool":"runs_create","params":["name","prompt_id","tag_names","dataset_id","metric_ids","judge_model"]},{"tool":"runs_delete","params":["id"]},{"tool":"runs_generate","params":["id"]},{"tool":"runs_get","params":["id"]},{"tool":"runs_regrade","params":["id"]},{"tool":"runs_rerun","params":["id"]},{"tool":"runs_retry_failures","params":["id","only"]},{"tool":"runs_update","params":["id","name","tag_names","dataset_id","metric_ids","judge_model","output_column","expected_column"]},{"tool":"tags_create","params":["name"]},{"tool":"tags_delete","params":["id"]},{"tool":"tags_get","params":["id"]},{"tool":"tags_update","params":["id","name"]}],"note":"Deterministic fixes. Changes marked verify need a human look before shipping (renames, inferred annotations, inferred types, removed text). Parameter descriptions in `todo` need to be written by someone who knows the API."}