Eval runs

Create an eval suite (author-only, does not run)

Creates a runnable eval suite — the suite record plus its test cases — and responds 201 synchronously, WITHOUT executing anything. Use this to author a suite, then run it later with POST /eval-runs (passing the returned suiteId).

This is distinct from POST /eval-runs, which creates a run and detaches execution, responding 202 with a runId. There is no concurrency cap here (no run is started).

The body uses an ergonomic authoring shape: a suite-level default model (and optional provider) applies to every test unless the test overrides it; provider is derived from a provider/model id when neither is supplied. Each test’s case body is an ordered steps array (prompt / toolCall / interact / assert).

Guest callers are denied (suite creation is a write).

POST

projects

{projectId}

eval-suites

Create an eval suite (author-only, does not run)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "name": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "model": "anthropic/claude-haiku-4.5",
  "tests": [
    {
      "title": "lists files",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "List the files in the project root."
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "list_files",
            "args": {
              "args": {}
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"

payload = {
    "name": "smoke",
    "serverIds": ["srv_abc123"],
    "model": "anthropic/claude-haiku-4.5",
    "tests": [
        {
            "title": "lists files",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "List the files in the project root."
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "list_files",
                        "args": { "args": {} }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: 'smoke',
    serverIds: ['srv_abc123'],
    model: 'anthropic/claude-haiku-4.5',
    tests: [
      {
        title: 'lists files',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'List the files in the project root.'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'list_files', args: {args: {}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'model' => 'anthropic/claude-haiku-4.5',
    'tests' => [
        [
                'title' => 'lists files',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'List the files in the project root.'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'list_files',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"

	payload := strings.NewReader("{\n  \"name\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"model\": \"anthropic/claude-haiku-4.5\",\n  \"tests\": [\n    {\n      \"title\": \"lists files\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"List the files in the project root.\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"list_files\",\n            \"args\": {\n              \"args\": {}\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"model\": \"anthropic/claude-haiku-4.5\",\n  \"tests\": [\n    {\n      \"title\": \"lists files\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"List the files in the project root.\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"list_files\",\n            \"args\": {\n              \"args\": {}\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"model\": \"anthropic/claude-haiku-4.5\",\n  \"tests\": [\n    {\n      \"title\": \"lists files\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"List the files in the project root.\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"list_files\",\n            \"args\": {\n              \"args\": {}\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "suiteId": "<string>",
  "name": "<string>",
  "caseUpsert": {
    "committed": [
      {
        "id": "<string>",
        "name": "<string>"
      }
    ],
    "failed": [
      {
        "id": "<string>",
        "name": "<string>",
        "error": "<string>"
      }
    ]
  },
  "servers": [
    {
      "id": "<string>",
      "name": "<string>"
    }
  ],
  "hosts": [
    {
      "id": "<string>",
      "name": "<string>"
    }
  ]
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

Body

application/json

Author-only suite-create body. A suite-level default model (and optional provider) applies to every test unless the test overrides it.

name

string

required

Suite name.

Minimum string length: 1

serverIds

string[]

required

Servers (by canonical project ID) the suite's cases run against.

Minimum array length: 1

model

string

required

Suite-level default model id (e.g. anthropic/claude-haiku-4.5). Used for any test that omits model.

Minimum string length: 1

tests

object[]

required

Test cases to create in the suite. The object is CLOSED: an unknown key is a VALIDATION_ERROR, never a silently dropped field. repetitions, passThreshold and kind are per-case policy fields this surface cannot author — set them through POST /v1/projects/{projectId}/eval-suites/{suiteId}/cases.

Required array length: 1 - 100 elements

Show child attributes

description

string

serverNames

string[]

Optional display names, parallel to serverIds.

hosts

object[]

Clients (hosts) to attach the suite to, in attach order. A suite with no attached client reports none wherever it is listed, and a run has no host to select. Same shape as the PATCH hosts field.

Show child attributes

provider

string

Optional suite-level default provider. When omitted, the provider is derived from a provider/model id.

passCriteria

object

The floor a run must clear, as a PERCENT in [0, 100] — 80 means 80%. minimumPassRatePercent is the canonical spelling (the unit is in the name); minimumPassRate is the deprecated alias for it. Send exactly one. THE NAME DISAMBIGUATES THE UNIT: a value in (0, 1) is accepted on minimumPassRatePercent, where it unambiguously means a sub-1% floor the evaluator can act on, and REJECTED on the bare minimumPassRate, where 0.8 cannot be told apart from a fraction sent by mistake and would silently make the gate unfailable. Neither field ever reinterprets a value. A per-case passThreshold IS a fraction; this suite/run floor is not.

Option 1
Option 2

Show child attributes

Response

The suite was created.

suiteId

string

required

name

string

required

caseUpsert

object

required

Per-case create outcomes. Partial failures don't abort the suite; an all-failed new suite is rejected with VALIDATION_ERROR.

Show child attributes

servers

object[]

The servers attached to the suite. name is present when supplied.

Show child attributes

hosts

object[]

The clients (hosts) attached to the suite at create time. Empty when none were named.

Show child attributes

⌘I

Create an eval suite (author-only, does not run)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "name": "smoke",
  "serverIds": [
    "srv_abc123"
  ],
  "model": "anthropic/claude-haiku-4.5",
  "tests": [
    {
      "title": "lists files",
      "steps": [
        {
          "id": "s1",
          "kind": "prompt",
          "prompt": "List the files in the project root."
        },
        {
          "id": "s2",
          "kind": "assert",
          "assertion": {
            "type": "toolCalledWith",
            "toolName": "list_files",
            "args": {
              "args": {}
            }
          }
        }
      ]
    }
  ]
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"

payload = {
    "name": "smoke",
    "serverIds": ["srv_abc123"],
    "model": "anthropic/claude-haiku-4.5",
    "tests": [
        {
            "title": "lists files",
            "steps": [
                {
                    "id": "s1",
                    "kind": "prompt",
                    "prompt": "List the files in the project root."
                },
                {
                    "id": "s2",
                    "kind": "assert",
                    "assertion": {
                        "type": "toolCalledWith",
                        "toolName": "list_files",
                        "args": { "args": {} }
                    }
                }
            ]
        }
    ]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: 'smoke',
    serverIds: ['srv_abc123'],
    model: 'anthropic/claude-haiku-4.5',
    tests: [
      {
        title: 'lists files',
        steps: [
          {id: 's1', kind: 'prompt', prompt: 'List the files in the project root.'},
          {
            id: 's2',
            kind: 'assert',
            assertion: {type: 'toolCalledWith', toolName: 'list_files', args: {args: {}}}
          }
        ]
      }
    ]
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => 'smoke',
    'serverIds' => [
        'srv_abc123'
    ],
    'model' => 'anthropic/claude-haiku-4.5',
    'tests' => [
        [
                'title' => 'lists files',
                'steps' => [
                                [
                                                                'id' => 's1',
                                                                'kind' => 'prompt',
                                                                'prompt' => 'List the files in the project root.'
                                ],
                                [
                                                                'id' => 's2',
                                                                'kind' => 'assert',
                                                                'assertion' => [
                                                                                                                                'type' => 'toolCalledWith',
                                                                                                                                'toolName' => 'list_files',
                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                'args' => [
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                ]
                                                                                                                                ]
                                                                ]
                                ]
                ]
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites"

	payload := strings.NewReader("{\n  \"name\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"model\": \"anthropic/claude-haiku-4.5\",\n  \"tests\": [\n    {\n      \"title\": \"lists files\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"List the files in the project root.\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"list_files\",\n            \"args\": {\n              \"args\": {}\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"model\": \"anthropic/claude-haiku-4.5\",\n  \"tests\": [\n    {\n      \"title\": \"lists files\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"List the files in the project root.\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"list_files\",\n            \"args\": {\n              \"args\": {}\n            }\n          }\n        }\n      ]\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"smoke\",\n  \"serverIds\": [\n    \"srv_abc123\"\n  ],\n  \"model\": \"anthropic/claude-haiku-4.5\",\n  \"tests\": [\n    {\n      \"title\": \"lists files\",\n      \"steps\": [\n        {\n          \"id\": \"s1\",\n          \"kind\": \"prompt\",\n          \"prompt\": \"List the files in the project root.\"\n        },\n        {\n          \"id\": \"s2\",\n          \"kind\": \"assert\",\n          \"assertion\": {\n            \"type\": \"toolCalledWith\",\n            \"toolName\": \"list_files\",\n            \"args\": {\n              \"args\": {}\n            }\n          }\n        }\n      ]\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "suiteId": "<string>",
  "name": "<string>",
  "caseUpsert": {
    "committed": [
      {
        "id": "<string>",
        "name": "<string>"
      }
    ],
    "failed": [
      {
        "id": "<string>",
        "name": "<string>",
        "error": "<string>"
      }
    ]
  },
  "servers": [
    {
      "id": "<string>",
      "name": "<string>"
    }
  ],
  "hosts": [
    {
      "id": "<string>",
      "name": "<string>"
    }
  ]
}

API Guides

Endpoints

Create an eval suite (author-only, does not run)

Authorizations

Path Parameters

Body

Response