Run a multi-turn dry-run scenario

curl --request POST \
  --url https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "scenario": {
    "turns": [
      {
        "user_msg": "<string>"
      }
    ],
    "expected_tool_calls": [
      "<string>"
    ],
    "must_contain": [
      "<string>"
    ],
    "must_not_contain": [
      "<string>"
    ]
  },
  "mock_tools": true
}
'

import requests

url = "https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run"

payload = {
    "scenario": {
        "turns": [{ "user_msg": "<string>" }],
        "expected_tool_calls": ["<string>"],
        "must_contain": ["<string>"],
        "must_not_contain": ["<string>"]
    },
    "mock_tools": True
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    scenario: {
      turns: [{user_msg: '<string>'}],
      expected_tool_calls: ['<string>'],
      must_contain: ['<string>'],
      must_not_contain: ['<string>']
    },
    mock_tools: true
  })
};

fetch('https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'scenario' => [
        'turns' => [
                [
                                'user_msg' => '<string>'
                ]
        ],
        'expected_tool_calls' => [
                '<string>'
        ],
        'must_contain' => [
                '<string>'
        ],
        'must_not_contain' => [
                '<string>'
        ]
    ],
    'mock_tools' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run"

	payload := strings.NewReader("{\n  \"scenario\": {\n    \"turns\": [\n      {\n        \"user_msg\": \"<string>\"\n      }\n    ],\n    \"expected_tool_calls\": [\n      \"<string>\"\n    ],\n    \"must_contain\": [\n      \"<string>\"\n    ],\n    \"must_not_contain\": [\n      \"<string>\"\n    ]\n  },\n  \"mock_tools\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"scenario\": {\n    \"turns\": [\n      {\n        \"user_msg\": \"<string>\"\n      }\n    ],\n    \"expected_tool_calls\": [\n      \"<string>\"\n    ],\n    \"must_contain\": [\n      \"<string>\"\n    ],\n    \"must_not_contain\": [\n      \"<string>\"\n    ]\n  },\n  \"mock_tools\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"scenario\": {\n    \"turns\": [\n      {\n        \"user_msg\": \"<string>\"\n      }\n    ],\n    \"expected_tool_calls\": [\n      \"<string>\"\n    ],\n    \"must_contain\": [\n      \"<string>\"\n    ],\n    \"must_not_contain\": [\n      \"<string>\"\n    ]\n  },\n  \"mock_tools\": true\n}"

response = http.request(request)
puts response.read_body

{
  "data": {
    "agent_id": "agt_01J9Z8ABCDEF",
    "mock_tools": true,
    "status": "pass",
    "duration_ms": 4210,
    "trace": [
      {
        "index": 0,
        "user_msg": "Where is my order?",
        "assistant_response": "Your order shipped yesterday.",
        "tool_calls": [],
        "latency_ms": 1100,
        "tokens_used": 860
      }
    ],
    "assertions": [
      {
        "name": "must_contain",
        "status": "pass",
        "reason": "shipped"
      }
    ]
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "RATE_LIMITED",
    "message": "<string>",
    "status": 429,
    "retry_after": 2
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

Agents

Run a multi-turn dry-run scenario

Replay a scripted scenario against the agent in sandbox mode: the LLM call is real, but mutative tool side effects (calendar bookings, handoffs, memory writes) are short-circuited and billing is skipped, ending with a pass/fail verdict across the turn-by-turn trace and assertion results. The scenario.turns[] array supplies scripted user messages; optional assertions expected_tool_calls (every named tool must appear), must_contain / must_not_contain (transcript substring checks), plus LangSmith-style trajectory grading. Max 20 turns per scenario, 30s per turn. Matches the dashboard Practice Studio contract.

POST

api

agents

{id}

dry-run

Run a multi-turn dry-run scenario

curl --request POST \
  --url https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "scenario": {
    "turns": [
      {
        "user_msg": "<string>"
      }
    ],
    "expected_tool_calls": [
      "<string>"
    ],
    "must_contain": [
      "<string>"
    ],
    "must_not_contain": [
      "<string>"
    ]
  },
  "mock_tools": true
}
'

import requests

url = "https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run"

payload = {
    "scenario": {
        "turns": [{ "user_msg": "<string>" }],
        "expected_tool_calls": ["<string>"],
        "must_contain": ["<string>"],
        "must_not_contain": ["<string>"]
    },
    "mock_tools": True
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    scenario: {
      turns: [{user_msg: '<string>'}],
      expected_tool_calls: ['<string>'],
      must_contain: ['<string>'],
      must_not_contain: ['<string>']
    },
    mock_tools: true
  })
};

fetch('https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'scenario' => [
        'turns' => [
                [
                                'user_msg' => '<string>'
                ]
        ],
        'expected_tool_calls' => [
                '<string>'
        ],
        'must_contain' => [
                '<string>'
        ],
        'must_not_contain' => [
                '<string>'
        ]
    ],
    'mock_tools' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run"

	payload := strings.NewReader("{\n  \"scenario\": {\n    \"turns\": [\n      {\n        \"user_msg\": \"<string>\"\n      }\n    ],\n    \"expected_tool_calls\": [\n      \"<string>\"\n    ],\n    \"must_contain\": [\n      \"<string>\"\n    ],\n    \"must_not_contain\": [\n      \"<string>\"\n    ]\n  },\n  \"mock_tools\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"scenario\": {\n    \"turns\": [\n      {\n        \"user_msg\": \"<string>\"\n      }\n    ],\n    \"expected_tool_calls\": [\n      \"<string>\"\n    ],\n    \"must_contain\": [\n      \"<string>\"\n    ],\n    \"must_not_contain\": [\n      \"<string>\"\n    ]\n  },\n  \"mock_tools\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.orbit.devotel.io/api/v1/agents/{id}/dry-run")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"scenario\": {\n    \"turns\": [\n      {\n        \"user_msg\": \"<string>\"\n      }\n    ],\n    \"expected_tool_calls\": [\n      \"<string>\"\n    ],\n    \"must_contain\": [\n      \"<string>\"\n    ],\n    \"must_not_contain\": [\n      \"<string>\"\n    ]\n  },\n  \"mock_tools\": true\n}"

response = http.request(request)
puts response.read_body

{
  "data": {
    "agent_id": "agt_01J9Z8ABCDEF",
    "mock_tools": true,
    "status": "pass",
    "duration_ms": 4210,
    "trace": [
      {
        "index": 0,
        "user_msg": "Where is my order?",
        "assistant_response": "Your order shipped yesterday.",
        "tool_calls": [],
        "latency_ms": 1100,
        "tokens_used": 860
      }
    ],
    "assertions": [
      {
        "name": "must_contain",
        "status": "pass",
        "reason": "shipped"
      }
    ]
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "RATE_LIMITED",
    "message": "<string>",
    "status": 429,
    "retry_after": 2
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "status": 123,
    "details": {}
  },
  "meta": {
    "request_id": "<string>",
    "timestamp": "2023-11-07T05:31:56Z",
    "docs_url": "<string>"
  }
}

Authorizations

Authorization

string

header

required

Dashboard JWT token from Clerk

Headers

Idempotency-Key

string

Stripe-style idempotency token. Pass a stable, client-generated value (1-255 chars) to dedupe retries on transient timeouts. The same key+credential+path replays the original response for 24h on 2xx (5min on 4xx, 30s on 5xx). Returns 409 if a concurrent request with the same key is already in flight; replayed responses include the Idempotency-Replay: true response header.

Required string length: 1 - 255

X-Test-Mode

enum<string>

Sandbox opt-in for Clerk-session-authenticated requests. Set to true to route the call through the test-mode pipeline: no real provider delivery, no credits deducted, response meta.test_mode: true. Ignored for live API keys (dv_live_sk_*) — server-to-server clients must use a test-prefixed key (dv_test_sk_*) to exercise sandbox. Test-prefixed keys unconditionally enable sandbox regardless of this header.

Available options:

true,

false

Path Parameters

string

required

The agent id.

Minimum string length: 1

Body

application/json

scenario

object

required

Show child attributes

mock_tools

boolean

Sandbox mode already short-circuits tools; preserved so callers can opt out once live-tools sandbox exists. Default true.

Response

Pass/fail verdict with the turn-by-turn trace and assertion results in the standard { data, meta } envelope.

data

object

Show child attributes

API Reference

All Endpoints

Run a multi-turn dry-run scenario

Authorizations

Headers

Path Parameters

Body

Response