> ## Documentation Index
> Fetch the complete documentation index at: https://docs.cekura.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Evaluators

> Scenarios, test profiles, personalities, and dynamic variables

export const CopyPageButton = () => {
  if (typeof window !== 'undefined') {
    setTimeout(function () {
      if (document.getElementById('ck-tools')) return;
      var anchor = document.getElementById('content-area') || document.querySelector('.mdx-content');
      if (!anchor) return;
      if (!document.getElementById('ck-style')) {
        var s = document.createElement('style');
        s.id = 'ck-style';
        s.textContent = '#ck-tools{position:absolute;top:6px;right:0;z-index:100;font-family:inherit;}' + '.ck-row{display:inline-flex;align-items:stretch;border:1px solid rgba(0,0,0,0.15);border-radius:8px;overflow:hidden;background:#fff;}' + ':root.dark .ck-row{background:rgba(255,255,255,0.06);border-color:rgba(255,255,255,0.12);}' + '.ck-btn{padding:5px 12px;border:none;background:none;cursor:pointer;font-size:13px;font-weight:500;font-family:inherit;color:#374151;}' + ':root.dark .ck-btn{color:#d1d5db;}' + '.ck-btn:hover{background:rgba(0,0,0,0.04);}' + ':root.dark .ck-btn:hover{background:rgba(255,255,255,0.06);}' + '.ck-chevron{padding:5px 8px;border:none;background:none;cursor:pointer;font-size:14px;font-family:inherit;color:#374151;}' + ':root.dark .ck-chevron{color:#d1d5db;}' + '.ck-chevron:hover{background:rgba(0,0,0,0.04);}' + ':root.dark .ck-chevron:hover{background:rgba(255,255,255,0.06);}' + '.ck-divider{width:1px;background:rgba(0,0,0,0.12);flex-shrink:0;}' + ':root.dark .ck-divider{background:rgba(255,255,255,0.12);}' + '.ck-dd{position:absolute;top:calc(100% + 4px);right:0;min-width:180px;background:#fff;border:1px solid rgba(0,0,0,0.12);border-radius:8px;box-shadow:0 4px 12px rgba(0,0,0,0.1);padding:4px;display:none;z-index:200;}' + ':root.dark .ck-dd{background:#1f2937;border-color:rgba(255,255,255,0.1);box-shadow:0 4px 16px rgba(0,0,0,0.35);}' + '.ck-item{display:block;width:100%;padding:7px 12px;border:none;background:none;border-radius:6px;cursor:pointer;font-size:13px;font-family:inherit;text-align:left;color:#374151;}' + ':root.dark .ck-item{color:#d1d5db;}' + '.ck-item:hover{background:rgba(0,0,0,0.05);}' + ':root.dark .ck-item:hover{background:rgba(255,255,255,0.07);}';
        document.head.appendChild(s);
      }
      var wrap = document.createElement('div');
      wrap.id = 'ck-tools';
      var row = document.createElement('div');
      row.className = 'ck-row';
      var mainBtn = document.createElement('button');
      mainBtn.className = 'ck-btn';
      mainBtn.textContent = 'Copy page';
      var divider = document.createElement('span');
      divider.className = 'ck-divider';
      var chevron = document.createElement('button');
      chevron.className = 'ck-chevron';
      chevron.textContent = '▾';
      var dd = document.createElement('div');
      dd.className = 'ck-dd';
      function closeDD() {
        dd.style.display = 'none';
      }
      function openDD() {
        dd.style.display = 'block';
      }
      chevron.onclick = function (e) {
        e.stopPropagation();
        if (dd.style.display === 'block') {
          closeDD();
        } else {
          openDD();
        }
      };
      document.addEventListener('click', function (e) {
        if (!e.target.closest('#ck-tools')) {
          closeDD();
        }
      });
      document.addEventListener('keydown', function (e) {
        if (e.key === 'Escape') {
          closeDD();
        }
      });
      function makeItem(label, fn) {
        var b = document.createElement('button');
        b.className = 'ck-item';
        b.textContent = label;
        b.onclick = function () {
          fn();
          closeDD();
        };
        return b;
      }
      function getMarkdown() {
        var walk = function (node) {
          if (!node) return '';
          if (node.nodeType === 3) return node.textContent || '';
          if (node.nodeType !== 1) return '';
          var tag = node.tagName.toLowerCase();
          var skip = ['script', 'style', 'svg', 'noscript', 'button', 'iframe'];
          if (skip.indexOf(tag) !== -1) return '';
          if (node.id === 'ck-tools') return '';
          var ch = Array.from(node.childNodes).map(walk).join('');
          if (tag === 'h1') return '\n# ' + ch.trim() + '\n\n';
          if (tag === 'h2') return '\n## ' + ch.trim() + '\n\n';
          if (tag === 'h3') return '\n### ' + ch.trim() + '\n\n';
          if (tag === 'p') return '\n' + ch.trim() + '\n\n';
          if (tag === 'pre') return '\n```\n' + node.textContent.trim() + '\n```\n\n';
          if (tag === 'li') return '- ' + ch.trim() + '\n';
          if (tag === 'code') return '`' + ch.trim() + '`';
          return ch;
        };
        var content = document.querySelector('.mdx-content') || document.getElementById('content-area') || document.body;
        return walk(content).replace(/\n\n\n+/g, '\n\n').trim();
      }
      function copyMd() {
        var md = getMarkdown();
        navigator.clipboard.writeText(md).then(function () {
          mainBtn.textContent = 'Copied!';
          setTimeout(function () {
            mainBtn.textContent = 'Copy page';
          }, 2000);
        });
      }
      function viewMd() {
        var md = getMarkdown();
        var safe = md.split('&').join('&amp;').split('<').join('&lt;').split('>').join('&gt;');
        var html = '<!DOCTYPE html><html><head><meta charset="utf-8"><style>body{font-family:monospace;max-width:860px;margin:40px auto;padding:0 24px;line-height:1.7;white-space:pre-wrap;word-wrap:break-word}</style></head><body>' + safe + '</body></html>';
        window.open(URL.createObjectURL(new Blob([html], {
          type: 'text/html'
        })), '_blank');
      }
      function openClaude() {
        var prompt = 'Can you read this Cekura docs page ' + window.location.href + ' so I can ask you questions?';
        window.open('https://claude.ai/new?q=' + encodeURIComponent(prompt), '_blank');
      }
      mainBtn.onclick = copyMd;
      dd.appendChild(makeItem('Copy page', copyMd));
      dd.appendChild(makeItem('View as Markdown', viewMd));
      dd.appendChild(makeItem('Open in Claude', openClaude));
      row.appendChild(mainBtn);
      row.appendChild(divider);
      row.appendChild(chevron);
      wrap.appendChild(row);
      wrap.appendChild(dd);
      anchor.style.position = 'relative';
      anchor.insertBefore(wrap, anchor.firstChild);
    }, 50);
  }
  return null;
};

<CopyPageButton />

<Note>
  Setup steps and authentication are in the [Overview](/cli-sdk/overview). This page covers the inputs your evaluations consume.
</Note>

An **evaluator** is the input to a run. The most important is a **scenario** — a structured test case describing what the simulated caller does and what the agent should achieve. Scenarios are paired with optional **test profiles** (transport / mode config), **personalities** (caller behavior), and **dynamic variables** (per-run overrides).

## Scenarios

### List, get, create, update, delete

<Tabs>
  <Tab title="CLI">
    ```bash theme={null}
    # List scenarios for an agent
    cekura scenarios list --agent-id 123

    # Get one
    cekura scenarios get 9876

    # Create from a JSON file
    cekura scenarios create --from-file my_scenario.json

    # Update one
    cekura scenarios update 9876 --from-file patch.json

    # Delete
    cekura scenarios delete 9876
    ```
  </Tab>

  <Tab title="SDK">
    ```python theme={null}
    from cekura import Cekura

    client = Cekura()

    # List
    scenarios = client.scenarios.list(agent_id=123)

    # Get one
    scenario = client.scenarios.get(scenario_id=9876)

    # Create
    sc = client.scenarios.create(
        agent=123,
        name="Reschedule appointment",
        description="Caller wants to move their booking to next week",
        scenario_type="conversation",
        tags=["smoke"],
    )

    # Update
    client.scenarios.update(scenario_id=sc["id"], description="Updated wording")

    # Delete
    client.scenarios.delete(scenario_id=9876)
    ```
  </Tab>
</Tabs>

### Bulk update

The bulk-update payload is a single JSON object with a top-level `scenarios` list, plus any of `metric_ids_to_add`, `metric_ids_to_remove`, `tool_ids_to_add`, etc.

<Tabs>
  <Tab title="CLI">
    Prepare `bulk.json`:

    ```json theme={null}
    {
      "scenarios": [
        { "id": 101, "tags": ["regression"] },
        { "id": 102, "tags": ["smoke"] }
      ],
      "metric_ids_to_add": [55],
      "tool_ids_to_remove": [12]
    }
    ```

    Apply:

    ```bash theme={null}
    cekura scenarios bulk-update --from-file bulk.json
    ```
  </Tab>

  <Tab title="SDK">
    ```python theme={null}
    client.scenarios.bulk_update(
        scenarios=[
            {"id": 101, "tags": ["regression"]},
            {"id": 102, "tags": ["smoke"]},
        ],
        metric_ids_to_add=[55],
        tool_ids_to_remove=[12],
    )
    ```
  </Tab>
</Tabs>

### Generate scenarios

Cekura can generate scenario coverage from a description or an existing call.

<Tabs>
  <Tab title="CLI">
    ```bash theme={null}
    cekura scenarios generate --agent-id 123 --description "Bookings flow — happy path and reschedules" --count 10
    cekura scenarios generate-progress --progress-id <id>
    ```
  </Tab>

  <Tab title="SDK">
    ```python theme={null}
    job = client.scenarios.generate_bg(
        agent_id=123,
        description="Bookings flow — happy path and reschedules",
        count=10,
    )
    client.scenarios.generate_progress(progress_id=job["progress_id"])
    ```
  </Tab>
</Tabs>

### Generate or improve scenario instructions

When you have a scenario shell and need Cekura to author (or rewrite) its instructions — for example, after you've added an `expected_outcome` and want the simulated caller's script regenerated to match — use the instructions pipeline. It runs in the background and exposes a single progress endpoint.

<Tabs>
  <Tab title="SDK">
    ```python theme={null}
    # AI-author instructions from an outline
    job = client.scenarios.generate_instructions(
        agent_id=123,
        extra_instructions="Caller wants to escalate after two failed retries",
    )

    # Improve the instructions on an existing scenario
    client.scenarios.improve_instructions(
        scenario_id=9876,
        feedback="Make the caller more impatient",
    )

    # Poll either job
    client.scenarios.instructions_progress(progress_id=job["progress_id"])
    ```
  </Tab>
</Tabs>

### Build a scenario from a transcript

Two flavors. `create_from_transcript` makes a brand-new scenario; `update_with_transcript` revises an existing one (useful when a production call exposes a scenario's gap).

<Tabs>
  <Tab title="SDK">
    ```python theme={null}
    # New scenario from a transcript
    client.scenarios.create_from_transcript(
        agent=123,
        transcript=[
            {"role": "agent", "text": "How can I help?"},
            {"role": "user",  "text": "I'd like to reschedule."},
        ],
        name="Reschedule — from call 7788",
    )

    # Refresh an existing scenario from a newer transcript
    client.scenarios.update_with_transcript(
        scenario_id=9876,
        transcript=[...],
    )
    ```
  </Tab>
</Tabs>

### Scenario agent (interactive AI builder)

The scenario agent is an LLM-driven authoring loop — you describe an intent and it iterates on a scenario for you. Kicks off in the background.

<Tabs>
  <Tab title="SDK">
    ```python theme={null}
    job = client.scenarios.scenario_agent(
        agent_id=123,
        prompt="Build a scenario where the caller cancels then immediately re-books",
    )
    client.scenarios.scenario_agent_progress(progress_id=job["progress_id"])
    ```
  </Tab>
</Tabs>

## Test profiles

A **test profile** bundles transport + mode settings (voice, text, SIP, livekit\_v2, pipecat\_v2, etc.) so you can reuse them across runs.

<Tabs>
  <Tab title="CLI">
    ```bash theme={null}
    cekura test-profiles list --agent-id 123
    cekura test-profiles create --from-file profile.json
    cekura test-profiles update 7 --from-file patch.json
    ```
  </Tab>

  <Tab title="SDK">
    ```python theme={null}
    client.test_profiles.list(agent_id=123)
    client.test_profiles.create(
        agent=123,
        name="Voice + Twilio",
        mode="voice",
        config={"transport": "twilio"},
    )
    client.test_profiles.update(profile_id=7, name="Voice + Plivo")
    ```
  </Tab>
</Tabs>

## Personalities

Personalities define how the simulated caller behaves (interruptive, accented, polite, etc.). The catalog is platform-managed — you enable or disable items per project.

<Tabs>
  <Tab title="CLI">
    ```bash theme={null}
    # Browse the catalog
    cekura personalities list

    # Enable / disable on a project
    cekura personalities enable-for-project 742 --personalities 3,29
    cekura personalities disable-for-project 742 --personalities 3
    ```
  </Tab>

  <Tab title="SDK">
    ```python theme={null}
    client.personalities.list()
    client.personalities.enable_for_project(project_id=742, personalities=[3, 29])
    client.personalities.disable_for_project(project_id=742, personalities=[3])
    ```
  </Tab>
</Tabs>

## Dynamic variables

Dynamic variables let you parameterize a scenario at run time (e.g. customer name, account ID).

<Tabs>
  <Tab title="CLI">
    Manage variables on an agent:

    ```bash theme={null}
    cekura agents dynamic-variables list 123
    cekura agents dynamic-variable-create 123 --from-file vars.json
    ```

    Pass values inline when triggering a run:

    ```bash theme={null}
    cekura scenarios run-text \
      --agent-id 123 \
      --scenario-ids 101 \
      --dynamic-variables '{"customer_name":"Alex","account_id":"A-552"}'
    ```
  </Tab>

  <Tab title="SDK">
    ```python theme={null}
    client.scenarios.run_text(
        agent_id=123,
        scenario_ids=[101],
        dynamic_variables={"customer_name": "Alex", "account_id": "A-552"},
    )
    ```
  </Tab>
</Tabs>

## See also

<CardGroup cols={2}>
  <Card title="Agents" icon="robot" href="/cli-sdk/agents">
    The agent under test that scenarios run against.
  </Card>

  <Card title="Runs & Results" icon="play" href="/cli-sdk/runs">
    Trigger evaluations using these scenarios.
  </Card>

  <Card title="Evaluator concepts" icon="vial" href="/documentation/key-concepts/evaluators/overview">
    Conditional actions, mock tools, and design best practices.
  </Card>

  <Card title="API Reference" icon="book" href="/api-reference">
    Full field reference for scenarios, test profiles, and personalities.
  </Card>
</CardGroup>