Create decision model rule

Error

A valid request URL is required to generate request examples

{
  "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "tenant_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "display_name": "<string>",
  "sampling_rate": 123,
  "webhooks": [
    {
      "url": "<string>",
      "headers": {}
    }
  ],
  "created_at": "2023-11-07T05:31:56Z",
  "updated_at": "2023-11-07T05:31:56Z",
  "evaluator_version": 123,
  "is_enabled": true,
  "session_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "session_name": "<string>",
  "dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "dataset_name": "<string>",
  "filter": "<string>",
  "trace_filter": "<string>",
  "tree_filter": "<string>",
  "add_to_annotation_queue_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "add_to_annotation_queue_name": "<string>",
  "add_to_dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "add_to_dataset_name": "<string>",
  "add_to_dataset_prefer_correction": false,
  "corrections_dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "use_corrections_dataset": false,
  "num_few_shot_examples": 123,
  "evaluators": [
    {
      "structured": {
        "hub_ref": "<string>",
        "prompt": [
          [
            "<string>",
            "<string>"
          ]
        ],
        "template_format": "<string>",
        "schema": {},
        "variable_mapping": {},
        "model": {},
        "playground_settings_id": "<string>"
      }
    }
  ],
  "code_evaluators": [
    {
      "code": "<string>",
      "language": "python",
      "require_attachments": false,
      "code_evaluator_input": "thread",
      "advanced_features_enabled": true
    }
  ],
  "alerts": [
    {
      "routing_key": "<string>",
      "type": "pagerduty",
      "summary": "<string>",
      "severity": "warning"
    }
  ],
  "extend_only": false,
  "is_managed_evaluator": false,
  "is_tracing_disabled": false,
  "extend_evaluator_trace_retention": true,
  "extend_dataset_trace_retention": true,
  "extend_annotation_queue_trace_retention": true,
  "extend_webhook_trace_retention": true,
  "include_extended_stats": false,
  "backfill_from": "2023-11-07T05:31:56Z",
  "backfill_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "backfill_status": "<string>",
  "backfill_progress": 123,
  "backfill_error": "<string>",
  "backfill_completed_at": "2023-11-07T05:31:56Z",
  "transient": false,
  "evaluator_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "evaluator_name": "<string>",
  "evaluator_type": "<string>",
  "alignment_annotation_queue_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "group_by": "thread_id",
  "spend_limit": {
    "limit_usd": "<string>",
    "window": "weekly"
  },
  "trace_count": 123,
  "spend_usd": 123
}

{
  "detail": [
    "<string>"
  ]
}

POST

api

runs

rules

decision-models

Error

A valid request URL is required to generate request examples

{
  "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "tenant_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "display_name": "<string>",
  "sampling_rate": 123,
  "webhooks": [
    {
      "url": "<string>",
      "headers": {}
    }
  ],
  "created_at": "2023-11-07T05:31:56Z",
  "updated_at": "2023-11-07T05:31:56Z",
  "evaluator_version": 123,
  "is_enabled": true,
  "session_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "session_name": "<string>",
  "dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "dataset_name": "<string>",
  "filter": "<string>",
  "trace_filter": "<string>",
  "tree_filter": "<string>",
  "add_to_annotation_queue_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "add_to_annotation_queue_name": "<string>",
  "add_to_dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "add_to_dataset_name": "<string>",
  "add_to_dataset_prefer_correction": false,
  "corrections_dataset_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "use_corrections_dataset": false,
  "num_few_shot_examples": 123,
  "evaluators": [
    {
      "structured": {
        "hub_ref": "<string>",
        "prompt": [
          [
            "<string>",
            "<string>"
          ]
        ],
        "template_format": "<string>",
        "schema": {},
        "variable_mapping": {},
        "model": {},
        "playground_settings_id": "<string>"
      }
    }
  ],
  "code_evaluators": [
    {
      "code": "<string>",
      "language": "python",
      "require_attachments": false,
      "code_evaluator_input": "thread",
      "advanced_features_enabled": true
    }
  ],
  "alerts": [
    {
      "routing_key": "<string>",
      "type": "pagerduty",
      "summary": "<string>",
      "severity": "warning"
    }
  ],
  "extend_only": false,
  "is_managed_evaluator": false,
  "is_tracing_disabled": false,
  "extend_evaluator_trace_retention": true,
  "extend_dataset_trace_retention": true,
  "extend_annotation_queue_trace_retention": true,
  "extend_webhook_trace_retention": true,
  "include_extended_stats": false,
  "backfill_from": "2023-11-07T05:31:56Z",
  "backfill_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "backfill_status": "<string>",
  "backfill_progress": 123,
  "backfill_error": "<string>",
  "backfill_completed_at": "2023-11-07T05:31:56Z",
  "transient": false,
  "evaluator_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "evaluator_name": "<string>",
  "evaluator_type": "<string>",
  "alignment_annotation_queue_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "group_by": "thread_id",
  "spend_limit": {
    "limit_usd": "<string>",
    "window": "weekly"
  },
  "trace_count": 123,
  "spend_usd": 123
}

{
  "detail": [
    "<string>"
  ]
}

Authorizations

X-API-Key

string

header

required

Body

application/json

A decision-model evaluator to create and attach to a project or dataset.

display_name

string

required

prompt

tuple[]

required

The prompt the model reads, as (role, template) pairs.

Show child attributes

questions

Questions · object

required

The questions, keyed by feedback key, in the order to ask them.

Show child attributes

session_id

string<uuid> | null

The tracing project to evaluate.

dataset_id

string<uuid> | null

The dataset whose experiments to evaluate.

template_format

enum<string>

default:mustache

Available options:

mustache,

f-string

variable_mapping

Variable Mapping · object | null

Maps the prompt's variables to run fields.

Show child attributes

provider

enum<string>

default:typesafe

typesafe for Jev and SemIf, typesafe_compatible for a TypeSafe-compatible endpoint, or openai_decisions.

Available options:

typesafe,

typesafe_compatible,

openai_decisions

model

string | null

Defaults to jev-latest for typesafe and gpt-6-luna for openai_decisions. Use semif-qwen3.5-4b for SemIf.

base_url

string | null

Call this endpoint directly instead of through the LangSmith gateway, with the workspace's TYPESAFE_API_KEY or OPENAI_API_KEY secret. Required for typesafe_compatible.

model_headers

Model Headers · object | null

Headers sent with each model call. They are saved in the evaluator's prompt as given, so do not put API keys here.

Show child attributes

timeout_seconds

integer | null

Seconds to wait for each model call. openai_decisions only.

max_retries

integer | null

How many times to retry a failed model call. openai_decisions only.

playground_settings_id

string<uuid> | null

A saved model configuration to start from: its model, endpoint, key and settings apply unless set here.

sampling_rate

number

default:1

The fraction of runs to evaluate.

filter

string | null

Only evaluate runs that match this filter.

Response

Successful Response

Run rules schema.

string<uuid>

required

tenant_id

string<uuid>

required

display_name

string

required

sampling_rate

number

required

webhooks

RunRulesWebhookSchema · object[] | null

required

Show child attributes

created_at

string<date-time>

required

updated_at

string<date-time>

required

evaluator_version

integer

required

is_enabled

boolean

default:true

session_id

string<uuid> | null

session_name

string | null

dataset_id

string<uuid> | null

dataset_name

string | null

filter

string | null

trace_filter

string | null

tree_filter

string | null

add_to_annotation_queue_id

string<uuid> | null

add_to_annotation_queue_name

string | null

add_to_dataset_id

string<uuid> | null

add_to_dataset_name

string | null

add_to_dataset_prefer_correction

boolean

default:false

corrections_dataset_id

string<uuid> | null

use_corrections_dataset

boolean

default:false

num_few_shot_examples

integer | null

evaluators

EvaluatorTopLevel · object[] | null

Show child attributes

code_evaluators

CodeEvaluatorTopLevel · object[] | null

Show child attributes

alerts

RunRulesPagerdutyAlertSchema · object[] | null

Show child attributes

extend_only

boolean

default:false

is_managed_evaluator

boolean

default:false

is_tracing_disabled

boolean

default:false

extend_evaluator_trace_retention

boolean | null

extend_dataset_trace_retention

boolean | null

extend_annotation_queue_trace_retention

boolean | null

extend_webhook_trace_retention

boolean | null

include_extended_stats

boolean

default:false

backfill_from

string<date-time> | null

backfill_id

string<uuid> | null

backfill_status

string | null

backfill_progress

number | null

backfill_error

string | null

backfill_completed_at

string<date-time> | null

transient

boolean

default:false

evaluator_id

string<uuid> | null

evaluator_name

string | null

evaluator_type

string | null

alignment_annotation_queue_id

string<uuid> | null

group_by

enum<string> | null

Available options:

thread_id,

trajectory_id

spend_limit

RunRuleSpendLimitSchema · object | null

Show child attributes

trace_count

integer | null

spend_usd

number | null

SmithDB SDK migration

LangSmith REST API

Authorizations

Body

Response