Sign in

v1

latestOpenAPI 3.0.02026-07-141,4077,1665.2 MB
LLM Observability

Run an LLM inference

Run an LLM inference request through the specified integration and account, returning the model response and token usage.

post/api/v2/llm-obs/v1/integrations/{integration}/{account_id}/inference

Path parameters

integration'openai' | 'amazon_bedrock' | 'anthropic' | 'azure_openai' | 'vertex_ai' | 'llmproxy' required

The name of a supported LLM provider integration.

Example:openai

The name of the LLM integration.

account_idstring required

The ID of the integration account.

Request body

frequency_penaltynumber double nullable

Penalty for token frequency to reduce repetition.

json_schemastring nullable

JSON schema for structured output, if supported by the model.

max_completion_tokensinteger nullable

Maximum number of completion tokens to generate (alternative to max_tokens for some providers).

max_tokensinteger nullable

Maximum number of tokens to generate.

model_idstring required

The model identifier to use for inference.

presence_penaltynumber double nullable

Penalty for token presence to encourage topic diversity.

temperaturenumber double nullable

Sampling temperature between 0 and 2. Higher values produce more random output.

top_kinteger nullable

Top-K sampling parameter.

top_pnumber double nullable

Nucleus sampling probability mass.

Example request

{
  "anthropic_metadata": {
    "effort": "medium",
    "thinking": {
      "budget_tokens": 1024,
      "type": "enabled"
    }
  },
  "azure_openai_metadata": {
    "deployment_id": "my-gpt4-deployment",
    "model_version": "0613",
    "resource_name": "my-azure-resource"
  },
  "bedrock_metadata": {
    "region": "us-east-1"
  },
  "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}",
  "max_completion_tokens": 1024,
  "max_tokens": 1024,
  "messages": [
    {
      "content": "What is the capital of France?",
      "contents": [
        {
          "type": "text",
          "value": {
            "text": "Hello, how can I help you?",
            "tool_call": {
              "arguments": {
                "location": "San Francisco"
              },
              "name": "get_weather",
              "tool_id": "call_abc123",
              "type": "function"
            },
            "tool_call_result": {
              "name": "get_weather",
              "result": "The weather in San Francisco is 68°F and sunny.",
              "tool_id": "call_abc123",
              "type": "function"
            }
          }
        }
      ],
      "id": "msg_001",
      "role": "user",
      "tool_calls": [
        {
          "arguments": {
            "location": "San Francisco"
          },
          "name": "get_weather",
          "tool_id": "call_abc123",
          "type": "function"
        }
      ],
      "tool_results": [
        {
          "name": "get_weather",
          "result": "The weather in San Francisco is 68°F and sunny.",
          "tool_id": "call_abc123",
          "type": "function"
        }
      ]
    }
  ],
  "model_id": "gpt-4o",
  "openai_metadata": {
    "reasoning_effort": "medium",
    "reasoning_summary": "auto"
  },
  "temperature": 0.7,
  "tools": [
    {
      "function": {
        "description": "Get the current weather for a location.",
        "name": "get_weather",
        "parameters": {
          "properties": {
            "location": {
              "type": "string"
            }
          },
          "type": "object"
        }
      },
      "type": "function"
    }
  ],
  "top_k": 50,
  "top_p": 1,
  "vertex_ai_metadata": {
    "location": "us-central1",
    "project": "my-gcp-project",
    "project_ids": [
      "my-gcp-project"
    ]
  }
}

Response

OK

frequency_penaltynumber double nullable

Frequency penalty that was applied.

json_schemastring nullable

JSON schema that was applied for structured output.

max_completion_tokensinteger nullable

Maximum number of completion tokens that were configured.

max_tokensinteger nullable

Maximum number of tokens that were configured.

model_idstring required

The model identifier used for inference.

presence_penaltynumber double nullable

Presence penalty that was applied.

temperaturenumber double nullable

Sampling temperature that was used.

top_kinteger nullable

Top-K sampling parameter that was used.

top_pnumber double nullable

Nucleus sampling parameter that was used.

Example response

{
  "anthropic_metadata": {
    "effort": "medium",
    "thinking": {
      "budget_tokens": 1024,
      "type": "enabled"
    }
  },
  "azure_openai_metadata": {
    "deployment_id": "my-gpt4-deployment",
    "model_version": "0613",
    "resource_name": "my-azure-resource"
  },
  "bedrock_metadata": {
    "region": "us-east-1"
  },
  "error_response": {
    "message": "The model does not exist.",
    "type": "invalid_request_error"
  },
  "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}",
  "max_completion_tokens": 1024,
  "max_tokens": 1024,
  "messages": [
    {
      "content": "What is the capital of France?",
      "contents": [
        {
          "type": "text",
          "value": {
            "text": "Hello, how can I help you?",
            "tool_call": {
              "arguments": {
                "location": "San Francisco"
              },
              "name": "get_weather",
              "tool_id": "call_abc123",
              "type": "function"
            },
            "tool_call_result": {
              "name": "get_weather",
              "result": "The weather in San Francisco is 68°F and sunny.",
              "tool_id": "call_abc123",
              "type": "function"
            }
          }
        }
      ],
      "id": "msg_001",
      "role": "user",
      "tool_calls": [
        {
          "arguments": {
            "location": "San Francisco"
          },
          "name": "get_weather",
          "tool_id": "call_abc123",
          "type": "function"
        }
      ],
      "tool_results": [
        {
          "name": "get_weather",
          "result": "The weather in San Francisco is 68°F and sunny.",
          "tool_id": "call_abc123",
          "type": "function"
        }
      ]
    }
  ],
  "model_id": "gpt-4o",
  "openai_metadata": {
    "reasoning_effort": "medium",
    "reasoning_summary": "auto"
  },
  "response": {
    "assessment": "pass",
    "content": "The capital of France is Paris.",
    "finish_reason": "stop",
    "inference_codes": [
      {
        "code": "import openai\nclient = openai.OpenAI()\n...",
        "id": "code-python-001",
        "type": "python"
      }
    ],
    "input_tokens": 15,
    "internal_reasoning": {
      "reasoning_tokens": 256,
      "text": "Let me think about this step by step..."
    },
    "latency": 843,
    "output_tokens": 10,
    "tools": [
      {
        "function": {
          "description": "Get the current weather for a location.",
          "name": "get_weather",
          "parameters": {
            "properties": {
              "location": {
                "type": "string"
              }
            },
            "type": "object"
          }
        },
        "type": "function"
      }
    ],
    "total_tokens": 25
  },
  "temperature": 0.7,
  "tools": [
    {
      "function": {
        "description": "Get the current weather for a location.",
        "name": "get_weather",
        "parameters": {
          "properties": {
            "location": {
              "type": "string"
            }
          },
          "type": "object"
        }
      },
      "type": "function"
    }
  ],
  "top_k": 50,
  "top_p": 1,
  "vertex_ai_metadata": {
    "location": "us-central1",
    "project": "my-gcp-project",
    "project_ids": [
      "my-gcp-project"
    ]
  }
}