Evaluations

For instructions on how to authenticate to use this endpoint, see API overview.

Endpoints

`GET`
`POST`
`GET`
`POST`
`GET`
`POST`
`POST`
`GET`
`PATCH`
`DELETE`
`POST`

List all evaluations

Required API key scopes

evaluation:read

Query parameters

directory_id
string
directory_id__isnull
boolean
enabled
boolean
evaluation_type
string
One of: "hog""llm_judge""sentiment"
id__in
array
limit
integer
offset
integer
order_by
array
search
string

Response

Example request

GET /api/projects/:project_id/evaluations
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/evaluations/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "count": 123,
  "next": "http://api.example.org/accounts/?offset=400&limit=100",
  "previous": "http://api.example.org/accounts/?offset=200&limit=100",
  "results": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "name": "string",
      "description": "string",
      "directory_id": "dca58562-24cf-4cbc-aa50-5a4039358aa5",
      "enabled": true,
      "status": "active",
      "status_reason": "provider_key_required",
      "status_reason_detail": "string",
      "evaluation_type": "llm_judge",
      "evaluation_config": {
        "prompt": "string"
      },
      "output_type": "boolean",
      "output_config": {
        "allows_na": true,
        "true_is_failure": true
      },
      "conditions": [
        {
          "id": "string",
          "rollout_percentage": 100,
          "properties": [
            {}
          ]
        }
      ],
      "target": "generation",
      "target_config": {
        "strategy": "fixed_window",
        "window_seconds": 10
      },
      "model_configuration": {
        "provider": "openai",
        "model": "string",
        "provider_key_id": "f265db88-9bcc-4e5b-add5-bfd9a815465c",
        "provider_key_name": "string"
      },
      "created_at": "2019-08-24T14:15:22Z",
      "updated_at": "2019-08-24T14:15:22Z",
      "created_by": {
        "id": 0,
        "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
        "distinct_id": "string",
        "first_name": "string",
        "last_name": "string",
        "email": "user@example.com",
        "is_email_verified": true,
        "hedgehog_config": {},
        "role_at_organization": "engineering"
      },
      "deleted": true,
      "user_access_level": "string"
    }
  ]
}

Create evaluations

Required API key scopes

evaluation:write

Request parameters

name
string
description
string
directory_id
stringnull
enabled
boolean
evaluation_type
evaluation_config
output_type
output_config
object
conditions
array
target
target_config
model_configuration
deleted
boolean

Response

Example request

POST /api/projects/:project_id/evaluations
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/\
	-d name="string",\
	-d evaluation_type=undefined,\
	-d output_type=undefined
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/evaluations/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"name": "string"
	}
).json()

Example response

Status 201

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "name": "string",
  "description": "string",
  "directory_id": "dca58562-24cf-4cbc-aa50-5a4039358aa5",
  "enabled": true,
  "status": "active",
  "status_reason": "provider_key_required",
  "status_reason_detail": "string",
  "evaluation_type": "llm_judge",
  "evaluation_config": {
    "prompt": "string"
  },
  "output_type": "boolean",
  "output_config": {
    "allows_na": true,
    "true_is_failure": true
  },
  "conditions": [
    {
      "id": "string",
      "rollout_percentage": 100,
      "properties": [
        {}
      ]
    }
  ],
  "target": "generation",
  "target_config": {
    "strategy": "fixed_window",
    "window_seconds": 10
  },
  "model_configuration": {
    "provider": "openai",
    "model": "string",
    "provider_key_id": "f265db88-9bcc-4e5b-add5-bfd9a815465c",
    "provider_key_name": "string"
  },
  "created_at": "2019-08-24T14:15:22Z",
  "updated_at": "2019-08-24T14:15:22Z",
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "deleted": true,
  "user_access_level": "string"
}

List all evaluations backfills

Historical runs of one evaluation over a closed time window (nested under an evaluation).

Required API key scopes

evaluation:read

Path parameters

evaluation_id
string

Query parameters

limit
integer
offset
integer

Response

Example request

GET /api/projects/:project_id/evaluations/:evaluation_id/backfills
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:evaluation_id/backfills/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/backfills/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "count": 123,
  "next": "http://api.example.org/accounts/?offset=400&limit=100",
  "previous": "http://api.example.org/accounts/?offset=200&limit=100",
  "results": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "status": "running",
      "target": "generation",
      "window_start": "2019-08-24T14:15:22Z",
      "window_end": "2019-08-24T14:15:22Z",
      "conditions": [
        {
          "properties": [
            {}
          ],
          "rollout_percentage": 100
        }
      ],
      "rerun_existing": true,
      "total_count": 0,
      "dispatched_count": 0,
      "skipped_count": 0,
      "created_by": {
        "id": 0,
        "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
        "distinct_id": "string",
        "first_name": "string",
        "last_name": "string",
        "email": "user@example.com",
        "is_email_verified": true,
        "hedgehog_config": {},
        "role_at_organization": "engineering"
      },
      "created_at": "2019-08-24T14:15:22Z",
      "finished_at": "2019-08-24T14:15:22Z"
    }
  ]
}

Create evaluations backfills

Create a backfill: freeze the conditions, count the units, start the walk.

Required API key scopes

evaluation:write

Path parameters

evaluation_id
string

Request parameters

window_start
string
window_end
string
conditions
array
rerun_existing
boolean
Default: false

Response

Example request

POST /api/projects/:project_id/evaluations/:evaluation_id/backfills
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:evaluation_id/backfills/\
	-d window_start="string",\
	-d window_end="string"
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/backfills/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"window_start": "string",
		"window_end": "string"
	}
).json()

Example response

Status 201

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "status": "running",
  "target": "generation",
  "window_start": "2019-08-24T14:15:22Z",
  "window_end": "2019-08-24T14:15:22Z",
  "conditions": [
    {
      "properties": [
        {}
      ],
      "rollout_percentage": 100
    }
  ],
  "rerun_existing": true,
  "total_count": 0,
  "dispatched_count": 0,
  "skipped_count": 0,
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z"
}

Retrieve evaluations backfills

Historical runs of one evaluation over a closed time window (nested under an evaluation).

Required API key scopes

evaluation:read

Path parameters

evaluation_id
string
id
string

Response

Example request

GET /api/projects/:project_id/evaluations/:evaluation_id/backfills/:id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:evaluation_id/backfills/:id/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/backfills/{evaluation_id}/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>",
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "status": "running",
  "target": "generation",
  "window_start": "2019-08-24T14:15:22Z",
  "window_end": "2019-08-24T14:15:22Z",
  "conditions": [
    {
      "properties": [
        {}
      ],
      "rollout_percentage": 100
    }
  ],
  "rerun_existing": true,
  "total_count": 0,
  "dispatched_count": 0,
  "skipped_count": 0,
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z"
}

Create evaluations backfills cancel

Stop a running backfill. Evaluations already dispatched still finish.

Required API key scopes

evaluation:write

Path parameters

evaluation_id
string
id
string

Response

Example request

POST /api/projects/:project_id/evaluations/:evaluation_id/backfills/:id/cancel
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:evaluation_id/backfills/:id/cancel/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/backfills/{evaluation_id}/cancel/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>",
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "status": "running",
  "target": "generation",
  "window_start": "2019-08-24T14:15:22Z",
  "window_end": "2019-08-24T14:15:22Z",
  "conditions": [
    {
      "properties": [
        {}
      ],
      "rollout_percentage": 100
    }
  ],
  "rerun_existing": true,
  "total_count": 0,
  "dispatched_count": 0,
  "skipped_count": 0,
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z"
}

Create evaluations backfills estimate

Count what a backfill over the given window would evaluate, without creating one.

Required API key scopes

evaluation:read

Path parameters

evaluation_id
string

Request parameters

window_start
string
window_end
string
conditions
array
rerun_existing
boolean
Default: false

Response

Example request

POST /api/projects/:project_id/evaluations/:evaluation_id/backfills/estimate
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:evaluation_id/backfills/estimate/\
	-d window_start="string",\
	-d window_end="string"
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/backfills/estimate/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"window_start": "string",
		"window_end": "string"
	}
).json()

Example response

Status 200

RESPONSE
{
  "total_units": 0,
  "unit": "generation",
  "window_start": "2019-08-24T14:15:22Z",
  "window_end": "2019-08-24T14:15:22Z"
}

Retrieve evaluations

Required API key scopes

evaluation:read

Path parameters

id
string

Response

Example request

GET /api/projects/:project_id/evaluations/:id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:id/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "name": "string",
  "description": "string",
  "directory_id": "dca58562-24cf-4cbc-aa50-5a4039358aa5",
  "enabled": true,
  "status": "active",
  "status_reason": "provider_key_required",
  "status_reason_detail": "string",
  "evaluation_type": "llm_judge",
  "evaluation_config": {
    "prompt": "string"
  },
  "output_type": "boolean",
  "output_config": {
    "allows_na": true,
    "true_is_failure": true
  },
  "conditions": [
    {
      "id": "string",
      "rollout_percentage": 100,
      "properties": [
        {}
      ]
    }
  ],
  "target": "generation",
  "target_config": {
    "strategy": "fixed_window",
    "window_seconds": 10
  },
  "model_configuration": {
    "provider": "openai",
    "model": "string",
    "provider_key_id": "f265db88-9bcc-4e5b-add5-bfd9a815465c",
    "provider_key_name": "string"
  },
  "created_at": "2019-08-24T14:15:22Z",
  "updated_at": "2019-08-24T14:15:22Z",
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "deleted": true,
  "user_access_level": "string"
}

Update evaluations

Required API key scopes

evaluation:write

Path parameters

id
string

Request parameters

name
string
description
string
directory_id
stringnull
enabled
boolean
evaluation_type
evaluation_config
output_type
output_config
object
conditions
array
target
target_config
model_configuration
deleted
boolean

Response

Example request

PATCH /api/projects/:project_id/evaluations/:id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl -X PATCH \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:id/\
	-d name="string"
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.patch(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"name": "string"
	}
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "name": "string",
  "description": "string",
  "directory_id": "dca58562-24cf-4cbc-aa50-5a4039358aa5",
  "enabled": true,
  "status": "active",
  "status_reason": "provider_key_required",
  "status_reason_detail": "string",
  "evaluation_type": "llm_judge",
  "evaluation_config": {
    "prompt": "string"
  },
  "output_type": "boolean",
  "output_config": {
    "allows_na": true,
    "true_is_failure": true
  },
  "conditions": [
    {
      "id": "string",
      "rollout_percentage": 100,
      "properties": [
        {}
      ]
    }
  ],
  "target": "generation",
  "target_config": {
    "strategy": "fixed_window",
    "window_seconds": 10
  },
  "model_configuration": {
    "provider": "openai",
    "model": "string",
    "provider_key_id": "f265db88-9bcc-4e5b-add5-bfd9a815465c",
    "provider_key_name": "string"
  },
  "created_at": "2019-08-24T14:15:22Z",
  "updated_at": "2019-08-24T14:15:22Z",
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "deleted": true,
  "user_access_level": "string"
}

Delete evaluations

Hard delete of this model is not allowed. Use a patch API call to set "deleted" to true

Required API key scopes

evaluation:write

Path parameters

id
string

Example request

DELETE /api/projects/:project_id/evaluations/:id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl  -X DELETE \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/:id/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.delete(
    "<ph_app_host>/api/projects/{project_id}/evaluations/{evaluation_id}/".format(
        project_id=project_id,
		evaluation_id="<the evaluation id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
)

Example response

Status 405 No response body

Create evaluations test hog

Test Hog evaluation code against sample events without saving.

Required API key scopes

evaluation:read

Request parameters

source
string
sample_count
integer
Default: 5
allows_na
boolean
Default: false
conditions
array
target
Default: generation
target_config

Response

Example request

POST /api/projects/:project_id/evaluations/test_hog
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/evaluations/test_hog/\
	-d source="string"
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/evaluations/test_hog/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"source": "string"
	}
).json()

Example response

Status 200

RESPONSE
{
  "results": [
    {
      "sample_id": "string",
      "sample_type": "generation",
      "event_uuid": "string",
      "trace_id": "string",
      "input_preview": "string",
      "output_preview": "string",
      "result": true,
      "reasoning": "string",
      "error": "string"
    }
  ],
  "message": "string"
}