curl --request GET \
--url https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"judgment": "<string>",
"type": "bool",
"version": 2,
"outcomes": 1,
"epochs": [
{
"engine": "<string>",
"engine_version": "<string>",
"outcomes": 1,
"method": "platt",
"fitted_at": "2023-11-07T05:31:56Z",
"from_previous_epoch": true,
"raw": {
"accuracy": 0.5,
"expected_calibration_error": 1,
"log_loss": 1,
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
],
"mean_level_distance": 1
},
"calibrated": {
"accuracy": 0.5,
"expected_calibration_error": 1,
"log_loss": 1,
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
],
"mean_level_distance": 1
}
}
]
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}Get the calibration report for the active version
How well the active version’s answers match the outcomes joined to its
evaluations, per engine epoch, before and after calibration (§6.10).
Calibration is fitted per judgment version and per engine epoch, first
shortly after the judgment’s first outcomes and then nightly. From 100
outcomes it is Platt scaling for bool and temperature scaling for
choice and score; from 1,000, isotonic regression. Below 100
outcomes an epoch has no calibration, and its calibrated metrics are
null. The raw metrics are computed at request time and include every
outcome posted so far. See calibration.
On a prefix path the report pools the outcomes of every namespace under the prefix, which is the fit inherited answers use (§7.7).
curl --request GET \
--url https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.vainona.ai/v1/namespaces/{ns}/judgments/{name}/calibration")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"judgment": "<string>",
"type": "bool",
"version": 2,
"outcomes": 1,
"epochs": [
{
"engine": "<string>",
"engine_version": "<string>",
"outcomes": 1,
"method": "platt",
"fitted_at": "2023-11-07T05:31:56Z",
"from_previous_epoch": true,
"raw": {
"accuracy": 0.5,
"expected_calibration_error": 1,
"log_loss": 1,
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
],
"mean_level_distance": 1
},
"calibrated": {
"accuracy": 0.5,
"expected_calibration_error": 1,
"log_loss": 1,
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
],
"mean_level_distance": 1
}
}
]
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {}
}
}Authorizations
An organization API key. Keys carry a role (read_write or
read_only) and may be restricted to a namespace prefix such as
acme/*, or to one namespace such as acme/prod/tenant_1. A prefix
matches on a / boundary: acme/prod/tenant_1* covers
acme/prod/tenant_1 and everything under acme/prod/tenant_1/,
never acme/prod/tenant_12.
Path Parameters
A namespace name, or a template prefix ending in /* (§7.7), with any
/ sent as %2F: acme%2Fprod%2Ftenant_123 or acme%2Fprod%2F*.
A namespace name, or a template prefix: a namespace path ending in
/*, such as acme/prod/*, which every namespace under acme/prod/
inherits judgments from (§7.7). Up to 256 bytes.
1 - 256^[A-Za-z0-9._:/-]+(/\*)?$A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..
1 - 128^[A-Za-z0-9_-]+$Response
The report.
How well the active version's answers match their outcomes (§6.10).
epochs holds one entry per engine epoch with outcomes, newest first.
An epoch is the engine_version its evaluations were computed under:
each Jev epoch label, or the exact version for a pinned engine.
A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..
1 - 128^[A-Za-z0-9_-]+$bool, choice, score The active version the report is for.
x >= 1Outcomes joined to evaluations of this version, across every epoch.
x >= 0Show child attributes
Show child attributes