{"openapi":"3.1.0","info":{"title":"2027.dev evals — public REST API","version":"1.0.0","description":"Org-scoped eval automation. Auth via Bearer token (API key from /<org>/settings)."},"servers":[{"url":"https://2027.dev/evals"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","description":"API key from /<org>/settings"}},"schemas":{"PromptSummary":{"type":"object","properties":{"id":{"type":"string","format":"uuid","example":"8e2f3b4c-1234-4abc-9def-0123456789ab"},"title":{"type":["string","null"],"example":"Sign up and create a project"},"text":{"type":"string","example":"Sign up at example.com, create a new project, and invite a teammate."},"endGoal":{"type":["string","null"],"example":"Successful first API call"},"status":{"type":"string","example":"active"},"templateVars":{"type":["array","null"],"items":{"type":"string"},"example":["accountId","scenario"]},"createdAt":{"type":"string","format":"date-time","example":"2026-03-01T12:34:56.000Z"}},"required":["id","title","text","endGoal","status","templateVars","createdAt"]},"Prompt":{"type":"object","properties":{"id":{"type":"string","format":"uuid","example":"8e2f3b4c-1234-4abc-9def-0123456789ab"},"title":{"type":["string","null"],"example":"Sign up and create a project"},"text":{"type":"string","example":"Sign up at example.com, create a new project, and invite a teammate."},"endGoal":{"type":["string","null"],"example":"Successful first API call"},"status":{"type":"string","example":"active"},"frequency":{"type":["string","null"],"example":"weekly"},"runsPerEval":{"type":["integer","null"],"example":3},"targetProviderId":{"type":["string","null"],"format":"uuid","example":"11111111-2222-3333-4444-555555555555"},"additionalUrls":{"type":["array","null"],"items":{"type":"object","additionalProperties":{}},"example":[{"url":"https://competitor.com","name":"Competitor"}]},"templateVars":{"type":["array","null"],"items":{"type":"string"},"example":["accountId","scenario"]},"createdAt":{"type":"string","format":"date-time","example":"2026-03-01T12:34:56.000Z"}},"required":["id","title","text","endGoal","status","frequency","runsPerEval","targetProviderId","additionalUrls","templateVars","createdAt"]},"Run":{"type":"object","properties":{"runId":{"type":"string","format":"uuid"},"status":{"type":"string","example":"pending"},"report":{"type":["object","null"],"properties":{"slug":{"type":"string","example":"abc123-report-slug"},"url":{"type":"string","format":"uri","example":"https://2027.dev/evals/example.com/reports/abc123-report-slug"},"score":{"type":["number","null"],"example":78.5},"grade":{"type":["string","null"],"example":"B+"},"metrics":{"type":"object","properties":{"time":{"type":"string"},"timeDetail":{"type":"string"},"errors":{"type":"number"},"errorsDetail":{"type":"string"},"toolCalls":{"type":"number"},"toolCallsDetail":{"type":"string"},"cost":{"type":"string"},"costDetail":{"type":"string"},"interruptions":{"type":"number"},"timeSeconds":{"type":["number","null"],"example":134},"costUsd":{"type":["number","null"],"example":0.12}},"required":["time","timeDetail","errors","errorsDetail","toolCalls","toolCallsDetail","cost","costDetail","interruptions"],"example":{"time":"2m 14s","timeDetail":"from first prompt to final answer","errors":1,"errorsDetail":"1 transient 500 from /api/run","toolCalls":12,"toolCallsDetail":"8 reads, 4 writes","cost":"$0.12","costDetail":"tokens + tool budget","interruptions":0,"timeSeconds":134,"costUsd":0.12}},"dimensions":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string"},"weight":{"type":"string"},"score":{"type":"number"},"grade":{"type":"string"},"desc":{"type":"string"}},"required":["name","weight","score","grade","desc"]},"example":[{"name":"Setup Friction","weight":"25%","score":80,"grade":"B+","desc":"Smooth onboarding overall."}]}},"required":["slug","url"]},"statusUrl":{"type":"string","format":"uri"},"runUrl":{"type":"string","format":"uri","example":"https://2027.dev/evals/example.com/runs/abc12345-1234-4abc-9def-0123456789ab"},"completedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":"string","format":"date-time","example":"2026-05-08T20:00:00.000Z"},"failureReason":{"type":["string","null"]},"templateArgs":{"type":["object","null"],"additionalProperties":{"type":"string"},"example":{"accountId":"acme-ci","scenario":"checkout"},"description":"Resolved template variable values for this run. Stored and returned in plaintext — do not pass secrets here."},"prompt":{"$ref":"#/components/schemas/PromptSummary"}},"required":["runId","status","report","statusUrl","runUrl","completedAt","createdAt","failureReason","prompt"]},"RunRequest":{"type":"object","properties":{"urlMap":{"type":"object","additionalProperties":{"type":"string","format":"uri"},"example":{"default":"https://example.com"}},"templateArgs":{"type":"object","additionalProperties":{"type":"string","maxLength":2000},"example":{"accountId":"acme-ci","scenario":"checkout"}}},"required":["urlMap"]},"RunResponse":{"type":"object","properties":{"runId":{"type":"string","format":"uuid"},"statusUrl":{"type":"string","format":"uri"},"runUrl":{"type":"string","format":"uri","example":"https://2027.dev/evals/example.com/runs/abc12345-1234-4abc-9def-0123456789ab"}},"required":["runId","statusUrl","runUrl"]},"Error":{"type":"object","properties":{"error":{"type":"string"},"detail":{"type":"string"}},"required":["error"]}},"parameters":{}},"paths":{"/api/v1/prompts":{"get":{"tags":["v1"],"summary":"List prompts for the authenticated org","security":[{"bearerAuth":[]}],"parameters":[{"schema":{"type":"string","enum":["1"],"description":"Pass '1' to include archived prompts"},"required":false,"description":"Pass '1' to include archived prompts","name":"includeArchived","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":1000,"default":100,"description":"Max rows to return"},"required":false,"description":"Max rows to return","name":"limit","in":"query"},{"schema":{"type":["integer","null"],"minimum":0,"maximum":100000,"default":0,"description":"Rows to skip"},"required":false,"description":"Rows to skip","name":"offset","in":"query"}],"responses":{"200":{"description":"List of prompts","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/PromptSummary"}}}}},"400":{"description":"Bad request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"401":{"description":"Unauthorized","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"500":{"description":"Internal server error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/v1/prompts/{promptId}":{"get":{"tags":["v1"],"summary":"Get a single prompt by ID","security":[{"bearerAuth":[]}],"parameters":[{"schema":{"type":"string","format":"uuid"},"required":true,"name":"promptId","in":"path"}],"responses":{"200":{"description":"Prompt detail","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Prompt"}}}},"401":{"description":"Unauthorized","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"500":{"description":"Internal server error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/v1/prompts/{promptId}/run":{"post":{"tags":["v1"],"summary":"Trigger a new eval run","security":[{"bearerAuth":[]}],"parameters":[{"schema":{"type":"string","format":"uuid"},"required":true,"name":"promptId","in":"path"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunRequest"}}}},"responses":{"200":{"description":"Run accepted","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunResponse"}}}},"400":{"description":"Bad request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"401":{"description":"Unauthorized","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"500":{"description":"Internal server error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/v1/runs":{"get":{"tags":["v1"],"summary":"List eval runs for the authenticated org (newest first)","security":[{"bearerAuth":[]}],"parameters":[{"schema":{"type":"string","format":"uuid","description":"Filter runs by prompt id"},"required":false,"description":"Filter runs by prompt id","name":"promptId","in":"query"},{"schema":{"type":"string","enum":["published","draft","archived"],"description":"Filter by report status (e.g. 'published')"},"required":false,"description":"Filter by report status (e.g. 'published')","name":"reportStatus","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":1000,"default":100,"description":"Max rows to return"},"required":false,"description":"Max rows to return","name":"limit","in":"query"},{"schema":{"type":["integer","null"],"minimum":0,"maximum":100000,"default":0,"description":"Rows to skip"},"required":false,"description":"Rows to skip","name":"offset","in":"query"}],"responses":{"200":{"description":"List of runs","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/Run"}}}}},"400":{"description":"Bad request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"401":{"description":"Unauthorized","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"500":{"description":"Internal server error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/v1/runs/{runId}":{"get":{"tags":["v1"],"summary":"Get the status of an eval run","security":[{"bearerAuth":[]}],"parameters":[{"schema":{"type":"string","format":"uuid"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Run status","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Run"}}}},"401":{"description":"Unauthorized","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"404":{"description":"Not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"500":{"description":"Internal server error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}}},"webhooks":{}}