LLM Integrations

prompts/summarise_ticket.yaml

prompt:  summarise_ticket
version: 14
system: |
  You are a support analyst. Be concise.
  Never include customer PII.
tests:   38 passing  # runs in CI

schemas/triage.ts

const Triage = z.object({
  intent:   z.enum(["refund", "bug", "billing"]),
  priority: z.number().min(1).max(4),
  summary:  z.string().max(280),
})
// validated before it reaches your code

tools/orders.ts

tools: {
  lookupOrder: {
    input: z.object({ id: z.string() }),
    run: ({ id }) => orders.get(id),
    // scoped: read-only, own tenant
  },
}

stream · /v1/generate

event: token  data: "Your refund"
event: token  data: " has been issued"
event: tool   data: lookupOrder(4821)
event: token  data: " and should arrive"
event: done   data: { tokens: 212 }

routing.yaml

routes:
  - when: task == "classify"
    use:  small-fast
  - when: input.tokens > 50000
    use:  long-context
  - default: frontier

fallbacks.yaml

fallbacks:
  frontier:  [secondary, oss-hosted]
  trigger:   [timeout > 8s, 429, 5xx]
  retries:   2  # exponential backoff
  parity:    eval suite must pass

eval · support-replies

cases         212
accuracy      0.94  ▲ +0.03
policy_safe   1.00
tone_match    0.91
p95_latency   1.8s   ✓ budget 2.5s

trace · 7f3a

summarise_ticket
├ route      frontier            4ms
├ model      1,284 → 212 tokens  1.42s
├ guardrail  pii_filter          ok
└ cost       $0.0061

budgets.yaml

budgets:
  per_request:  $0.02
  per_user_day: $1.50
cache:          semantic · 24h
downgrade_when: budget > 80%