\u5BFC\u8A00\uFF1A\u5F53 LLM \u5E94\u7528\u957F\u51FA"\u57FA\u7840\u8BBE\u65BD\u5C42"

2025 \u5E74\u4E4B\u524D\uFF0C\u5927\u591A\u6570\u56E2\u961F\u8C03\u7528 OpenAI\u3001Anthropic \u8FD8\u662F\u76F4\u63A5\u5728\u4E1A\u52A1\u4EE3\u7801\u91CC\u5199"fetch + API key"\u3002\u4E00\u5E74\u8FC7\u53BB\uFF0C\u8FD9\u79CD"\u88F8\u8C03"\u6A21\u5F0F\u5728\u751F\u4EA7\u73AF\u5883\u91CC\u66B4\u9732\u51FA\u4E09\u7C7B\u95EE\u9898\uFF1A

1. \u4F9B\u5E94\u5546\u8026\u5408\uFF1A\u6A21\u578B\u8C03\u7528\u6563\u843D\u5728\u51E0\u5341\u4E2A\u6587\u4EF6\u91CC\uFF0C\u6362 Anthropic\u2192OpenAI \u8981\u5168\u5C40\u641C\u66FF\u6362\uFF0C\u6BCF\u4E2A\u8C03\u7528\u70B9\u90FD\u8981\u91CD\u5199\u9519\u8BEF\u5904\u7406\u4E0E\u91CD\u8BD5\u903B\u8F91\u3002
2. \u6210\u672C\u4E0D\u53EF\u89C1\uFF1A\u6309\u90E8\u95E8/\u9879\u76EE/\u7528\u6237\u7C92\u5EA6\u7684 token \u6D88\u8017\u9700\u8981\u81EA\u5DF1\u62FC\u65E5\u5FD7\u3001\u6E05\u6D17\u3001\u5165\u5E93\uFF1B\u505A\u51FA\u6765\u7ECF\u5E38\u6EDE\u540E 24 \u5C0F\u65F6\u3002
3. \u53EF\u9760\u6027\u9760\u8FD0\u6C14\uFF1A\u5F53\u4E0A\u6E38 API \u6296\u52A8\u3001\u9650\u6D41\u3001\u964D\u7EA7\uFF0C\u4E1A\u52A1\u53EA\u80FD"\u786C\u6302"\u2014\u2014\u6CA1\u6709 fallback\u3001\u6CA1\u6709 cache\u3001\u6CA1\u6709 circuit breaker\u3002

\u89E3\u51B3\u65B9\u6848\u5728 2024\u20132026 \u5E74\u9010\u6E10\u6536\u655B\u6210\u4E00\u5C42\u72EC\u7ACB\u7684"AI Gateway"\u3002\u5B83\u50CF\u4E03\u5C42\u7F51\u7EDC\u91CC\u7684\u53CD\u5411\u4EE3\u7406 + WAF\uFF1A\u5355\u70B9\u5165\u53E3\u3001\u7EDF\u4E00\u9274\u6743\u3001\u667A\u80FD\u8DEF\u7531\u3001\u53EF\u89C2\u6D4B\u3001\u9650\u6D41\u964D\u7EA7\u3002\u4E0D\u540C\u5F00\u6E90/\u5546\u4E1A\u5B9E\u73B0\u5DF2\u7ECF\u628A\u8FD9\u4E2A\u62BD\u8C61\u505A\u5230\u4E86\u53EF\u751F\u4EA7\u72B6\u6001\u3002

\u672C\u6587\u9009\u56DB\u79CD\u5178\u578B\u65B9\u6848\u505A\u5DE5\u7A0B\u62C6\u89E3\uFF1ALiteLLM\uFF08Python \u751F\u6001\u7684\u4E8B\u5B9E\u6807\u51C6\uFF09\u3001Portkey\uFF08TypeScript \u7684\u73B0\u4EE3\u9009\u62E9\uFF09\u3001Cloudflare AI Gateway\uFF08\u8FB9\u7F18\u7F51\u5173 / SaaS\uFF09\u3001OpenRouter\uFF08\u591A\u4F9B\u5E94\u5546\u805A\u5408\u5E73\u53F0\uFF09\u3002\u91CD\u70B9\u4E0D\u662F"\u54EA\u5BB6\u6700\u597D"\uFF0C\u800C\u662F\u628A\u6BCF\u5BB6\u5728\u5DE5\u7A0B\u4E0A\u8981\u89E3\u51B3\u7684\u540C\u4E00\u7C7B\u95EE\u9898\u2014\u2014\u8DEF\u7531\u3001\u7F13\u5B58\u3001\u53EF\u89C2\u6D4B\u3001\u9650\u6D41\u3001Guardrails\u2014\u2014\u843D\u5230\u5177\u4F53\u63A5\u53E3\u3001\u5177\u4F53\u5B57\u6BB5\u3001\u5177\u4F53 YAML/TypeScript \u914D\u7F6E\u4E0A\u3002

\u4E00\u3001\u7EDF\u4E00\u63A5\u53E3\uFF1A\u628A 100+ \u6A21\u578B\u6536\u6210\u4E00\u4E2A OpenAI \u517C\u5BB9\u7AEF\u70B9

1.1 \u884C\u4E1A\u6807\u51C6\u6536\u655B\u4E8E OpenAI \u683C\u5F0F

\u622A\u81F3 2026 \u5E74\u4E2D\uFF0C\u51E0\u4E4E\u6240\u6709\u4E3B\u6D41\u4F9B\u5E94\u5546\u90FD\u63D0\u4F9B OpenAI Chat Completions \u517C\u5BB9\u7AEF\u70B9\u3002\u8FD9\u610F\u5473\u7740\u7F51\u5173\u5C42\u7684"\u7FFB\u8BD1"\u5DE5\u4F5C\u88AB\u6781\u5927\u538B\u7F29\u2014\u2014\u7F51\u5173\u53EA\u9700\u8981\u5904\u7406\u9274\u6743\u3001\u8BF7\u6C42\u4F53\u5B57\u6BB5\u5DEE\u5F02\uFF08\u5982 Anthropic \u7684 `system` \u9876\u7EA7\u5B57\u6BB5\uFF09\u3001\u54CD\u5E94\u5B57\u6BB5\u5DEE\u5F02\uFF08\u5982 Anthropic \u7684 `content[].text` vs OpenAI \u7684 `choices[0].message.content`\uFF09\u3002

LiteLLM \u628A\u8FD9\u79CD"\u7FFB\u8BD1"\u505A\u5230\u6781\u81F4\u2014\u2014\u5B83\u672C\u8EAB\u662F\u4E00\u4E2A Python SDK + Proxy \u53CC\u5F62\u6001\u7684\u5E93\uFF0C\u5BF9\u5916\u66B4\u9732 OpenAI \u517C\u5BB9\u7684 `/v1/chat/completions` \u7AEF\u70B9\uFF0C\u628A `openai/gpt-4o`\u3001`anthropic/claude-sonnet-4.5`\u3001`bedrock/anthropic.claude-3` \u7B49\u547D\u540D\u76F4\u63A5\u8DEF\u7531\u5230\u5BF9\u5E94\u4F9B\u5E94\u5546\u3002GitHub \u6570\u636E\uFF082026-06-13 \u62C9\u53D6\uFF09\uFF1A50,208 stars / 8,837 forks\u3002

```python

LiteLLM Python SDK - \u4E00\u884C\u8C03\u7528\u4EFB\u610F\u6A21\u578B

from litellm import completion import os

os.environ["OPENAI_API_KEY"] = "sk-..." os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."

resp = completion( model="anthropic/claude-sonnet-4.5", messages=[{"role":"user","content":"\u7528\u4E00\u53E5\u8BDD\u4ECB\u7ECD AI Gateway"}], ) print(resp.choices[0].message.content) ```

Portkey\uFF0812,049 stars / 1,125 forks\uFF0CMIT\uFF0CTypeScript \u5B9E\u73B0\uFF0C\u622A\u81F3 2026-06-13\uFF09\u7684\u5B9A\u4F4D\u7A0D\u6709\u4E0D\u540C\u2014\u2014\u5B83\u5F3A\u8C03"AI Gateway + Guardrails \u4E00\u4F53\u5316"\uFF0C\u81EA\u79F0\u652F\u6301 1,600+ \u6A21\u578B\u300150+ \u5185\u7F6E Guardrails\u3002\u5B83\u7684 SDK \u4E5F\u517C\u5BB9 OpenAI\uFF0C\u4F46\u66F4\u7A81\u51FA production-grade \u7279\u6027\uFF1Afallback chains\u3001conditional routing\u3001automatic retries\u3001semantic caching\u3002

```typescript // Portkey TypeScript SDK import { Portkey } from 'portkey-ai';

const client = new Portkey({ apiKey: process.env.PORTKEY_API_KEY!, config: 'pc-***', // Portkey Config ID\uFF08\u8DEF\u7531/\u91CD\u8BD5\u7B56\u7565\u4FDD\u5B58\u4E91\u7AEF\uFF09 });

const response = await client.chat.completions.create({ model: 'gpt-4o', messages: [{ role: 'user', content: '\u4F60\u597D' }], }); ```

1.2 \u4E24\u79CD\u90E8\u7F72\u5F62\u6001\uFF1A\u81EA\u6258\u7BA1 Proxy vs SaaS

```mermaid graph LR A[\u4E1A\u52A1\u4EE3\u7801] -->|OpenAI \u683C\u5F0F HTTP| B(AI Gateway) B -->|OpenAI \u683C\u5F0F| C[OpenAI] B -->|Anthropic \u683C\u5F0F| D[Anthropic] B -->|Vertex \u683C\u5F0F| E[Google Vertex] B -->|\u81EA\u5EFA vLLM \u683C\u5F0F| F[\u81EA\u6258\u7BA1 vLLM] B --> G[\u7EDF\u4E00\u65E5\u5FD7 / \u8BA1\u91CF] B --> H[\u8BED\u4E49\u7F13\u5B58\u5C42] ```

\u5224\u65AD\u6807\u51C6\uFF1A\u6570\u636E\u5408\u89C4\u8981\u6C42\u9AD8\uFF08\u91D1\u878D/\u533B\u7597\uFF09\u2192 LiteLLM Proxy \u81EA\u6258\u7BA1\uFF1B\u5DF2\u6709 Cloudflare \u751F\u6001 \u2192 AI Gateway\uFF1B\u56E2\u961F < 5 \u4EBA\u4E14\u4E0D\u613F\u8FD0\u7EF4 \u2192 OpenRouter\uFF1B\u9700\u8981 guardrails / \u590D\u6742 fallback \u94FE \u2192 Portkey\u3002

\u4E8C\u3001\u667A\u80FD\u8DEF\u7531\uFF1A\u6545\u969C\u5207\u6362\u3001\u8D1F\u8F7D\u5747\u8861\u3001\u6309\u6210\u672C\u8C03\u5EA6

\u7F51\u5173\u5C42\u6700\u91CD\u8981\u7684\u5DE5\u7A0B\u4EF7\u503C\u662F**\u5728\u591A\u4E2A\u4E0A\u6E38\u4E4B\u95F4\u505A\u51B3\u7B56**\uFF0C\u800C\u4E0D\u662F\u5355\u7EAF\u8F6C\u53D1\u3002

2.1 \u6545\u969C\u5207\u6362\uFF08Fallback Chains\uFF09

LiteLLM Router \u652F\u6301 `fallbacks` \u914D\u7F6E\uFF1A\u5B9A\u4E49\u4E00\u4E2A\u6A21\u578B\u5217\u8868\uFF0C\u6309\u987A\u5E8F\u91CD\u8BD5\uFF0C\u7B2C\u4E00\u4E2A\u5931\u8D25\u81EA\u52A8\u5207\u5230\u4E0B\u4E00\u4E2A\uFF1A

```python

LiteLLM Router - \u4E09\u7EA7 fallback

from litellm import Router

model_list = [ {"model_name": "gpt4o-prod", "litellm_params": {"model": "openai/gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}}, {"model_name": "claude-prod", "litellm_params": {"model": "anthropic/claude-sonnet-4.5", "api_key": os.environ["ANTHROPIC_API_KEY"]}}, {"model_name": "deepseek-prod", "litellm_params": {"model": "deepseek/deepseek-chat", "api_key": os.environ["DEEPSEEK_API_KEY"]}}, ]

router = Router( model_list=model_list, fallbacks=[{"gpt4o-prod": ["claude-prod"]}, {"claude-prod": ["deepseek-prod"]}], num_retries=2, timeout=30, ) ```

Portkey \u7684 `config` JSON \u8868\u8FBE\u529B\u66F4\u5F3A\uFF1A

```json { "strategy": { "mode": "fallback" }, "targets": [ { "provider": "openai", "model": "gpt-4o", "weight": 0.7 }, { "provider": "anthropic", "model": "claude-sonnet-4.5", "weight": 0.3 } ], "retry": { "attempts": 3, "on_status_codes": [429, 500, 502, 503] } } ```

2.2 \u8D1F\u8F7D\u5747\u8861\u4E0E\u6309\u6210\u672C\u8DEF\u7531

\u5F53\u4E00\u4E2A\u6A21\u578B\u591A\u5BB6\u4F9B\u5E94\u5546\u90FD\u63D0\u4F9B\u65F6\uFF08\u4F8B\u5982 Llama-3-70B \u540C\u65F6\u5728 Together / Fireworks / Groq \u4E0A\uFF09\uFF0C\u53EF\u4EE5\u901A\u8FC7\u6743\u91CD\u6216"\u4EF7\u683C\u4F18\u5148"\u7B56\u7565\u52A8\u6001\u5206\u914D\u3002

```python

LiteLLM - \u540C\u6A21\u578B\u591A\u90E8\u7F72\u8D1F\u8F7D\u5747\u8861

model_list = [ {"model_name": "llama3-70b", "litellm_params": {"model": "together_ai/meta-llama/Meta-Llama-3-70B", "api_key": "..."}}, {"model_name": "llama3-70b", "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct", "api_key": "..."}}, {"model_name": "llama3-70b", "litellm_params": {"model": "groq/llama3-70b-8192", "api_key": "..."}}, ] router = Router(model_list=model_list, routing_strategy="usage-based-routing-v2") ```

`usage-based-routing-v2` \u662F LiteLLM \u5185\u7F6E\u7B56\u7565\uFF0C\u4F1A\u628A\u8BF7\u6C42\u53D1\u5F80"\u6700\u8FD1\u6700\u5C11\u7528"\u7684\u4E0A\u6E38\uFF0C\u907F\u514D\u5355\u4E00\u4F9B\u5E94\u5546\u89E6\u53D1\u9650\u6D41\u3002

2.3 Cloudflare AI Gateway \u7684 Provider-Native \u6A21\u5F0F

Cloudflare AI Gateway \u63D0\u4F9B\u4E24\u79CD\u8C03\u7528\u6A21\u5F0F\uFF1AUnified API\uFF08OpenAI \u517C\u5BB9\uFF09\u548C Provider Native\uFF08\u76F4\u63A5\u4EE3\u7406\u5230\u76EE\u6807\u4F9B\u5E94\u5546\uFF0C\u4EC5\u505A\u65E5\u5FD7/\u7F13\u5B58/\u8BA1\u8D39\u589E\u5F3A\uFF09\u3002\u540E\u8005\u5BF9\u5DF2\u6709 Anthropic SDK \u7684\u8FC1\u79FB\u6210\u672C\u4E3A\u96F6\u2014\u2014\u53EA\u628A `base_url` \u6539\u6210 CF \u7684 endpoint \u5373\u53EF\u3002

```typescript // \u628A Anthropic SDK \u6307\u5411 Cloudflare AI Gateway\uFF08Provider Native \u6A21\u5F0F\uFF09 import Anthropic from '@anthropic-ai/sdk';

const anthropic = new Anthropic({ apiKey: process.env.CF_AIGATEWAY_TOKEN, // CF Gateway \u9881\u53D1\u7684 token baseURL: 'https://gateway.ai.cloudflare.com/v1/<account_id>/<gateway_id>/anthropic', });

const msg = await anthropic.messages.create({ model: 'claude-sonnet-4.5', max_tokens: 1024, messages: [{ role: 'user', content: 'Hello' }], }); ```

Cloudflare \u5B98\u65B9\u6587\u6863\uFF08\u622A\u81F3 2026-06-13\uFF09\u652F\u6301\u7684\u4F9B\u5E94\u5546\u5217\u8868\u5305\u62EC\uFF1AOpenAI\u3001Anthropic\u3001Workers AI\u3001Amazon Bedrock\u3001Google Vertex AI\u3001Azure OpenAI\u3001Groq\u3001HuggingFace\u3001Mistral\u3001Replicate\u3001xAI\u3001DeepSeek \u7B49\u2014\u2014\u57FA\u672C\u8986\u76D6\u4E86 2026 \u5E74\u6240\u6709\u4E3B\u6D41\u9009\u9879\u3002

\u4E09\u3001\u8BED\u4E49\u7F13\u5B58\uFF1A\u628A"\u91CD\u590D\u63D0\u95EE"\u7684\u6210\u672C\u6253\u5230\u63A5\u8FD1\u96F6

LLM \u8C03\u7528\u6700\u5927\u7684\u9690\u6027\u6210\u672C\u662F**\u91CD\u590D\u63D0\u95EE**\u3002\u5BA2\u670D FAQ\u3001\u6587\u6863\u95EE\u7B54\u3001\u4EE3\u7801\u8865\u5168\u90FD\u5B58\u5728\u5927\u91CF\u76F8\u4F3C\u8BF7\u6C42\u3002\u5982\u679C\u6BCF\u6B21\u90FD\u91CD\u65B0\u8C03\u7528\u6A21\u578B\uFF0C\u7B49\u4E8E\u5728\u70E7\u94B1\u3002

3.1 LiteLLM \u8BED\u4E49\u7F13\u5B58

LiteLLM Proxy \u5185\u7F6E\u8BED\u4E49\u7F13\u5B58\uFF08\u57FA\u4E8E embedding \u76F8\u4F3C\u5EA6\uFF09\uFF0C\u53EF\u901A\u8FC7\u914D\u7F6E\u542F\u7528\uFF1A

```yaml

litellm config.yaml

litellm_settings: cache: True cache_params: type: qdrant-semantic # \u9700\u914D\u5408 Qdrant \u5411\u91CF\u5E93 qdrant_url: http://localhost:6333 qdrant_collection_name: llm-cache similarity_threshold: 0.95 # \u4F59\u5F26\u76F8\u4F3C\u5EA6\u9608\u503C

model_list:

• model_name: gpt4o litellm_params: model: openai/gpt-4o api_key: os.environ/OPENAI_API_KEY ```

\u652F\u6301\u7684 backend \u5217\u8868\u5305\u62EC Redis\u3001Qdrant\u3001Postgres\uFF08pgvector\uFF09\u3001S3\u3001In-memory\u3002`similarity_threshold` \u662F\u6838\u5FC3\u65CB\u94AE\u2014\u20140.95 \u610F\u5473\u7740"\u51E0\u4E4E\u76F8\u540C\u95EE\u9898\u624D\u547D\u4E2D"\uFF0C0.85 \u53EF\u80FD\u8BEF\u547D\u4E2D\u3002

3.2 Portkey \u5185\u7F6E\u8BED\u4E49\u7F13\u5B58

Portkey \u9ED8\u8BA4\u5F00\u542F simple cache\uFF08\u7CBE\u786E\u5339\u914D\uFF09\uFF0C\u5F00\u542F semantic cache \u9700\u5728 config \u91CC\u6307\u5B9A\uFF1A

```json { "cache": { "mode": "semantic", "max_age": 3600, "similarity_threshold": 0.85 } } ```

Portkey \u7684\u4F18\u52BF\u662F**\u7F13\u5B58\u7C92\u5EA6\u53EF\u914D\u7F6E\u5230 namespace + user \u7EF4\u5EA6**\u2014\u2014\u540C\u4E00\u95EE\u9898\u5728\u5BA2\u6237 A \u7684\u5BF9\u8BDD\u91CC\u547D\u4E2D\u4E00\u6B21\uFF0C\u4E0D\u4F1A\u6C61\u67D3\u5BA2\u6237 B \u7684\u4E0A\u4E0B\u6587\u3002

3.3 Cloudflare AI Gateway \u8FB9\u7F18\u7F13\u5B58

Cloudflare \u7684\u7F13\u5B58\u7B56\u7565\u6700"\u6FC0\u8FDB"\u2014\u2014\u57FA\u4E8E\u54CD\u5E94\u5934\u7684 `Cache-Control` \u6216\u81EA\u5B9A\u4E49 `cf-aig-cache-ttl` header \u51B3\u5B9A TTL\u3002\u547D\u4E2D\u53D1\u751F\u5728 300+ \u8FB9\u7F18\u8282\u70B9\uFF0C\u5BF9\u5168\u7403\u7528\u6237\u5EF6\u8FDF\u6781\u4F4E\u3002

```typescript // \u5728\u8BF7\u6C42\u91CC\u8BBE\u7F6E\u7F13\u5B58 TTL fetch('https://gateway.ai.cloudflare.com/v1/.../openai/chat/completions', { method: 'POST', headers: { 'Authorization': 'Bearer *** 'cf-aig-cache-ttl': '86400', // 24 \u5C0F\u65F6 'Content-Type': 'application/json', }, body: JSON.stringify({...}), }); ```

\u5751\uFF1A\u8BED\u4E49\u7F13\u5B58**\u7EDD\u4E0D**\u9002\u5408"\u7528\u6237\u7279\u5B9A\u4E0A\u4E0B\u6587"\u7684\u8BF7\u6C42\uFF08\u5982\u4E2A\u6027\u5316\u63A8\u8350\u3001\u7528\u6237\u5386\u53F2\u5BF9\u8BDD\uFF09\u3002\u8981\u6309 `user_id` \u6216 `session_id` \u5212\u5206\u7F13\u5B58 namespace\uFF0C\u5426\u5219\u4E0D\u540C\u7528\u6237\u4F1A\u8BFB\u5230\u5BF9\u65B9\u7684\u56DE\u7B54\u3002

\u56DB\u3001\u53EF\u89C2\u6D4B\u6027\uFF1A\u4ECE"\u8C03\u7528\u65E5\u5FD7"\u5230"\u8C03\u7528\u753B\u50CF"

AI \u7F51\u5173\u628A"\u8C03\u7528\u4E00\u6B21 LLM"\u8FD9\u4EF6\u4E8B\u7684\u5168\u94FE\u8DEF\u5143\u6570\u636E\u6C89\u6DC0\u4E0B\u6765\u2014\u2014\u8FD9\u4E00\u6B65\u662F\u4ECE demo \u8D70\u5411 production \u7684\u5206\u6C34\u5CAD\u3002

4.1 LiteLLM + Langfuse / Helicone

LiteLLM Proxy \u652F\u6301\u628A\u6BCF\u6B21\u8BF7\u6C42/\u54CD\u5E94\u8F6C\u53D1\u5230 Langfuse\uFF0828,999 stars / 3,006 forks\uFF0C\u622A\u81F3 2026-06-13\uFF09\u3002\u5728 `config.yaml` \u91CC\u52A0\u51E0\u884C\uFF1A

```yaml litellm_settings: success_callback: ["langfuse"] failure_callback: ["langfuse"]

environment_variables: LANGFUSE_PUBLIC_KEY: pk-lf-*** LANGFUSE_SECRET_KEY: sk-lf-*** LANGFUSE_HOST: https://cloud.langfuse.com ```

\u542F\u7528\u540E Langfuse UI \u91CC\u4F1A\u51FA\u73B0\uFF1Atrace ID\u3001prompt\u3001completion\u3001token \u6570\u3001latency\u3001\u6210\u672C\u4F30\u7B97\u3001\u6309 tag/user/model \u805A\u5408\u7684 dashboard\u3002Helicone \u96C6\u6210\u65B9\u5F0F\u7C7B\u4F3C\uFF08\u4E00\u4E2A callback \u5B57\u7B26\u4E32\u5207\u6362\u5373\u53EF\uFF09\u3002

4.2 Cloudflare AI Gateway \u5185\u7F6E Analytics

Cloudflare \u7684\u4F18\u52BF\u662F**\u96F6\u914D\u7F6E\u5C31\u6709 dashboard**\u2014\u2014\u6BCF\u4E2A Gateway \u5728 CF \u63A7\u5236\u53F0\u81EA\u5E26\uFF1A

• \u8BF7\u6C42\u91CF\uFF08\u6309\u6A21\u578B/\u4F9B\u5E94\u5546/\u72B6\u6001\u7801\u5206\u7EC4\uFF09
• P50 / P95 / P99 latency
• Token \u6D88\u8017 + \u4F30\u7B97\u6210\u672C
• \u9519\u8BEF\u7387 + \u7F13\u5B58\u547D\u4E2D\u7387
• \u81EA\u5B9A\u4E49\u5B57\u6BB5\uFF08\u8BF7\u6C42 header \u900F\u4F20\uFF0C\u4FBF\u4E8E\u6309\u7528\u6237/feature flag \u5207\u7247\uFF09

\u5B9E\u6218\u7ECF\u9A8C\uFF1ACF AI Gateway \u7684"\u81EA\u5B9A\u4E49\u65E5\u5FD7\u5B57\u6BB5"\u662F\u6740\u624B\u950F\u2014\u2014\u5728\u8BF7\u6C42\u91CC\u52A0 `cf-aig-metadata: {"feature":"customer-support","tier":"pro"}`\uFF0Cdashboard \u5C31\u80FD\u6309 feature/tier \u5207\u5206\u6210\u672C\u3002\u8FD9\u5BF9"\u7B97 AI \u8D26"\u662F\u521A\u9700\u3002

4.3 OpenRouter \u7684 Routes / Apps \u6570\u636E

OpenRouter \u63D0\u4F9B\u516C\u5F00\u7684 Rankings \u9875\u9762\uFF0C\u7EDF\u8BA1\u6BCF\u4E2A\u6A21\u578B/\u5E94\u7528\u5546\u7684\u8C03\u7528\u91CF\u3002\u622A\u81F3 2026-06-13 \u6293\u53D6\u7684\u5B98\u65B9\u6570\u636E\uFF1A\u6708 token \u91CF 100T+\uFF0C\u5168\u7403\u7528\u6237 8M+\uFF0C\u8986\u76D6 60+ \u4F9B\u5E94\u5546 / 400+ \u6A21\u578B\u3002\u5BF9**\u9009\u578B\u51B3\u7B56**\uFF08"\u54EA\u4E2A\u6A21\u578B\u6027\u4EF7\u6BD4\u6700\u597D"\uFF09\u5F88\u6709\u53C2\u8003\u4EF7\u503C\u3002

\u4E94\u3001\u9650\u6D41\u3001\u964D\u7EA7\u3001Guardrails\uFF1A\u628A"\u4E0A\u6E38\u5931\u63A7"\u6321\u5728\u7F51\u5173\u5916

5.1 \u9650\u6D41\u4E0E\u914D\u989D

LiteLLM Proxy \u7684\u9650\u6D41\u9897\u7C92\u5EA6\u53EF\u5230"\u865A\u62DF key"\u2014\u2014\u7ED9\u6BCF\u4E2A\u56E2\u961F/\u9879\u76EE/\u5BA2\u6237\u5206\u914D\u72EC\u7ACB\u7684 API key\uFF0C\u5206\u522B\u8BBE\u7F6E RPM/TPM \u4E0A\u9650\uFF1A

```yaml general_settings: master_key: sk-litellm-master database_url: postgresql://...

litellm_settings: key_generation_settings: team_key_generation: true

\u901A\u8FC7 admin UI \u6216 API \u521B\u5EFA virtual key

curl -X POST 'http://localhost:4000/key/generate' \

-H 'Authorization: Bearer sk-litellm-master' \

-d '{"models":["gpt-4o"],"max_budget":100,"budget_duration":"30d","team_id":"team-marketing"}'

```

Cloudflare AI Gateway \u7684"Rate limiting"\u662F Beta \u9636\u6BB5\u7684\u80FD\u529B\uFF08\u5728 `/features` \u5217\u8868\u6807\u8BB0 Beta\uFF09\uFF0C\u901A\u8FC7 Workers \u811A\u672C\u7ED1\u5B9A\u81EA\u5B9A\u4E49\u89C4\u5219\u3002

5.2 Fallback \u4E0E Circuit Breaker

\u5F53\u4E0A\u6E38\u8FDE\u7EED\u5931\u8D25 N \u6B21\uFF0C\u7F51\u5173\u5E94\u8BE5\u81EA\u52A8\u5207\u65AD\u4E00\u6BB5\u65F6\u95F4\uFF08circuit open\uFF09\uFF0C\u907F\u514D\u96EA\u5D29\u3002LiteLLM \u7684 `Router` \u9ED8\u8BA4\u5E26\u8FD9\u4E2A\u80FD\u529B\uFF1A

```python router = Router( model_list=[...], fallbacks=[{"gpt4o": ["claude-sonnet-4.5"]}], num_retries=2, timeout=15, allowed_fails=3, # \u8FDE\u7EED\u5931\u8D25 3 \u6B21\u540E\u5207\u5230\u4E0B\u4E00\u4E2A cooldown_time=30, # \u51B7\u5374 30 \u79D2 ) ```

5.3 Guardrails\uFF1APrompt Injection \u68C0\u6D4B\u4E0E PII \u8FC7\u6EE4

Portkey \u5185\u7F6E 50+ Guardrails\uFF08\u622A\u81F3 2026-06-13 \u4ED3\u5E93 README \u63CF\u8FF0\uFF09\uFF0C\u8986\u76D6\uFF1A

• PII \u8FC7\u6EE4\uFF1A\u81EA\u52A8\u8BC6\u522B\u5E76\u8131\u654F\u90AE\u7BB1\u3001\u7535\u8BDD\u3001\u8EAB\u4EFD\u8BC1\u53F7
• Prompt Injection \u68C0\u6D4B\uFF1A\u8BC6\u522B"\u5FFD\u7565\u4E4B\u524D\u6240\u6709\u6307\u4EE4"\u7C7B\u653B\u51FB
• \u5185\u5BB9\u5B89\u5168\uFF1ANSFW\u3001\u66B4\u529B\u3001\u653F\u6CBB\u654F\u611F\u8FC7\u6EE4
• \u5173\u952E\u8BCD\u9ED1/\u767D\u540D\u5355
• JSON Schema \u6821\u9A8C\uFF1A\u5F3A\u5236 LLM \u8F93\u51FA\u7B26\u5408 schema

```json { "input_guardrails": [ { "id": "pii-redaction", "params": { "entities": ["email", "phone"] } }, { "id": "prompt-injection-detection", "params": { "threshold": 0.9 } } ], "output_guardrails": [ { "id": "json-schema", "params": { "schema": {...} } } ] } ```

Cloudflare AI Gateway \u628A Guardrails \u653E\u5728"Workers AI Binding"\u5C42\u2014\u2014\u901A\u8FC7 Cloudflare Workers \u811A\u672C\u63D2\u5165 DLP\uFF08\u6570\u636E\u4E22\u5931\u9632\u62A4\uFF09\u903B\u8F91\u3002Portkey \u7684\u4F18\u52BF\u662F**\u5F00\u7BB1\u5373\u7528 + 50+ \u5185\u7F6E\u89C4\u5219**\uFF0C\u81EA\u5B9A\u4E49 Workers \u66F4\u7075\u6D3B\u4F46\u9700\u8981\u5199\u4EE3\u7801\u3002

\u516D\u3001\u5B9E\u8DF5\u9009\u578B\u6307\u5357\uFF1A\u56DB\u79CD\u65B9\u6848\u7684\u771F\u5B9E\u5BF9\u6BD4

\u6CE8\uFF1AGitHub star \u6570\u4E3A 2026-06-13 \u901A\u8FC7 `api.github.com/repos/...` \u5B9E\u65F6\u62C9\u53D6\uFF1B\u6A21\u578B\u8986\u76D6\u6570\u5B57\u6E90\u81EA\u5404\u65B9\u6848\u5B98\u65B9 README/\u5B98\u7F51\uFF0C\u622A\u81F3 2026-06 \u4E2D\u3002

\u63A8\u8350\u7EC4\u5408

• \u521D\u521B\u516C\u53F8/MVP\uFF1AOpenRouter \u2192 \u5355 API key\u3001\u7EDF\u4E00\u8D26\u5355\u3001\u6708\u5EA6 100T token \u91CF\u8DB3\u4EE5\u8986\u76D6\u65E9\u671F\u9700\u6C42
• \u4E2D\u578B\u4EA7\u54C1/\u6709\u81EA\u6258\u7BA1\u80FD\u529B\uFF1ALiteLLM Proxy + Langfuse \u2192 \u5B8C\u5168\u53EF\u63A7\u3001callback \u751F\u6001\u4E30\u5BCC
• \u5DF2\u7528 Cloudflare \u5168\u5BB6\u6876\uFF1ACF AI Gateway \u2192 \u96F6\u8FD0\u7EF4\u3001\u5168\u7403\u8FB9\u7F18\u7F13\u5B58\u3001Workers \u96C6\u6210
• \u9AD8\u5B89\u5168/\u91D1\u878D/\u533B\u7597\uFF1ALiteLLM Proxy \u81EA\u6258\u7BA1 + Portkey Guardrails \u2192 \u6570\u636E\u4E0D\u51FA\u7F51\u3001\u5185\u7F6E PII \u9632\u62A4

\u4E03\u3001\u5751\u4E0E\u6559\u8BAD

\u8E29\u8FC7\u7684\u51E0\u4E2A\u771F\u5B9E\u5751\uFF0C\u6309\u91CD\u8981\u6027\u6392\u5E8F\uFF1A

1. \u4E0D\u8981\u5728 `config` JSON \u91CC\u786C\u7F16\u7801 API key\u2014\u2014\u7528\u73AF\u5883\u53D8\u91CF\u6216\u5BC6\u94A5\u7BA1\u7406\uFF08Vault / AWS Secrets Manager\uFF09\u3002LiteLLM Proxy \u7684 `config.yaml` \u91CC\u652F\u6301 `os.environ/OPENAI_API_KEY` \u8FD9\u79CD\u8BED\u6CD5\uFF0C\u662F\u60EF\u4F8B\u3002

2. \u7F13\u5B58 namespace \u5FC5\u987B\u5305\u542B `user_id` \u6216 `session_id`\u2014\u2014\u5426\u5219\u4F1A\u8DE8\u7528\u6237\u6C61\u67D3\uFF08pitfall \u89C1 LiteLLM cache docs\uFF09\u3002\u6700\u7B80\u5355\u7684\u5DE5\u7A0B\u505A\u6CD5\u662F\u628A user_id \u62FC\u8FDB\u7F13\u5B58 key\u3002

3. fallback \u94FE\u4E0D\u8981\u8D85\u8FC7 3 \u5C42\u2014\u20143 \u5C42\u4EE5\u4E0A\u8BF4\u660E\u4E0A\u6E38\u9009\u578B\u6709\u95EE\u9898\uFF0C\u5E94\u8BE5\u6536\u655B\u4F9B\u5E94\u5546\u3002\u590D\u6742 fallback \u94FE\u5728\u751F\u4EA7\u4E8B\u6545\u65F6\u6392\u67E5\u6781\u56F0\u96BE\u3002

4. Cloudflare AI Gateway \u7684 Rate Limiting \u8FD8\u5728 Beta\uFF08\u622A\u81F3 2026-06-13 \u5B98\u65B9\u6587\u6863\u6807\u8BB0\uFF09\u2014\u2014\u751F\u4EA7\u7528\u524D\u5173\u6CE8 changelog\u3002

5. Portkey \u7684 `config` JSON \u901A\u8FC7 API \u5B58\u4E91\u7AEF\u2014\u2014\u610F\u5473\u7740\u4F60\u7684\u8DEF\u7531\u7B56\u7565\u662F\u6258\u7BA1\u5728 Portkey \u5E73\u53F0\u7684\uFF1B\u5982\u679C\u9700\u8981"\u5B8C\u5168\u81EA\u6258\u7BA1\u7B56\u7565"\uFF0C\u5FC5\u987B\u7528 Portkey \u5F00\u6E90 Gateway \u81EA\u90E8\u7F72\u3002

6. OpenRouter \u4E0D\u662F"\u514D\u8D39\u66FF\u4EE3\u54C1"\u2014\u2014\u5B83\u662F\u805A\u5408\u5C42\uFF0C\u6700\u7EC8\u8FD8\u662F\u8C03\u7528\u4E0A\u6E38\u6A21\u578B\uFF0C\u5B9A\u4EF7\u968F\u4E0A\u6E38\u6CE2\u52A8\u3002\u628A\u5B83\u5F53"\u7EDF\u4E00\u63A5\u53E3"\u7528\uFF0C\u4E0D\u8981\u671F\u5F85"\u66F4\u4FBF\u5B9C"\u3002

7. \u4E0D\u8981\u628A prompt \u6A21\u677F\u585E\u8FDB config\u2014\u2014\u5F88\u591A\u56E2\u961F\u56FE\u7701\u4E8B\u628A\u5B8C\u6574 system prompt \u5199\u8FDB\u7F51\u5173 config JSON \u91CC\u3002\u95EE\u9898\u662F\uFF1Aprompt \u8FED\u4EE3\u9891\u7387\u8FDC\u9AD8\u4E8E\u8DEF\u7531\u7B56\u7565\uFF0C\u5199\u5728 config \u91CC\u4F1A\u8BA9 review \u4E0E diff \u53D8\u5F97\u56F0\u96BE\u3002\u6B63\u786E\u505A\u6CD5\u662F prompt \u6A21\u677F\u4F5C\u4E3A\u72EC\u7ACB\u8D44\u4EA7\uFF08Langfuse \u7684 prompt management \u6A21\u5757\u3001\u6216\u81EA\u5EFA\u7684 prompt registry\uFF09\uFF0C\u7F51\u5173 config \u53EA\u5F15\u7528 prompt ID \u4E0E\u53D8\u91CF\u3002

8. \u591A\u4F9B\u5E94\u5546\u5207\u6362\u65F6\u6210\u672C\u6A21\u578B\u4F1A\u53D8\u2014\u2014Anthropic \u7684 prompt cache \u547D\u4E2D\u6309 1.25 \u500D\u8BA1\u8D39\u3001\u5199\u5165\u6309 1.25 \u500D\u8BA1\u8D39\uFF1BOpenAI \u7684 prompt cache \u547D\u4E2D\u6309 0.5 \u500D\u3001\u5199\u5165\u6309 2 \u500D\u8BA1\u8D39\u3002\u5982\u679C\u751F\u4EA7\u4EE3\u7801\u5728\u4E24\u4E2A\u4F9B\u5E94\u5546\u4E4B\u95F4\u9891\u7E41\u5207\u6362\uFF0C\u8BB0\u5F97\u628A"cost calculation"\u903B\u8F91\u4E5F\u642C\u5230\u7F51\u5173\u5C42\u7EDF\u4E00\u7B97\uFF0C\u4E0D\u8981\u6BCF\u4E2A\u4E1A\u52A1\u8C03\u7528\u70B9\u81EA\u5DF1\u4E58\u7CFB\u6570\u3002

9. Worker / \u8FB9\u7F18\u7F13\u5B58\u5728 streaming \u573A\u666F\u4E0B\u4E0D\u53CB\u597D\u2014\u2014Cloudflare AI Gateway \u7684\u8FB9\u7F18\u7F13\u5B58\u5728 streaming \u54CD\u5E94\u573A\u666F\u4E0B\u4F1A\u7B49\u5F85\u5B8C\u6574 body \u624D\u8FD4\u56DE\uFF0C\u5BF9\u957F\u751F\u6210\u573A\u666F\u4F1A\u663E\u8457\u589E\u52A0\u9996 token \u5EF6\u8FDF\u3002\u8981\u4E48\u5173\u95ED streaming\uFF0C\u8981\u4E48\u63A5\u53D7\u8FD9\u4E2A\u5EF6\u8FDF\u4EE3\u4EF7\uFF0C\u6216\u8005\u628A"\u662F\u5426\u53EF\u7F13\u5B58"\u505A\u6210 prompt \u7EA7\u522B\u7684\u5C5E\u6027\u2014\u2014\u4F8B\u5982\u5206\u7C7B\u573A\u666F\u53EF\u7F13\u5B58\u3001\u5BF9\u8BDD\u751F\u6210\u4E0D\u53EF\u7F13\u5B58\u3002

\u603B\u7ED3

AI Gateway \u4E0D\u662F\u94F6\u5F39\uFF0C\u4F46\u5B83**\u89E3\u51B3\u4E86\u4E00\u7C7B\u88AB\u53CD\u590D\u8E29\u8FC7\u7684\u5DE5\u7A0B\u95EE\u9898**\uFF1A\u591A\u4F9B\u5E94\u5546\u7BA1\u7406\u3001\u6210\u672C\u53EF\u89C2\u6D4B\u3001\u6545\u969C\u964D\u7EA7\u3001\u7F13\u5B58\u52A0\u901F\u3001\u5B89\u5168\u9632\u62A4\u30022026 \u5E74\u8FD9\u4E2A\u8D5B\u9053\u5DF2\u7ECF\u6E05\u6670\u6536\u655B\u5230 4 \u7C7B\u73A9\u5BB6\u2014\u2014\u5F00\u6E90\u81EA\u6258\u7BA1\uFF08LiteLLM\uFF09\u3001\u73B0\u4EE3 SaaS\uFF08Portkey\uFF09\u3001\u8FB9\u7F18\u96C6\u6210\uFF08Cloudflare\uFF09\u3001\u805A\u5408\u5E73\u53F0\uFF08OpenRouter\uFF09\u3002

\u5BF9\u5DE5\u7A0B\u56E2\u961F\u7684**\u5B9E\u9645\u5EFA\u8BAE**\uFF1A

• \u7B2C\u4E00\u9636\u6BB5\uFF08< 10 \u4E07 token/\u5929\uFF09\uFF1A\u76F4\u63A5\u7528 OpenRouter\uFF0C\u96F6\u8FD0\u7EF4\u3002
• \u7B2C\u4E8C\u9636\u6BB5\uFF08> 100 \u4E07 token/\u5929\uFF0C> 5 \u4E2A\u8C03\u7528\u65B9\uFF09\uFF1A\u4E0A LiteLLM \u6216 Portkey\uFF0C\u628A\u65E5\u5FD7/\u914D\u989D/\u7F13\u5B58\u88C5\u8D77\u6765\u3002
• \u7B2C\u4E09\u9636\u6BB5\uFF08> 1 \u4EBF token/\u5929\uFF0C\u5168\u7403\u7528\u6237 / \u5408\u89C4\u8981\u6C42\uFF09\uFF1A\u8003\u8651 Cloudflare AI Gateway \u8FB9\u7F18\u7F13\u5B58 + \u81EA\u6258\u7BA1 LiteLLM \u53CC\u5C42\u67B6\u6784\u3002

\u4E0D\u8981\u7B49\u5230\u7B2C\u4E00\u6B21"\u4E0A\u6E38\u6302\u4E86\u5BFC\u81F4\u6574\u7AD9\u5D29\u6E83"\u624D\u8865\u8FD9\u5C42\u3002

\u53C2\u8003\u8D44\u6599

1. LiteLLM GitHub: https://github.com/BerriAI/litellm\uFF0850,208 stars\uFF0C2026-06-13 \u62C9\u53D6\uFF09
2. Portkey-AI/gateway GitHub: https://github.com/Portkey-AI/gateway\uFF0812,049 stars\uFF0C2026-06-13 \u62C9\u53D6\uFF09
3. Cloudflare AI Gateway \u5B98\u65B9\u6587\u6863: https://developers.cloudflare.com/ai-gateway/
4. OpenRouter \u5B98\u65B9: https://openrouter.ai/\uFF08"100T Monthly Tokens, 8M+ Global Users, 60+ Providers, 400+ Models"\uFF0C\u5B98\u7F51\u9996\u9875\u622A\u81F3 2026-06-13\uFF09
5. Langfuse GitHub: https://github.com/langfuse/langfuse\uFF0828,999 stars\uFF0C2026-06-13 \u62C9\u53D6\uFF09
6. vLLM GitHub: https://github.com/vllm-project/vllm\uFF0882,730 stars\uFF0C2026-06-13 \u62C9\u53D6\uFF09
7. SGLang GitHub: https://github.com/sgl-project/sglang\uFF0828,947 stars\uFF0C2026-06-13 \u62C9\u53D6\uFF09
8. LiteLLM Routing & Load Balancing: https://docs.litellm.ai/docs/routing-load-balancing