added streaming support to anthropic models
This commit is contained in:
parent
735ae69dbd
commit
b847ef8812
|
|
@ -51,15 +51,14 @@ impl ChatClient {
|
|||
ChatClient::OpenAI(client) => {
|
||||
client.chat_completion_stream(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort, stream_callback).await
|
||||
}
|
||||
ChatClient::Anthropic(_) => {
|
||||
// Fallback to non-streaming for Anthropic
|
||||
self.chat_completion(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort).await
|
||||
ChatClient::Anthropic(client) => {
|
||||
client.chat_completion_stream(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort, stream_callback).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn supports_streaming(&self) -> bool {
|
||||
matches!(self, ChatClient::OpenAI(_))
|
||||
matches!(self, ChatClient::OpenAI(_) | ChatClient::Anthropic(_))
|
||||
}
|
||||
|
||||
pub fn supports_feature(&self, feature: &str) -> bool {
|
||||
|
|
@ -229,6 +228,65 @@ struct AnthropicContent {
|
|||
text: String,
|
||||
}
|
||||
|
||||
// Anthropic streaming structures
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicStreamEvent {
|
||||
#[serde(rename = "type")]
|
||||
event_type: String,
|
||||
#[serde(flatten)]
|
||||
data: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicMessageStart {
|
||||
message: AnthropicMessage,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicMessage {
|
||||
id: String,
|
||||
#[serde(rename = "type")]
|
||||
message_type: String,
|
||||
role: String,
|
||||
content: Vec<AnthropicContentBlock>,
|
||||
model: String,
|
||||
stop_reason: Option<String>,
|
||||
stop_sequence: Option<String>,
|
||||
usage: AnthropicUsage,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicContentBlock {
|
||||
#[serde(rename = "type")]
|
||||
content_type: String,
|
||||
text: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicUsage {
|
||||
input_tokens: u32,
|
||||
output_tokens: u32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicContentBlockStart {
|
||||
index: u32,
|
||||
content_block: AnthropicContentBlock,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicContentBlockDelta {
|
||||
index: u32,
|
||||
delta: AnthropicDelta,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct AnthropicDelta {
|
||||
#[serde(rename = "type")]
|
||||
delta_type: String,
|
||||
text: Option<String>,
|
||||
}
|
||||
|
||||
impl OpenAIClient {
|
||||
pub fn new(config: &Config) -> Result<Self> {
|
||||
let api_key = env::var("OPENAI_API_KEY")
|
||||
|
|
@ -859,15 +917,116 @@ impl AnthropicClient {
|
|||
Ok(content.clone())
|
||||
}
|
||||
|
||||
pub async fn chat_completion_stream(
|
||||
&self,
|
||||
model: &str,
|
||||
messages: &[Message],
|
||||
_enable_web_search: bool,
|
||||
_enable_reasoning_summary: bool,
|
||||
_reasoning_effort: &str,
|
||||
stream_callback: StreamCallback,
|
||||
) -> Result<String> {
|
||||
let url = format!("{}/messages", self.base_url);
|
||||
|
||||
let (system_prompt, user_messages) = Self::convert_messages(messages);
|
||||
|
||||
let config = crate::config::Config::load().unwrap_or_default();
|
||||
let mut payload = json!({
|
||||
"model": model,
|
||||
"max_tokens": config.limits.max_tokens_anthropic,
|
||||
"messages": user_messages,
|
||||
"stream": true
|
||||
});
|
||||
|
||||
if let Some(system) = system_prompt {
|
||||
payload["system"] = json!(system);
|
||||
}
|
||||
|
||||
let response = self
|
||||
.client
|
||||
.post(&url)
|
||||
.header("x-api-key", &self.api_key)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("anthropic-version", &config.api.anthropic_version)
|
||||
.json(&payload)
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to send request to Anthropic API")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let error_text = response.text().await.unwrap_or_else(|_| "Unknown error".to_string());
|
||||
return Err(anyhow::anyhow!("Anthropic API error: {}", error_text));
|
||||
}
|
||||
|
||||
let mut full_response = String::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("Failed to read chunk from Anthropic stream")?;
|
||||
let chunk_str = std::str::from_utf8(&chunk)
|
||||
.context("Failed to parse Anthropic chunk as UTF-8")?;
|
||||
|
||||
// Parse server-sent events for Anthropic
|
||||
for line in chunk_str.lines() {
|
||||
if line.starts_with("data: ") {
|
||||
let data = &line[6..];
|
||||
|
||||
// Skip empty data lines and [DONE] markers
|
||||
if data.trim().is_empty() || data == "[DONE]" {
|
||||
continue;
|
||||
}
|
||||
|
||||
match serde_json::from_str::<AnthropicStreamEvent>(data) {
|
||||
Ok(event) => {
|
||||
match event.event_type.as_str() {
|
||||
"content_block_delta" => {
|
||||
if let Ok(delta_event) = serde_json::from_value::<AnthropicContentBlockDelta>(event.data) {
|
||||
if let Some(text) = delta_event.delta.text {
|
||||
if !text.is_empty() {
|
||||
full_response.push_str(&text);
|
||||
stream_callback(&text).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"message_start" | "content_block_start" | "content_block_stop" | "message_delta" | "message_stop" => {
|
||||
// Handle other event types if needed
|
||||
}
|
||||
_ => {
|
||||
// Unknown event type, skip
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// Skip malformed JSON chunks
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else if line.starts_with("event: ") {
|
||||
// Event type line, can be ignored as we parse it from the data
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if full_response.is_empty() {
|
||||
return Err(anyhow::anyhow!("No content found in Anthropic stream response"));
|
||||
}
|
||||
|
||||
Ok(full_response)
|
||||
}
|
||||
|
||||
pub fn supports_feature(&self, feature: &str) -> bool {
|
||||
match feature {
|
||||
"web_search" | "reasoning_summary" => false,
|
||||
"streaming" => true,
|
||||
"web_search" | "reasoning_summary" | "reasoning_effort" => false,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn supports_feature_for_model(&self, feature: &str, _model: &str) -> bool {
|
||||
match feature {
|
||||
"streaming" => true,
|
||||
"web_search" | "reasoning_summary" | "reasoning_effort" => false,
|
||||
_ => false,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,10 +36,10 @@ pub fn get_supported_models() -> HashMap<Provider, Vec<&'static str>> {
|
|||
models.insert(
|
||||
Provider::Anthropic,
|
||||
vec![
|
||||
"claude-3-5-sonnet-20241022",
|
||||
"claude-opus-4-1-20250805",
|
||||
"claude-sonnet-4-20250514",
|
||||
"claude-3-7-sonnet-20250219",
|
||||
"claude-3-5-haiku-20241022",
|
||||
"claude-3-opus-20240229",
|
||||
"claude-3-sonnet-20240229",
|
||||
"claude-3-haiku-20240307",
|
||||
],
|
||||
);
|
||||
|
|
|
|||
|
|
@ -210,8 +210,8 @@ Environment Variables:
|
|||
|
||||
Supported Models:
|
||||
OpenAI: gpt-4.1, gpt-4.1-mini, gpt-4o, gpt-5, gpt-5-chat-latest, o1, o3, o4-mini, o3-mini
|
||||
Anthropic: claude-3-5-sonnet-20241022, claude-3-5-haiku-20241022,
|
||||
claude-3-opus-20240229, claude-3-sonnet-20240229,
|
||||
Anthropic: claude-opus-4-1-20250805, claude-sonnet-4-20250514,
|
||||
claude-3-7-sonnet-20250219, claude-3-5-haiku-20241022,
|
||||
claude-3-haiku-20240307
|
||||
"#;
|
||||
println!("{}", style(help_text).dim());
|
||||
|
|
|
|||
Loading…
Reference in New Issue