added streaming support to anthropic models

This commit is contained in:
leach 2025-08-19 00:59:46 -04:00
parent 735ae69dbd
commit b847ef8812
3 changed files with 169 additions and 10 deletions

View File

@ -51,15 +51,14 @@ impl ChatClient {
ChatClient::OpenAI(client) => {
client.chat_completion_stream(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort, stream_callback).await
}
ChatClient::Anthropic(_) => {
// Fallback to non-streaming for Anthropic
self.chat_completion(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort).await
ChatClient::Anthropic(client) => {
client.chat_completion_stream(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort, stream_callback).await
}
}
}
pub fn supports_streaming(&self) -> bool {
matches!(self, ChatClient::OpenAI(_))
matches!(self, ChatClient::OpenAI(_) | ChatClient::Anthropic(_))
}
pub fn supports_feature(&self, feature: &str) -> bool {
@ -229,6 +228,65 @@ struct AnthropicContent {
text: String,
}
// Anthropic streaming structures
#[derive(Deserialize, Debug)]
struct AnthropicStreamEvent {
#[serde(rename = "type")]
event_type: String,
#[serde(flatten)]
data: serde_json::Value,
}
#[derive(Deserialize, Debug)]
struct AnthropicMessageStart {
message: AnthropicMessage,
}
#[derive(Deserialize, Debug)]
struct AnthropicMessage {
id: String,
#[serde(rename = "type")]
message_type: String,
role: String,
content: Vec<AnthropicContentBlock>,
model: String,
stop_reason: Option<String>,
stop_sequence: Option<String>,
usage: AnthropicUsage,
}
#[derive(Deserialize, Debug)]
struct AnthropicContentBlock {
#[serde(rename = "type")]
content_type: String,
text: Option<String>,
}
#[derive(Deserialize, Debug)]
struct AnthropicUsage {
input_tokens: u32,
output_tokens: u32,
}
#[derive(Deserialize, Debug)]
struct AnthropicContentBlockStart {
index: u32,
content_block: AnthropicContentBlock,
}
#[derive(Deserialize, Debug)]
struct AnthropicContentBlockDelta {
index: u32,
delta: AnthropicDelta,
}
#[derive(Deserialize, Debug)]
struct AnthropicDelta {
#[serde(rename = "type")]
delta_type: String,
text: Option<String>,
}
impl OpenAIClient {
pub fn new(config: &Config) -> Result<Self> {
let api_key = env::var("OPENAI_API_KEY")
@ -859,15 +917,116 @@ impl AnthropicClient {
Ok(content.clone())
}
pub async fn chat_completion_stream(
&self,
model: &str,
messages: &[Message],
_enable_web_search: bool,
_enable_reasoning_summary: bool,
_reasoning_effort: &str,
stream_callback: StreamCallback,
) -> Result<String> {
let url = format!("{}/messages", self.base_url);
let (system_prompt, user_messages) = Self::convert_messages(messages);
let config = crate::config::Config::load().unwrap_or_default();
let mut payload = json!({
"model": model,
"max_tokens": config.limits.max_tokens_anthropic,
"messages": user_messages,
"stream": true
});
if let Some(system) = system_prompt {
payload["system"] = json!(system);
}
let response = self
.client
.post(&url)
.header("x-api-key", &self.api_key)
.header("Content-Type", "application/json")
.header("anthropic-version", &config.api.anthropic_version)
.json(&payload)
.send()
.await
.context("Failed to send request to Anthropic API")?;
if !response.status().is_success() {
let error_text = response.text().await.unwrap_or_else(|_| "Unknown error".to_string());
return Err(anyhow::anyhow!("Anthropic API error: {}", error_text));
}
let mut full_response = String::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("Failed to read chunk from Anthropic stream")?;
let chunk_str = std::str::from_utf8(&chunk)
.context("Failed to parse Anthropic chunk as UTF-8")?;
// Parse server-sent events for Anthropic
for line in chunk_str.lines() {
if line.starts_with("data: ") {
let data = &line[6..];
// Skip empty data lines and [DONE] markers
if data.trim().is_empty() || data == "[DONE]" {
continue;
}
match serde_json::from_str::<AnthropicStreamEvent>(data) {
Ok(event) => {
match event.event_type.as_str() {
"content_block_delta" => {
if let Ok(delta_event) = serde_json::from_value::<AnthropicContentBlockDelta>(event.data) {
if let Some(text) = delta_event.delta.text {
if !text.is_empty() {
full_response.push_str(&text);
stream_callback(&text).await;
}
}
}
}
"message_start" | "content_block_start" | "content_block_stop" | "message_delta" | "message_stop" => {
// Handle other event types if needed
}
_ => {
// Unknown event type, skip
}
}
}
Err(_) => {
// Skip malformed JSON chunks
continue;
}
}
} else if line.starts_with("event: ") {
// Event type line, can be ignored as we parse it from the data
continue;
}
}
}
if full_response.is_empty() {
return Err(anyhow::anyhow!("No content found in Anthropic stream response"));
}
Ok(full_response)
}
pub fn supports_feature(&self, feature: &str) -> bool {
match feature {
"web_search" | "reasoning_summary" => false,
"streaming" => true,
"web_search" | "reasoning_summary" | "reasoning_effort" => false,
_ => false,
}
}
pub fn supports_feature_for_model(&self, feature: &str, _model: &str) -> bool {
match feature {
"streaming" => true,
"web_search" | "reasoning_summary" | "reasoning_effort" => false,
_ => false,
}

View File

@ -36,10 +36,10 @@ pub fn get_supported_models() -> HashMap<Provider, Vec<&'static str>> {
models.insert(
Provider::Anthropic,
vec![
"claude-3-5-sonnet-20241022",
"claude-opus-4-1-20250805",
"claude-sonnet-4-20250514",
"claude-3-7-sonnet-20250219",
"claude-3-5-haiku-20241022",
"claude-3-opus-20240229",
"claude-3-sonnet-20240229",
"claude-3-haiku-20240307",
],
);

View File

@ -210,8 +210,8 @@ Environment Variables:
Supported Models:
OpenAI: gpt-4.1, gpt-4.1-mini, gpt-4o, gpt-5, gpt-5-chat-latest, o1, o3, o4-mini, o3-mini
Anthropic: claude-3-5-sonnet-20241022, claude-3-5-haiku-20241022,
claude-3-opus-20240229, claude-3-sonnet-20240229,
Anthropic: claude-opus-4-1-20250805, claude-sonnet-4-20250514,
claude-3-7-sonnet-20250219, claude-3-5-haiku-20241022,
claude-3-haiku-20240307
"#;
println!("{}", style(help_text).dim());