added streaming support to anthropic models
This commit is contained in:
parent
735ae69dbd
commit
b847ef8812
|
|
@ -51,15 +51,14 @@ impl ChatClient {
|
||||||
ChatClient::OpenAI(client) => {
|
ChatClient::OpenAI(client) => {
|
||||||
client.chat_completion_stream(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort, stream_callback).await
|
client.chat_completion_stream(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort, stream_callback).await
|
||||||
}
|
}
|
||||||
ChatClient::Anthropic(_) => {
|
ChatClient::Anthropic(client) => {
|
||||||
// Fallback to non-streaming for Anthropic
|
client.chat_completion_stream(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort, stream_callback).await
|
||||||
self.chat_completion(model, messages, enable_web_search, enable_reasoning_summary, reasoning_effort).await
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn supports_streaming(&self) -> bool {
|
pub fn supports_streaming(&self) -> bool {
|
||||||
matches!(self, ChatClient::OpenAI(_))
|
matches!(self, ChatClient::OpenAI(_) | ChatClient::Anthropic(_))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn supports_feature(&self, feature: &str) -> bool {
|
pub fn supports_feature(&self, feature: &str) -> bool {
|
||||||
|
|
@ -229,6 +228,65 @@ struct AnthropicContent {
|
||||||
text: String,
|
text: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Anthropic streaming structures
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicStreamEvent {
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
event_type: String,
|
||||||
|
#[serde(flatten)]
|
||||||
|
data: serde_json::Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicMessageStart {
|
||||||
|
message: AnthropicMessage,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicMessage {
|
||||||
|
id: String,
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
message_type: String,
|
||||||
|
role: String,
|
||||||
|
content: Vec<AnthropicContentBlock>,
|
||||||
|
model: String,
|
||||||
|
stop_reason: Option<String>,
|
||||||
|
stop_sequence: Option<String>,
|
||||||
|
usage: AnthropicUsage,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicContentBlock {
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
content_type: String,
|
||||||
|
text: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicUsage {
|
||||||
|
input_tokens: u32,
|
||||||
|
output_tokens: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicContentBlockStart {
|
||||||
|
index: u32,
|
||||||
|
content_block: AnthropicContentBlock,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicContentBlockDelta {
|
||||||
|
index: u32,
|
||||||
|
delta: AnthropicDelta,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct AnthropicDelta {
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
delta_type: String,
|
||||||
|
text: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
impl OpenAIClient {
|
impl OpenAIClient {
|
||||||
pub fn new(config: &Config) -> Result<Self> {
|
pub fn new(config: &Config) -> Result<Self> {
|
||||||
let api_key = env::var("OPENAI_API_KEY")
|
let api_key = env::var("OPENAI_API_KEY")
|
||||||
|
|
@ -859,15 +917,116 @@ impl AnthropicClient {
|
||||||
Ok(content.clone())
|
Ok(content.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn chat_completion_stream(
|
||||||
|
&self,
|
||||||
|
model: &str,
|
||||||
|
messages: &[Message],
|
||||||
|
_enable_web_search: bool,
|
||||||
|
_enable_reasoning_summary: bool,
|
||||||
|
_reasoning_effort: &str,
|
||||||
|
stream_callback: StreamCallback,
|
||||||
|
) -> Result<String> {
|
||||||
|
let url = format!("{}/messages", self.base_url);
|
||||||
|
|
||||||
|
let (system_prompt, user_messages) = Self::convert_messages(messages);
|
||||||
|
|
||||||
|
let config = crate::config::Config::load().unwrap_or_default();
|
||||||
|
let mut payload = json!({
|
||||||
|
"model": model,
|
||||||
|
"max_tokens": config.limits.max_tokens_anthropic,
|
||||||
|
"messages": user_messages,
|
||||||
|
"stream": true
|
||||||
|
});
|
||||||
|
|
||||||
|
if let Some(system) = system_prompt {
|
||||||
|
payload["system"] = json!(system);
|
||||||
|
}
|
||||||
|
|
||||||
|
let response = self
|
||||||
|
.client
|
||||||
|
.post(&url)
|
||||||
|
.header("x-api-key", &self.api_key)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.header("anthropic-version", &config.api.anthropic_version)
|
||||||
|
.json(&payload)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.context("Failed to send request to Anthropic API")?;
|
||||||
|
|
||||||
|
if !response.status().is_success() {
|
||||||
|
let error_text = response.text().await.unwrap_or_else(|_| "Unknown error".to_string());
|
||||||
|
return Err(anyhow::anyhow!("Anthropic API error: {}", error_text));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut full_response = String::new();
|
||||||
|
let mut stream = response.bytes_stream();
|
||||||
|
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
let chunk = chunk.context("Failed to read chunk from Anthropic stream")?;
|
||||||
|
let chunk_str = std::str::from_utf8(&chunk)
|
||||||
|
.context("Failed to parse Anthropic chunk as UTF-8")?;
|
||||||
|
|
||||||
|
// Parse server-sent events for Anthropic
|
||||||
|
for line in chunk_str.lines() {
|
||||||
|
if line.starts_with("data: ") {
|
||||||
|
let data = &line[6..];
|
||||||
|
|
||||||
|
// Skip empty data lines and [DONE] markers
|
||||||
|
if data.trim().is_empty() || data == "[DONE]" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
match serde_json::from_str::<AnthropicStreamEvent>(data) {
|
||||||
|
Ok(event) => {
|
||||||
|
match event.event_type.as_str() {
|
||||||
|
"content_block_delta" => {
|
||||||
|
if let Ok(delta_event) = serde_json::from_value::<AnthropicContentBlockDelta>(event.data) {
|
||||||
|
if let Some(text) = delta_event.delta.text {
|
||||||
|
if !text.is_empty() {
|
||||||
|
full_response.push_str(&text);
|
||||||
|
stream_callback(&text).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"message_start" | "content_block_start" | "content_block_stop" | "message_delta" | "message_stop" => {
|
||||||
|
// Handle other event types if needed
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
// Unknown event type, skip
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// Skip malformed JSON chunks
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if line.starts_with("event: ") {
|
||||||
|
// Event type line, can be ignored as we parse it from the data
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if full_response.is_empty() {
|
||||||
|
return Err(anyhow::anyhow!("No content found in Anthropic stream response"));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(full_response)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn supports_feature(&self, feature: &str) -> bool {
|
pub fn supports_feature(&self, feature: &str) -> bool {
|
||||||
match feature {
|
match feature {
|
||||||
"web_search" | "reasoning_summary" => false,
|
"streaming" => true,
|
||||||
|
"web_search" | "reasoning_summary" | "reasoning_effort" => false,
|
||||||
_ => false,
|
_ => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn supports_feature_for_model(&self, feature: &str, _model: &str) -> bool {
|
pub fn supports_feature_for_model(&self, feature: &str, _model: &str) -> bool {
|
||||||
match feature {
|
match feature {
|
||||||
|
"streaming" => true,
|
||||||
"web_search" | "reasoning_summary" | "reasoning_effort" => false,
|
"web_search" | "reasoning_summary" | "reasoning_effort" => false,
|
||||||
_ => false,
|
_ => false,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -36,10 +36,10 @@ pub fn get_supported_models() -> HashMap<Provider, Vec<&'static str>> {
|
||||||
models.insert(
|
models.insert(
|
||||||
Provider::Anthropic,
|
Provider::Anthropic,
|
||||||
vec![
|
vec![
|
||||||
"claude-3-5-sonnet-20241022",
|
"claude-opus-4-1-20250805",
|
||||||
|
"claude-sonnet-4-20250514",
|
||||||
|
"claude-3-7-sonnet-20250219",
|
||||||
"claude-3-5-haiku-20241022",
|
"claude-3-5-haiku-20241022",
|
||||||
"claude-3-opus-20240229",
|
|
||||||
"claude-3-sonnet-20240229",
|
|
||||||
"claude-3-haiku-20240307",
|
"claude-3-haiku-20240307",
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -210,8 +210,8 @@ Environment Variables:
|
||||||
|
|
||||||
Supported Models:
|
Supported Models:
|
||||||
OpenAI: gpt-4.1, gpt-4.1-mini, gpt-4o, gpt-5, gpt-5-chat-latest, o1, o3, o4-mini, o3-mini
|
OpenAI: gpt-4.1, gpt-4.1-mini, gpt-4o, gpt-5, gpt-5-chat-latest, o1, o3, o4-mini, o3-mini
|
||||||
Anthropic: claude-3-5-sonnet-20241022, claude-3-5-haiku-20241022,
|
Anthropic: claude-opus-4-1-20250805, claude-sonnet-4-20250514,
|
||||||
claude-3-opus-20240229, claude-3-sonnet-20240229,
|
claude-3-7-sonnet-20250219, claude-3-5-haiku-20241022,
|
||||||
claude-3-haiku-20240307
|
claude-3-haiku-20240307
|
||||||
"#;
|
"#;
|
||||||
println!("{}", style(help_text).dim());
|
println!("{}", style(help_text).dim());
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue