fix(stream): extract input tokens from message_start event
message_delta only contains output_tokens. Input tokens are in the message_start event under message.usage.input_tokens. This was causing input token counts to be near-zero for all streaming requests.
This commit is contained in:
@@ -278,11 +278,13 @@ func handleStream(c *gin.Context, upstream *UpstreamClient, san *Sanitizer, pool
|
||||
c.Writer.WriteString(line + "\n")
|
||||
flusher.Flush()
|
||||
|
||||
// Extract token usage from message_delta event
|
||||
if len(line) > 5 && line[:5] == "data:" {
|
||||
data := line[5:]
|
||||
if gjson.Get(data, "type").String() == "message_delta" {
|
||||
inputTokens = gjson.Get(data, "usage.input_tokens").Int()
|
||||
eventType := gjson.Get(data, "type").String()
|
||||
switch eventType {
|
||||
case "message_start":
|
||||
inputTokens = gjson.Get(data, "message.usage.input_tokens").Int()
|
||||
case "message_delta":
|
||||
outputTokens = gjson.Get(data, "usage.output_tokens").Int()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user