v0.0.5: context compaction, minimal config, retry/search/code symbols/diff/status/test commands

internal/llm/token_estimate_test.go

@@ -0,0 +1,45 @@
+package llm
+
+import "testing"
+
+func TestEstimateTokensEmpty(t *testing.T) {
+	if got := EstimateTokens(nil, nil); got != 0 {
+		t.Fatalf("EstimateTokens(nil, nil) = %d, want 0", got)
+	}
+}
+
+func TestEstimateTokens(t *testing.T) {
+	shortMessages := []Message{{Role: RoleUser, Content: "hello"}}
+	short := EstimateTokens(shortMessages, nil)
+	if short <= 4 {
+		t.Fatalf("short estimate = %d, want message framing plus content", short)
+	}
+	if short > 50 {
+		t.Fatalf("short estimate = %d, looks too high for one short message", short)
+	}
+
+	longMessages := []Message{
+		{Role: RoleUser, Content: "hello"},
+		{Role: RoleAssistant, Content: "This is a longer response with enough content to exceed the short estimate."},
+	}
+	long := EstimateTokens(longMessages, nil)
+	if long <= short {
+		t.Fatalf("long estimate = %d, want > short estimate %d", long, short)
+	}
+
+	withTools := EstimateTokens(shortMessages, []Tool{{
+		Type: "function",
+		Function: ToolFunction{
+			Name:        "file_read",
+			Description: "Read a file",
+			Parameters:  []byte(`{"type":"object"}`),
+		},
+	}})
+	if withTools <= short {
+		t.Fatalf("withTools estimate = %d, want > short estimate %d", withTools, short)
+	}
+
+	if msgTokens := estimateMessageTokens(shortMessages[0]); msgTokens <= 4 {
+		t.Fatalf("estimateMessageTokens = %d, want message framing plus content", msgTokens)
+	}
+}