From 8dbcb1147fa60d0d1b24d10fd6bc6908cd6daf0d Mon Sep 17 00:00:00 2001 From: loveuer Date: Tue, 23 Jun 2026 09:59:24 +0800 Subject: [PATCH] v0.0.3: add built-in web tools Add no-key web_search and web_fetch tools. Fetch pages with net/http and parse HTML content. Document built-in web capabilities. --- README.md | 10 + cmd/agentu/main.go | 2 + go.mod | 2 +- internal/tools/tools.go | 13 +- internal/tools/tools_test.go | 30 ++ internal/tools/web_builtin.go | 597 +++++++++++++++++++++++++++++ internal/tools/web_builtin_test.go | 96 +++++ 7 files changed, 747 insertions(+), 3 deletions(-) create mode 100644 internal/tools/web_builtin.go create mode 100644 internal/tools/web_builtin_test.go diff --git a/README.md b/README.md index 8d8ff34..f7a24d1 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ model/provider switching, and local tools. - Multiple providers from `~/.agentu/config.yaml`. - Session-only provider, model, and thinking-level switching. - Read-only project tools enabled by default. +- Built-in web search and URL fetching. - `--yolo` mode for file writes and shell execution. ## Quick Start @@ -58,6 +59,10 @@ agent: tool_timeout: 30s ``` +Web tools are enabled by default with a built-in no-key backend. agentu exposes +`web_search` for current or external web information and `web_fetch` for reading +a known URL. + Provider names are the keys under `providers`. At startup, agentu selects the first provider by name. Use `/model provider ` to switch providers for the current session. @@ -112,6 +117,11 @@ Read-only tools are available by default: - `file_list` - `file_search` +Web tools are also available in read-only mode: + +- `web_search` +- `web_fetch` + `--yolo` additionally enables: - `file_write` diff --git a/cmd/agentu/main.go b/cmd/agentu/main.go index 1ed8bf7..62fe710 100644 --- a/cmd/agentu/main.go +++ b/cmd/agentu/main.go @@ -60,6 +60,8 @@ func run() error { } else { registry = tools.ReadOnlyBuiltins(cfg.Agent.WorkingDir) } + registry.Register(tools.NewBuiltinSearchTool(http.DefaultClient)) + registry.Register(tools.NewBuiltinFetchTool(http.DefaultClient)) systemPrompt := strings.TrimSpace(cfg.Agent.SystemPrompt + "\n\n" + tools.AgentInstructions(cfg.Agent.WorkingDir, *yolo)) assistant := agent.New(agent.Options{ diff --git a/go.mod b/go.mod index 46f6550..9a5afd9 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/glamour v1.0.0 github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 + golang.org/x/net v0.38.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -39,7 +40,6 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yuin/goldmark v1.7.13 // indirect github.com/yuin/goldmark-emoji v1.0.6 // indirect - golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/term v0.36.0 // indirect golang.org/x/text v0.30.0 // indirect diff --git a/internal/tools/tools.go b/internal/tools/tools.go index 4ae08e6..3879505 100644 --- a/internal/tools/tools.go +++ b/internal/tools/tools.go @@ -82,6 +82,9 @@ func Builtins(workingDir string) *Registry { } func AgentInstructions(workingDir string, yolo bool) string { + searchInstructions := webSearchInstructions() + readOnlyTools := "file_list, file_read, file_search, web_search, web_fetch" + yoloTools := "file_list, file_read, file_search, file_write, shell_run, web_search, web_fetch" if yolo { return fmt.Sprintf(`Local project context: - The project working directory is %q. @@ -92,8 +95,9 @@ func AgentInstructions(workingDir string, yolo bool) string { - When the user asks to run, execute, test, check, debug, or inspect a shell command, use shell_run. This includes local commands written in backticks, such as go, make, git, pwd, ls, ps, and similar terminal commands. - Do not use file tools as a substitute for a command execution request. - Do not start interactive or long-running sessions. Prefer bounded, non-interactive local commands with clear completion conditions. +- %s - Tool paths should normally be relative to the project working directory. -- Available tools: file_list, file_read, file_search, file_write, shell_run.`, workingDir) +- Available tools: %s.`, workingDir, searchInstructions, yoloTools) } return fmt.Sprintf(`Local project context: - The project working directory is %q. @@ -102,8 +106,13 @@ func AgentInstructions(workingDir string, yolo bool) string { - Do not explore the project preemptively. Do not call file_list just to discover context unless the user asks about project files or the answer truly depends on local files. - When the user asks to inspect, list, search, or read project files, use file_list, file_search, and file_read instead of saying you cannot access local files. - Shell command execution is disabled in this mode. When the user asks to run, execute, test, check, debug, or inspect a shell command, do not use file tools as a substitute; explain that they need to restart agentu with --yolo to enable shell_run. +- %s - Tool paths should normally be relative to the project working directory. -- Available tools: file_list, file_read, file_search.`, workingDir) +- Available tools: %s.`, workingDir, searchInstructions, readOnlyTools) +} + +func webSearchInstructions() string { + return "When the user's request depends on current or external web information, use web_search. This includes news, prices, laws, schedules, software versions, product recommendations, or requests for the latest information. Use web_fetch to read a source URL when search snippets are not enough, when the user provides a URL, or when the user names a specific website to inspect. Base answers on returned sources and include source URLs when useful." } func JSONSchema(schema string) json.RawMessage { diff --git a/internal/tools/tools_test.go b/internal/tools/tools_test.go index a3faf69..df8768b 100644 --- a/internal/tools/tools_test.go +++ b/internal/tools/tools_test.go @@ -84,6 +84,27 @@ func TestReadOnlyBuiltinsExposeOpenAICompatibleNamesAndAliases(t *testing.T) { } } +func TestRegistryCanExposeWebSearch(t *testing.T) { + registry := ReadOnlyBuiltins(t.TempDir()) + registry.Register(NewBuiltinSearchTool(nil)) + registry.Register(NewBuiltinFetchTool(nil)) + + var names []string + for _, def := range registry.Definitions() { + names = append(names, def.Function.Name) + } + for _, want := range []string{"web_search", "web_fetch"} { + if !slices.Contains(names, want) { + t.Fatalf("definitions missing %s: %#v", want, names) + } + } + for _, want := range []string{"web_search", "web_fetch"} { + if _, ok := registry.Get(want); !ok { + t.Fatalf("%s missing", want) + } + } +} + func TestBuiltinsExposeYoloTools(t *testing.T) { registry := Builtins(t.TempDir()) for _, name := range []string{"file_write", "shell_run", "file.write", "shell.run"} { @@ -108,3 +129,12 @@ func TestAgentInstructionsExplainCommandModes(t *testing.T) { } } } + +func TestAgentInstructionsExplainWebSearch(t *testing.T) { + withSearch := AgentInstructions("/tmp/project", false) + for _, want := range []string{"use web_search", "Use web_fetch", "latest information", "source URLs", "Available tools: file_list, file_read, file_search, web_search, web_fetch"} { + if !strings.Contains(withSearch, want) { + t.Fatalf("web search instructions missing %q:\n%s", want, withSearch) + } + } +} diff --git a/internal/tools/web_builtin.go b/internal/tools/web_builtin.go new file mode 100644 index 0000000..7c6c027 --- /dev/null +++ b/internal/tools/web_builtin.go @@ -0,0 +1,597 @@ +package tools + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "agentu/internal/llm" + + "golang.org/x/net/html" +) + +const ( + defaultSearchBaseURL = "https://html.duckduckgo.com/html/" + defaultUserAgent = "agentu/0.0.3" + maxSearchResults = 20 + maxWebReadBytes = 2 * 1024 * 1024 +) + +type BuiltinSearchTool struct { + baseURL string + maxResults int + client *http.Client +} + +func NewBuiltinSearchTool(client *http.Client) *BuiltinSearchTool { + return NewBuiltinSearchToolWithBaseURL(defaultSearchBaseURL, 5, client) +} + +func NewBuiltinSearchToolWithBaseURL(baseURL string, maxResults int, client *http.Client) *BuiltinSearchTool { + if strings.TrimSpace(baseURL) == "" { + baseURL = defaultSearchBaseURL + } + if maxResults <= 0 { + maxResults = 5 + } + if client == nil { + client = http.DefaultClient + } + return &BuiltinSearchTool{ + baseURL: baseURL, + maxResults: maxResults, + client: client, + } +} + +func (t *BuiltinSearchTool) Definition() llm.Tool { + return webSearchDefinition() +} + +func webSearchDefinition() llm.Tool { + return llm.Tool{ + Type: "function", + Function: llm.ToolFunction{ + Name: "web_search", + Description: "Search the public web for current or external information. Use for recent facts, news, prices, schedules, versions, policies, recommendations, or other questions that need online sources. Results include source URLs.", + Parameters: JSONSchema(`{ + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query."}, + "max_results": {"type": "integer", "description": "Optional result count from 1 to 20. Defaults to 5.", "minimum": 1, "maximum": 20}, + "topic": {"type": "string", "description": "Optional topic hint. Accepted for compatibility but the built-in backend may ignore it.", "enum": ["general", "news", "finance"]} + }, + "required": ["query"], + "additionalProperties": false +}`), + }, + } +} + +func (t *BuiltinSearchTool) Execute(ctx context.Context, raw json.RawMessage) (string, error) { + args, err := parseWebSearchArgs(raw, t.maxResults) + if err != nil { + return "", err + } + + endpoint, err := url.Parse(t.baseURL) + if err != nil { + return "", fmt.Errorf("parse search endpoint: %w", err) + } + query := endpoint.Query() + query.Set("q", args.Query) + endpoint.RawQuery = query.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) + if err != nil { + return "", fmt.Errorf("create search request: %w", err) + } + req.Header.Set("User-Agent", defaultUserAgent) + req.Header.Set("Accept", "text/html,application/xhtml+xml") + + resp, err := t.client.Do(req) + if err != nil { + return "", fmt.Errorf("search request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8*1024)) + return "", fmt.Errorf("web search failed: status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + results, err := parseSearchResults(resp.Body, args.MaxResults) + if err != nil { + return "", err + } + return FormatResult(formatWebSearchResults(args.Query, "", results)), nil +} + +type BuiltinFetchTool struct { + client *http.Client +} + +type webSearchArgs struct { + Query string `json:"query"` + MaxResults int `json:"max_results"` + Topic string `json:"topic"` +} + +type webSearchResult struct { + Title string `json:"title"` + URL string `json:"url"` + Content string `json:"content"` + Score float64 `json:"score"` +} + +type webFetchArgs struct { + URL string `json:"url"` + Query string `json:"query"` +} + +type webFetchResponse struct { + Results []webFetchResult `json:"results"` + FailedResults []webFetchFailed `json:"failed_results"` +} + +type webFetchResult struct { + URL string `json:"url"` + RawContent string `json:"raw_content"` +} + +type webFetchFailed struct { + URL string `json:"url"` + Error string `json:"error"` +} + +func NewBuiltinFetchTool(client *http.Client) *BuiltinFetchTool { + if client == nil { + client = http.DefaultClient + } + return &BuiltinFetchTool{client: client} +} + +func (t *BuiltinFetchTool) Definition() llm.Tool { + return webFetchDefinition() +} + +func webFetchDefinition() llm.Tool { + return llm.Tool{ + Type: "function", + Function: llm.ToolFunction{ + Name: "web_fetch", + Description: "Fetch and extract readable Markdown content from a public web URL. Use after web_search when snippets are not enough, or when the user gives a URL or website to inspect. Results include the source URL.", + Parameters: JSONSchema(`{ + "type": "object", + "properties": { + "url": {"type": "string", "description": "Public http or https URL to fetch. URLs without a scheme are treated as https URLs."}, + "query": {"type": "string", "description": "Optional extraction focus. Accepted for compatibility but the built-in backend may ignore it."} + }, + "required": ["url"], + "additionalProperties": false +}`), + }, + } +} + +func (t *BuiltinFetchTool) Execute(ctx context.Context, raw json.RawMessage) (string, error) { + args, err := parseWebFetchArgs(raw) + if err != nil { + return "", err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, args.URL, nil) + if err != nil { + return "", fmt.Errorf("create fetch request: %w", err) + } + req.Header.Set("User-Agent", defaultUserAgent) + req.Header.Set("Accept", "text/html,text/plain,application/xhtml+xml") + + resp, err := t.client.Do(req) + if err != nil { + return "", fmt.Errorf("fetch request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8*1024)) + return "", fmt.Errorf("web fetch failed: status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + data, err := io.ReadAll(io.LimitReader(resp.Body, maxWebReadBytes)) + if err != nil { + return "", fmt.Errorf("read response: %w", err) + } + contentType := strings.ToLower(resp.Header.Get("Content-Type")) + var content string + if strings.Contains(contentType, "html") || contentType == "" { + content, err = htmlToMarkdown(string(data), args.URL) + if err != nil { + return "", err + } + } else if strings.Contains(contentType, "text/plain") || strings.Contains(contentType, "markdown") { + content = strings.TrimSpace(string(data)) + } else { + return "", fmt.Errorf("unsupported content type: %s", contentType) + } + + result := webFetchResponse{ + Results: []webFetchResult{{ + URL: args.URL, + RawContent: content, + }}, + } + return FormatResult(formatWebFetchResults(args.URL, result)), nil +} + +func parseWebSearchArgs(raw json.RawMessage, defaultMaxResults int) (webSearchArgs, error) { + var args webSearchArgs + if err := json.Unmarshal(raw, &args); err != nil { + return args, fmt.Errorf("parse arguments: %w", err) + } + args.Query = strings.TrimSpace(args.Query) + if args.Query == "" { + return args, errors.New("query is required") + } + if args.MaxResults == 0 { + args.MaxResults = defaultMaxResults + } + if args.MaxResults < 1 || args.MaxResults > maxSearchResults { + return args, fmt.Errorf("max_results must be between 1 and %d", maxSearchResults) + } + args.Topic = strings.ToLower(strings.TrimSpace(args.Topic)) + if args.Topic == "" { + args.Topic = "general" + } + switch args.Topic { + case "general", "news", "finance": + default: + return args, fmt.Errorf("topic must be one of: general, news, finance") + } + return args, nil +} + +func parseWebFetchArgs(raw json.RawMessage) (webFetchArgs, error) { + var args webFetchArgs + if err := json.Unmarshal(raw, &args); err != nil { + return args, fmt.Errorf("parse arguments: %w", err) + } + args.URL = strings.TrimSpace(args.URL) + if args.URL == "" { + return args, errors.New("url is required") + } + if !strings.Contains(args.URL, "://") { + args.URL = "https://" + args.URL + } + parsed, err := url.Parse(args.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return args, errors.New("url must be a valid http or https URL") + } + args.Query = strings.TrimSpace(args.Query) + return args, nil +} + +func formatWebSearchResults(query string, answer string, results []webSearchResult) string { + if len(results) == 0 { + return fmt.Sprintf("No web search results for %q.", query) + } + + var b strings.Builder + fmt.Fprintf(&b, "Web search results for %q:\n", query) + if strings.TrimSpace(answer) != "" { + fmt.Fprintf(&b, "\nSummary: %s\n", strings.TrimSpace(answer)) + } + for i, result := range results { + title := strings.TrimSpace(result.Title) + if title == "" { + title = "Untitled" + } + fmt.Fprintf(&b, "\n%d. %s\n", i+1, title) + fmt.Fprintf(&b, " Source: %s\n", strings.TrimSpace(result.URL)) + if content := strings.TrimSpace(result.Content); content != "" { + fmt.Fprintf(&b, " Snippet: %s\n", content) + } + if result.Score > 0 { + fmt.Fprintf(&b, " Score: %.3f\n", result.Score) + } + } + return strings.TrimRight(b.String(), "\n") +} + +func formatWebFetchResults(inputURL string, response webFetchResponse) string { + var b strings.Builder + if len(response.Results) == 0 { + fmt.Fprintf(&b, "No readable web content extracted from %s.", inputURL) + } else { + fmt.Fprintf(&b, "Fetched web content:\n") + for i, result := range response.Results { + sourceURL := strings.TrimSpace(result.URL) + if sourceURL == "" { + sourceURL = inputURL + } + fmt.Fprintf(&b, "\n%d. Source: %s\n", i+1, sourceURL) + if content := strings.TrimSpace(result.RawContent); content != "" { + fmt.Fprintf(&b, "\n%s\n", content) + } + } + } + for _, failed := range response.FailedResults { + sourceURL := strings.TrimSpace(failed.URL) + if sourceURL == "" { + sourceURL = inputURL + } + if strings.TrimSpace(failed.Error) != "" { + fmt.Fprintf(&b, "\nFetch failed for %s: %s", sourceURL, strings.TrimSpace(failed.Error)) + } + } + return strings.TrimRight(b.String(), "\n") +} + +func parseSearchResults(r io.Reader, maxResults int) ([]webSearchResult, error) { + doc, err := html.Parse(r) + if err != nil { + return nil, fmt.Errorf("parse search results: %w", err) + } + + results := make([]webSearchResult, 0, maxResults) + var walk func(*html.Node) + walk = func(n *html.Node) { + if len(results) >= maxResults || n == nil { + return + } + if n.Type == html.ElementNode && n.Data == "div" && hasClass(n, "result") { + if result, ok := parseResultNode(n); ok { + results = append(results, result) + } + return + } + for child := n.FirstChild; child != nil; child = child.NextSibling { + walk(child) + } + } + walk(doc) + return results, nil +} + +func parseResultNode(n *html.Node) (webSearchResult, bool) { + var result webSearchResult + var walk func(*html.Node) + walk = func(node *html.Node) { + if node == nil { + return + } + if node.Type == html.ElementNode && node.Data == "a" { + if hasClass(node, "result__a") && result.URL == "" { + result.Title = cleanWebText(textContent(node)) + result.URL = decodeSearchURL(attr(node, "href")) + } + if hasClass(node, "result__snippet") && result.Content == "" { + result.Content = cleanWebText(textContent(node)) + } + } + if node.Type == html.ElementNode && (node.Data == "div" || node.Data == "span") && hasClass(node, "result__snippet") && result.Content == "" { + result.Content = cleanWebText(textContent(node)) + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + walk(child) + } + } + walk(n) + return result, result.Title != "" && result.URL != "" +} + +func decodeSearchURL(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + parsed, err := url.Parse(raw) + if err == nil { + if target := parsed.Query().Get("uddg"); target != "" { + if decoded, err := url.QueryUnescape(target); err == nil { + return decoded + } + return target + } + } + return raw +} + +func htmlToMarkdown(input string, sourceURL string) (string, error) { + doc, err := html.Parse(strings.NewReader(input)) + if err != nil { + return "", fmt.Errorf("parse html: %w", err) + } + + var b strings.Builder + var title string + var walk func(*html.Node, int) + walk = func(n *html.Node, listDepth int) { + if n == nil || skipHTMLNode(n) { + return + } + if n.Type == html.ElementNode && n.Data == "title" && title == "" { + title = cleanWebText(textContent(n)) + return + } + if n.Type == html.TextNode { + writeInlineText(&b, n.Data) + return + } + if n.Type != html.ElementNode { + for child := n.FirstChild; child != nil; child = child.NextSibling { + walk(child, listDepth) + } + return + } + + switch n.Data { + case "h1", "h2", "h3", "h4", "h5", "h6": + writeBlockBreak(&b) + level := int(n.Data[1] - '0') + b.WriteString(strings.Repeat("#", level)) + b.WriteString(" ") + writeChildren(&b, n, listDepth, walk) + writeBlockBreak(&b) + case "p", "div", "section", "article", "header", "footer", "main", "tr": + writeBlockBreak(&b) + writeChildren(&b, n, listDepth, walk) + writeBlockBreak(&b) + case "br": + b.WriteString("\n") + case "li": + writeBlockBreak(&b) + b.WriteString(strings.Repeat(" ", listDepth)) + b.WriteString("- ") + writeChildren(&b, n, listDepth+1, walk) + writeBlockBreak(&b) + case "a": + label := cleanWebText(textContent(n)) + href := strings.TrimSpace(attr(n, "href")) + if href != "" && label != "" { + href = resolveURL(sourceURL, href) + b.WriteString(label) + b.WriteString(" (") + b.WriteString(href) + b.WriteString(")") + return + } + writeChildren(&b, n, listDepth, walk) + default: + writeChildren(&b, n, listDepth, walk) + } + } + walk(doc, 0) + + content := normalizeMarkdownText(b.String()) + if title != "" && !strings.Contains(content, title) { + content = "# " + title + "\n\n" + content + } + if strings.TrimSpace(content) == "" { + return "", errors.New("no readable content found") + } + return content, nil +} + +func writeChildren(b *strings.Builder, n *html.Node, listDepth int, walk func(*html.Node, int)) { + for child := n.FirstChild; child != nil; child = child.NextSibling { + walk(child, listDepth) + } +} + +func writeInlineText(b *strings.Builder, text string) { + text = cleanWebText(text) + if text == "" { + return + } + if b.Len() > 0 { + last := b.String()[b.Len()-1] + if last != '\n' && last != ' ' { + b.WriteByte(' ') + } + } + b.WriteString(text) +} + +func writeBlockBreak(b *strings.Builder) { + if b.Len() == 0 { + return + } + text := b.String() + if strings.HasSuffix(text, "\n\n") { + return + } + if strings.HasSuffix(text, "\n") { + b.WriteByte('\n') + return + } + b.WriteString("\n\n") +} + +func skipHTMLNode(n *html.Node) bool { + if n.Type != html.ElementNode { + return false + } + switch n.Data { + case "script", "style", "noscript", "svg", "canvas", "template": + return true + default: + return false + } +} + +func textContent(n *html.Node) string { + var b strings.Builder + var walk func(*html.Node) + walk = func(node *html.Node) { + if node == nil || skipHTMLNode(node) { + return + } + if node.Type == html.TextNode { + b.WriteString(node.Data) + b.WriteByte(' ') + return + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + walk(child) + } + } + walk(n) + return cleanWebText(b.String()) +} + +func hasClass(n *html.Node, class string) bool { + for _, item := range strings.Fields(attr(n, "class")) { + if item == class { + return true + } + } + return false +} + +func attr(n *html.Node, name string) string { + for _, item := range n.Attr { + if item.Key == name { + return item.Val + } + } + return "" +} + +func resolveURL(baseURL, raw string) string { + ref, err := url.Parse(raw) + if err != nil { + return raw + } + base, err := url.Parse(baseURL) + if err != nil { + return raw + } + return base.ResolveReference(ref).String() +} + +func cleanWebText(text string) string { + return strings.Join(strings.Fields(text), " ") +} + +func normalizeMarkdownText(text string) string { + lines := strings.Split(text, "\n") + out := make([]string, 0, len(lines)) + blank := false + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + if !blank && len(out) > 0 { + out = append(out, "") + } + blank = true + continue + } + out = append(out, line) + blank = false + } + return strings.TrimSpace(strings.Join(out, "\n")) +} diff --git a/internal/tools/web_builtin_test.go b/internal/tools/web_builtin_test.go new file mode 100644 index 0000000..5deb826 --- /dev/null +++ b/internal/tools/web_builtin_test.go @@ -0,0 +1,96 @@ +package tools + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestBuiltinSearchTool(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Fatalf("method = %s", r.Method) + } + if got := r.URL.Query().Get("q"); got != "agentu" { + t.Fatalf("query = %q", got) + } + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(` + +
+ Agentu release + Agentu adds built-in web tools. +
+`)) + })) + defer server.Close() + + tool := NewBuiltinSearchToolWithBaseURL(server.URL, 5, server.Client()) + out, err := tool.Execute(context.Background(), json.RawMessage(`{"query":"agentu"}`)) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"Web search results", "Agentu release", "https://example.com/agentu", "Agentu adds built-in web tools"} { + if !strings.Contains(out, want) { + t.Fatalf("output missing %q:\n%s", want, out) + } + } +} + +func TestBuiltinFetchTool(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Fatalf("method = %s", r.Method) + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(` + + Agentu + +

Agentu

+

Built-in fetch reads HTML.

+ + +`)) + })) + defer server.Close() + + tool := NewBuiltinFetchTool(server.Client()) + out, err := tool.Execute(context.Background(), json.RawMessage(`{"url":"`+server.URL+`"}`)) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"Fetched web content", server.URL, "# Agentu", "Built-in fetch reads HTML", "- Search", "- Fetch"} { + if !strings.Contains(out, want) { + t.Fatalf("output missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "hidden") { + t.Fatalf("script content should be ignored:\n%s", out) + } +} + +func TestBuiltinFetchToolAcceptsURLsWithoutScheme(t *testing.T) { + args, err := parseWebFetchArgs(json.RawMessage(`{"url":"example.com/page"}`)) + if err != nil { + t.Fatal(err) + } + if args.URL != "https://example.com/page" { + t.Fatalf("url = %q", args.URL) + } +} + +func TestBuiltinFetchToolFormatsFailedResults(t *testing.T) { + response := webFetchResponse{ + FailedResults: []webFetchFailed{{URL: "https://example.com", Error: "blocked"}}, + } + out := formatWebFetchResults("https://example.com", response) + for _, want := range []string{"No readable web content", "Fetch failed", "blocked"} { + if !strings.Contains(out, want) { + t.Fatalf("output missing %q:\n%s", want, out) + } + } +}