package tools import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" "agentu/pkg/llm" "golang.org/x/net/html" ) const ( defaultSearchBaseURL = "https://html.duckduckgo.com/html/" defaultUserAgent = "agentu/0.0.3" maxSearchResults = 20 maxWebReadBytes = 2 * 1024 * 1024 ) type BuiltinSearchTool struct { baseURL string maxResults int client *http.Client } func NewBuiltinSearchTool(client *http.Client) *BuiltinSearchTool { return NewBuiltinSearchToolWithBaseURL(defaultSearchBaseURL, 5, client) } func NewBuiltinSearchToolWithBaseURL(baseURL string, maxResults int, client *http.Client) *BuiltinSearchTool { if strings.TrimSpace(baseURL) == "" { baseURL = defaultSearchBaseURL } if maxResults <= 0 { maxResults = 5 } if client == nil { client = http.DefaultClient } return &BuiltinSearchTool{ baseURL: baseURL, maxResults: maxResults, client: client, } } func (t *BuiltinSearchTool) Definition() llm.Tool { return webSearchDefinition() } func webSearchDefinition() llm.Tool { return llm.Tool{ Type: "function", Function: llm.ToolFunction{ Name: "web_search", Description: "Search the public web for current or external information. Use for recent facts, news, prices, schedules, versions, policies, recommendations, or other questions that need online sources. Results include source URLs.", Parameters: JSONSchema(`{ "type": "object", "properties": { "query": {"type": "string", "description": "Search query."}, "max_results": {"type": "integer", "description": "Optional result count from 1 to 20. Defaults to 5.", "minimum": 1, "maximum": 20}, "topic": {"type": "string", "description": "Optional topic hint. Accepted for compatibility but the built-in backend may ignore it.", "enum": ["general", "news", "finance"]} }, "required": ["query"], "additionalProperties": false }`), }, } } func (t *BuiltinSearchTool) Execute(ctx context.Context, raw json.RawMessage) (string, error) { args, err := parseWebSearchArgs(raw, t.maxResults) if err != nil { return "", err } endpoint, err := url.Parse(t.baseURL) if err != nil { return "", fmt.Errorf("parse search endpoint: %w", err) } query := endpoint.Query() query.Set("q", args.Query) endpoint.RawQuery = query.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) if err != nil { return "", fmt.Errorf("create search request: %w", err) } req.Header.Set("User-Agent", defaultUserAgent) req.Header.Set("Accept", "text/html,application/xhtml+xml") resp, err := t.client.Do(req) if err != nil { return "", fmt.Errorf("search request: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 8*1024)) return "", fmt.Errorf("web search failed: status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) } results, err := parseSearchResults(resp.Body, args.MaxResults) if err != nil { return "", err } return FormatResult(formatWebSearchResults(args.Query, "", results)), nil } type BuiltinFetchTool struct { client *http.Client } type webSearchArgs struct { Query string `json:"query"` MaxResults int `json:"max_results"` Topic string `json:"topic"` } type webSearchResult struct { Title string `json:"title"` URL string `json:"url"` Content string `json:"content"` Score float64 `json:"score"` } type webFetchArgs struct { URL string `json:"url"` Query string `json:"query"` } type webFetchResponse struct { Results []webFetchResult `json:"results"` FailedResults []webFetchFailed `json:"failed_results"` } type webFetchResult struct { URL string `json:"url"` RawContent string `json:"raw_content"` } type webFetchFailed struct { URL string `json:"url"` Error string `json:"error"` } func NewBuiltinFetchTool(client *http.Client) *BuiltinFetchTool { if client == nil { client = http.DefaultClient } return &BuiltinFetchTool{client: client} } func (t *BuiltinFetchTool) Definition() llm.Tool { return webFetchDefinition() } func webFetchDefinition() llm.Tool { return llm.Tool{ Type: "function", Function: llm.ToolFunction{ Name: "web_fetch", Description: "Fetch and extract readable Markdown content from a public web URL. Use after web_search when snippets are not enough, or when the user gives a URL or website to inspect. Results include the source URL.", Parameters: JSONSchema(`{ "type": "object", "properties": { "url": {"type": "string", "description": "Public http or https URL to fetch. URLs without a scheme are treated as https URLs."}, "query": {"type": "string", "description": "Optional extraction focus. Accepted for compatibility but the built-in backend may ignore it."} }, "required": ["url"], "additionalProperties": false }`), }, } } func (t *BuiltinFetchTool) Execute(ctx context.Context, raw json.RawMessage) (string, error) { args, err := parseWebFetchArgs(raw) if err != nil { return "", err } req, err := http.NewRequestWithContext(ctx, http.MethodGet, args.URL, nil) if err != nil { return "", fmt.Errorf("create fetch request: %w", err) } req.Header.Set("User-Agent", defaultUserAgent) req.Header.Set("Accept", "text/html,text/plain,application/xhtml+xml") resp, err := t.client.Do(req) if err != nil { return "", fmt.Errorf("fetch request: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 8*1024)) return "", fmt.Errorf("web fetch failed: status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) } data, err := io.ReadAll(io.LimitReader(resp.Body, maxWebReadBytes)) if err != nil { return "", fmt.Errorf("read response: %w", err) } contentType := strings.ToLower(resp.Header.Get("Content-Type")) var content string if strings.Contains(contentType, "html") || contentType == "" { content, err = htmlToMarkdown(string(data), args.URL) if err != nil { return "", err } } else if strings.Contains(contentType, "text/plain") || strings.Contains(contentType, "markdown") { content = strings.TrimSpace(string(data)) } else { return "", fmt.Errorf("unsupported content type: %s", contentType) } result := webFetchResponse{ Results: []webFetchResult{{ URL: args.URL, RawContent: content, }}, } return FormatResult(formatWebFetchResults(args.URL, result)), nil } func parseWebSearchArgs(raw json.RawMessage, defaultMaxResults int) (webSearchArgs, error) { var args webSearchArgs if err := json.Unmarshal(raw, &args); err != nil { return args, fmt.Errorf("parse arguments: %w", err) } args.Query = strings.TrimSpace(args.Query) if args.Query == "" { return args, errors.New("query is required") } if args.MaxResults == 0 { args.MaxResults = defaultMaxResults } if args.MaxResults < 1 || args.MaxResults > maxSearchResults { return args, fmt.Errorf("max_results must be between 1 and %d", maxSearchResults) } args.Topic = strings.ToLower(strings.TrimSpace(args.Topic)) if args.Topic == "" { args.Topic = "general" } switch args.Topic { case "general", "news", "finance": default: return args, fmt.Errorf("topic must be one of: general, news, finance") } return args, nil } func parseWebFetchArgs(raw json.RawMessage) (webFetchArgs, error) { var args webFetchArgs if err := json.Unmarshal(raw, &args); err != nil { return args, fmt.Errorf("parse arguments: %w", err) } args.URL = strings.TrimSpace(args.URL) if args.URL == "" { return args, errors.New("url is required") } if !strings.Contains(args.URL, "://") { args.URL = "https://" + args.URL } parsed, err := url.Parse(args.URL) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { return args, errors.New("url must be a valid http or https URL") } args.Query = strings.TrimSpace(args.Query) return args, nil } func formatWebSearchResults(query string, answer string, results []webSearchResult) string { if len(results) == 0 { return fmt.Sprintf("No web search results for %q.", query) } var b strings.Builder fmt.Fprintf(&b, "Web search results for %q:\n", query) if strings.TrimSpace(answer) != "" { fmt.Fprintf(&b, "\nSummary: %s\n", strings.TrimSpace(answer)) } for i, result := range results { title := strings.TrimSpace(result.Title) if title == "" { title = "Untitled" } fmt.Fprintf(&b, "\n%d. %s\n", i+1, title) fmt.Fprintf(&b, " Source: %s\n", strings.TrimSpace(result.URL)) if content := strings.TrimSpace(result.Content); content != "" { fmt.Fprintf(&b, " Snippet: %s\n", content) } if result.Score > 0 { fmt.Fprintf(&b, " Score: %.3f\n", result.Score) } } return strings.TrimRight(b.String(), "\n") } func formatWebFetchResults(inputURL string, response webFetchResponse) string { var b strings.Builder if len(response.Results) == 0 { fmt.Fprintf(&b, "No readable web content extracted from %s.", inputURL) } else { fmt.Fprintf(&b, "Fetched web content:\n") for i, result := range response.Results { sourceURL := strings.TrimSpace(result.URL) if sourceURL == "" { sourceURL = inputURL } fmt.Fprintf(&b, "\n%d. Source: %s\n", i+1, sourceURL) if content := strings.TrimSpace(result.RawContent); content != "" { fmt.Fprintf(&b, "\n%s\n", content) } } } for _, failed := range response.FailedResults { sourceURL := strings.TrimSpace(failed.URL) if sourceURL == "" { sourceURL = inputURL } if strings.TrimSpace(failed.Error) != "" { fmt.Fprintf(&b, "\nFetch failed for %s: %s", sourceURL, strings.TrimSpace(failed.Error)) } } return strings.TrimRight(b.String(), "\n") } func parseSearchResults(r io.Reader, maxResults int) ([]webSearchResult, error) { doc, err := html.Parse(r) if err != nil { return nil, fmt.Errorf("parse search results: %w", err) } results := make([]webSearchResult, 0, maxResults) var walk func(*html.Node) walk = func(n *html.Node) { if len(results) >= maxResults || n == nil { return } if n.Type == html.ElementNode && n.Data == "div" && hasClass(n, "result") { if result, ok := parseResultNode(n); ok { results = append(results, result) } return } for child := n.FirstChild; child != nil; child = child.NextSibling { walk(child) } } walk(doc) return results, nil } func parseResultNode(n *html.Node) (webSearchResult, bool) { var result webSearchResult var walk func(*html.Node) walk = func(node *html.Node) { if node == nil { return } if node.Type == html.ElementNode && node.Data == "a" { if hasClass(node, "result__a") && result.URL == "" { result.Title = cleanWebText(textContent(node)) result.URL = decodeSearchURL(attr(node, "href")) } if hasClass(node, "result__snippet") && result.Content == "" { result.Content = cleanWebText(textContent(node)) } } if node.Type == html.ElementNode && (node.Data == "div" || node.Data == "span") && hasClass(node, "result__snippet") && result.Content == "" { result.Content = cleanWebText(textContent(node)) } for child := node.FirstChild; child != nil; child = child.NextSibling { walk(child) } } walk(n) return result, result.Title != "" && result.URL != "" } func decodeSearchURL(raw string) string { raw = strings.TrimSpace(raw) if raw == "" { return "" } parsed, err := url.Parse(raw) if err == nil { if target := parsed.Query().Get("uddg"); target != "" { if decoded, err := url.QueryUnescape(target); err == nil { return decoded } return target } } return raw } func htmlToMarkdown(input string, sourceURL string) (string, error) { doc, err := html.Parse(strings.NewReader(input)) if err != nil { return "", fmt.Errorf("parse html: %w", err) } var b strings.Builder var title string var walk func(*html.Node, int) walk = func(n *html.Node, listDepth int) { if n == nil || skipHTMLNode(n) { return } if n.Type == html.ElementNode && n.Data == "title" && title == "" { title = cleanWebText(textContent(n)) return } if n.Type == html.TextNode { writeInlineText(&b, n.Data) return } if n.Type != html.ElementNode { for child := n.FirstChild; child != nil; child = child.NextSibling { walk(child, listDepth) } return } switch n.Data { case "h1", "h2", "h3", "h4", "h5", "h6": writeBlockBreak(&b) level := int(n.Data[1] - '0') b.WriteString(strings.Repeat("#", level)) b.WriteString(" ") writeChildren(&b, n, listDepth, walk) writeBlockBreak(&b) case "p", "div", "section", "article", "header", "footer", "main", "tr": writeBlockBreak(&b) writeChildren(&b, n, listDepth, walk) writeBlockBreak(&b) case "br": b.WriteString("\n") case "li": writeBlockBreak(&b) b.WriteString(strings.Repeat(" ", listDepth)) b.WriteString("- ") writeChildren(&b, n, listDepth+1, walk) writeBlockBreak(&b) case "a": label := cleanWebText(textContent(n)) href := strings.TrimSpace(attr(n, "href")) if href != "" && label != "" { href = resolveURL(sourceURL, href) b.WriteString(label) b.WriteString(" (") b.WriteString(href) b.WriteString(")") return } writeChildren(&b, n, listDepth, walk) default: writeChildren(&b, n, listDepth, walk) } } walk(doc, 0) content := normalizeMarkdownText(b.String()) if title != "" && !strings.Contains(content, title) { content = "# " + title + "\n\n" + content } if strings.TrimSpace(content) == "" { return "", errors.New("no readable content found") } return content, nil } func writeChildren(b *strings.Builder, n *html.Node, listDepth int, walk func(*html.Node, int)) { for child := n.FirstChild; child != nil; child = child.NextSibling { walk(child, listDepth) } } func writeInlineText(b *strings.Builder, text string) { text = cleanWebText(text) if text == "" { return } if b.Len() > 0 { last := b.String()[b.Len()-1] if last != '\n' && last != ' ' { b.WriteByte(' ') } } b.WriteString(text) } func writeBlockBreak(b *strings.Builder) { if b.Len() == 0 { return } text := b.String() if strings.HasSuffix(text, "\n\n") { return } if strings.HasSuffix(text, "\n") { b.WriteByte('\n') return } b.WriteString("\n\n") } func skipHTMLNode(n *html.Node) bool { if n.Type != html.ElementNode { return false } switch n.Data { case "script", "style", "noscript", "svg", "canvas", "template": return true default: return false } } func textContent(n *html.Node) string { var b strings.Builder var walk func(*html.Node) walk = func(node *html.Node) { if node == nil || skipHTMLNode(node) { return } if node.Type == html.TextNode { b.WriteString(node.Data) b.WriteByte(' ') return } for child := node.FirstChild; child != nil; child = child.NextSibling { walk(child) } } walk(n) return cleanWebText(b.String()) } func hasClass(n *html.Node, class string) bool { for _, item := range strings.Fields(attr(n, "class")) { if item == class { return true } } return false } func attr(n *html.Node, name string) string { for _, item := range n.Attr { if item.Key == name { return item.Val } } return "" } func resolveURL(baseURL, raw string) string { ref, err := url.Parse(raw) if err != nil { return raw } base, err := url.Parse(baseURL) if err != nil { return raw } return base.ResolveReference(ref).String() } func cleanWebText(text string) string { return strings.Join(strings.Fields(text), " ") } func normalizeMarkdownText(text string) string { lines := strings.Split(text, "\n") out := make([]string, 0, len(lines)) blank := false for _, line := range lines { line = strings.TrimSpace(line) if line == "" { if !blank && len(out) > 0 { out = append(out, "") } blank = true continue } out = append(out, line) blank = false } return strings.TrimSpace(strings.Join(out, "\n")) }