loveuer
8dbcb1147f
Add no-key web_search and web_fetch tools. Fetch pages with net/http and parse HTML content. Document built-in web capabilities.
598 lines
16 KiB
Go
598 lines
16 KiB
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"agentu/internal/llm"
|
|
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
const (
|
|
defaultSearchBaseURL = "https://html.duckduckgo.com/html/"
|
|
defaultUserAgent = "agentu/0.0.3"
|
|
maxSearchResults = 20
|
|
maxWebReadBytes = 2 * 1024 * 1024
|
|
)
|
|
|
|
type BuiltinSearchTool struct {
|
|
baseURL string
|
|
maxResults int
|
|
client *http.Client
|
|
}
|
|
|
|
func NewBuiltinSearchTool(client *http.Client) *BuiltinSearchTool {
|
|
return NewBuiltinSearchToolWithBaseURL(defaultSearchBaseURL, 5, client)
|
|
}
|
|
|
|
func NewBuiltinSearchToolWithBaseURL(baseURL string, maxResults int, client *http.Client) *BuiltinSearchTool {
|
|
if strings.TrimSpace(baseURL) == "" {
|
|
baseURL = defaultSearchBaseURL
|
|
}
|
|
if maxResults <= 0 {
|
|
maxResults = 5
|
|
}
|
|
if client == nil {
|
|
client = http.DefaultClient
|
|
}
|
|
return &BuiltinSearchTool{
|
|
baseURL: baseURL,
|
|
maxResults: maxResults,
|
|
client: client,
|
|
}
|
|
}
|
|
|
|
func (t *BuiltinSearchTool) Definition() llm.Tool {
|
|
return webSearchDefinition()
|
|
}
|
|
|
|
func webSearchDefinition() llm.Tool {
|
|
return llm.Tool{
|
|
Type: "function",
|
|
Function: llm.ToolFunction{
|
|
Name: "web_search",
|
|
Description: "Search the public web for current or external information. Use for recent facts, news, prices, schedules, versions, policies, recommendations, or other questions that need online sources. Results include source URLs.",
|
|
Parameters: JSONSchema(`{
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {"type": "string", "description": "Search query."},
|
|
"max_results": {"type": "integer", "description": "Optional result count from 1 to 20. Defaults to 5.", "minimum": 1, "maximum": 20},
|
|
"topic": {"type": "string", "description": "Optional topic hint. Accepted for compatibility but the built-in backend may ignore it.", "enum": ["general", "news", "finance"]}
|
|
},
|
|
"required": ["query"],
|
|
"additionalProperties": false
|
|
}`),
|
|
},
|
|
}
|
|
}
|
|
|
|
func (t *BuiltinSearchTool) Execute(ctx context.Context, raw json.RawMessage) (string, error) {
|
|
args, err := parseWebSearchArgs(raw, t.maxResults)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
endpoint, err := url.Parse(t.baseURL)
|
|
if err != nil {
|
|
return "", fmt.Errorf("parse search endpoint: %w", err)
|
|
}
|
|
query := endpoint.Query()
|
|
query.Set("q", args.Query)
|
|
endpoint.RawQuery = query.Encode()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
|
|
if err != nil {
|
|
return "", fmt.Errorf("create search request: %w", err)
|
|
}
|
|
req.Header.Set("User-Agent", defaultUserAgent)
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
|
|
|
resp, err := t.client.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("search request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8*1024))
|
|
return "", fmt.Errorf("web search failed: status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
|
}
|
|
|
|
results, err := parseSearchResults(resp.Body, args.MaxResults)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return FormatResult(formatWebSearchResults(args.Query, "", results)), nil
|
|
}
|
|
|
|
type BuiltinFetchTool struct {
|
|
client *http.Client
|
|
}
|
|
|
|
type webSearchArgs struct {
|
|
Query string `json:"query"`
|
|
MaxResults int `json:"max_results"`
|
|
Topic string `json:"topic"`
|
|
}
|
|
|
|
type webSearchResult struct {
|
|
Title string `json:"title"`
|
|
URL string `json:"url"`
|
|
Content string `json:"content"`
|
|
Score float64 `json:"score"`
|
|
}
|
|
|
|
type webFetchArgs struct {
|
|
URL string `json:"url"`
|
|
Query string `json:"query"`
|
|
}
|
|
|
|
type webFetchResponse struct {
|
|
Results []webFetchResult `json:"results"`
|
|
FailedResults []webFetchFailed `json:"failed_results"`
|
|
}
|
|
|
|
type webFetchResult struct {
|
|
URL string `json:"url"`
|
|
RawContent string `json:"raw_content"`
|
|
}
|
|
|
|
type webFetchFailed struct {
|
|
URL string `json:"url"`
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
func NewBuiltinFetchTool(client *http.Client) *BuiltinFetchTool {
|
|
if client == nil {
|
|
client = http.DefaultClient
|
|
}
|
|
return &BuiltinFetchTool{client: client}
|
|
}
|
|
|
|
func (t *BuiltinFetchTool) Definition() llm.Tool {
|
|
return webFetchDefinition()
|
|
}
|
|
|
|
func webFetchDefinition() llm.Tool {
|
|
return llm.Tool{
|
|
Type: "function",
|
|
Function: llm.ToolFunction{
|
|
Name: "web_fetch",
|
|
Description: "Fetch and extract readable Markdown content from a public web URL. Use after web_search when snippets are not enough, or when the user gives a URL or website to inspect. Results include the source URL.",
|
|
Parameters: JSONSchema(`{
|
|
"type": "object",
|
|
"properties": {
|
|
"url": {"type": "string", "description": "Public http or https URL to fetch. URLs without a scheme are treated as https URLs."},
|
|
"query": {"type": "string", "description": "Optional extraction focus. Accepted for compatibility but the built-in backend may ignore it."}
|
|
},
|
|
"required": ["url"],
|
|
"additionalProperties": false
|
|
}`),
|
|
},
|
|
}
|
|
}
|
|
|
|
func (t *BuiltinFetchTool) Execute(ctx context.Context, raw json.RawMessage) (string, error) {
|
|
args, err := parseWebFetchArgs(raw)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, args.URL, nil)
|
|
if err != nil {
|
|
return "", fmt.Errorf("create fetch request: %w", err)
|
|
}
|
|
req.Header.Set("User-Agent", defaultUserAgent)
|
|
req.Header.Set("Accept", "text/html,text/plain,application/xhtml+xml")
|
|
|
|
resp, err := t.client.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("fetch request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8*1024))
|
|
return "", fmt.Errorf("web fetch failed: status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
|
}
|
|
|
|
data, err := io.ReadAll(io.LimitReader(resp.Body, maxWebReadBytes))
|
|
if err != nil {
|
|
return "", fmt.Errorf("read response: %w", err)
|
|
}
|
|
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
|
|
var content string
|
|
if strings.Contains(contentType, "html") || contentType == "" {
|
|
content, err = htmlToMarkdown(string(data), args.URL)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
} else if strings.Contains(contentType, "text/plain") || strings.Contains(contentType, "markdown") {
|
|
content = strings.TrimSpace(string(data))
|
|
} else {
|
|
return "", fmt.Errorf("unsupported content type: %s", contentType)
|
|
}
|
|
|
|
result := webFetchResponse{
|
|
Results: []webFetchResult{{
|
|
URL: args.URL,
|
|
RawContent: content,
|
|
}},
|
|
}
|
|
return FormatResult(formatWebFetchResults(args.URL, result)), nil
|
|
}
|
|
|
|
func parseWebSearchArgs(raw json.RawMessage, defaultMaxResults int) (webSearchArgs, error) {
|
|
var args webSearchArgs
|
|
if err := json.Unmarshal(raw, &args); err != nil {
|
|
return args, fmt.Errorf("parse arguments: %w", err)
|
|
}
|
|
args.Query = strings.TrimSpace(args.Query)
|
|
if args.Query == "" {
|
|
return args, errors.New("query is required")
|
|
}
|
|
if args.MaxResults == 0 {
|
|
args.MaxResults = defaultMaxResults
|
|
}
|
|
if args.MaxResults < 1 || args.MaxResults > maxSearchResults {
|
|
return args, fmt.Errorf("max_results must be between 1 and %d", maxSearchResults)
|
|
}
|
|
args.Topic = strings.ToLower(strings.TrimSpace(args.Topic))
|
|
if args.Topic == "" {
|
|
args.Topic = "general"
|
|
}
|
|
switch args.Topic {
|
|
case "general", "news", "finance":
|
|
default:
|
|
return args, fmt.Errorf("topic must be one of: general, news, finance")
|
|
}
|
|
return args, nil
|
|
}
|
|
|
|
func parseWebFetchArgs(raw json.RawMessage) (webFetchArgs, error) {
|
|
var args webFetchArgs
|
|
if err := json.Unmarshal(raw, &args); err != nil {
|
|
return args, fmt.Errorf("parse arguments: %w", err)
|
|
}
|
|
args.URL = strings.TrimSpace(args.URL)
|
|
if args.URL == "" {
|
|
return args, errors.New("url is required")
|
|
}
|
|
if !strings.Contains(args.URL, "://") {
|
|
args.URL = "https://" + args.URL
|
|
}
|
|
parsed, err := url.Parse(args.URL)
|
|
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
|
return args, errors.New("url must be a valid http or https URL")
|
|
}
|
|
args.Query = strings.TrimSpace(args.Query)
|
|
return args, nil
|
|
}
|
|
|
|
func formatWebSearchResults(query string, answer string, results []webSearchResult) string {
|
|
if len(results) == 0 {
|
|
return fmt.Sprintf("No web search results for %q.", query)
|
|
}
|
|
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "Web search results for %q:\n", query)
|
|
if strings.TrimSpace(answer) != "" {
|
|
fmt.Fprintf(&b, "\nSummary: %s\n", strings.TrimSpace(answer))
|
|
}
|
|
for i, result := range results {
|
|
title := strings.TrimSpace(result.Title)
|
|
if title == "" {
|
|
title = "Untitled"
|
|
}
|
|
fmt.Fprintf(&b, "\n%d. %s\n", i+1, title)
|
|
fmt.Fprintf(&b, " Source: %s\n", strings.TrimSpace(result.URL))
|
|
if content := strings.TrimSpace(result.Content); content != "" {
|
|
fmt.Fprintf(&b, " Snippet: %s\n", content)
|
|
}
|
|
if result.Score > 0 {
|
|
fmt.Fprintf(&b, " Score: %.3f\n", result.Score)
|
|
}
|
|
}
|
|
return strings.TrimRight(b.String(), "\n")
|
|
}
|
|
|
|
func formatWebFetchResults(inputURL string, response webFetchResponse) string {
|
|
var b strings.Builder
|
|
if len(response.Results) == 0 {
|
|
fmt.Fprintf(&b, "No readable web content extracted from %s.", inputURL)
|
|
} else {
|
|
fmt.Fprintf(&b, "Fetched web content:\n")
|
|
for i, result := range response.Results {
|
|
sourceURL := strings.TrimSpace(result.URL)
|
|
if sourceURL == "" {
|
|
sourceURL = inputURL
|
|
}
|
|
fmt.Fprintf(&b, "\n%d. Source: %s\n", i+1, sourceURL)
|
|
if content := strings.TrimSpace(result.RawContent); content != "" {
|
|
fmt.Fprintf(&b, "\n%s\n", content)
|
|
}
|
|
}
|
|
}
|
|
for _, failed := range response.FailedResults {
|
|
sourceURL := strings.TrimSpace(failed.URL)
|
|
if sourceURL == "" {
|
|
sourceURL = inputURL
|
|
}
|
|
if strings.TrimSpace(failed.Error) != "" {
|
|
fmt.Fprintf(&b, "\nFetch failed for %s: %s", sourceURL, strings.TrimSpace(failed.Error))
|
|
}
|
|
}
|
|
return strings.TrimRight(b.String(), "\n")
|
|
}
|
|
|
|
func parseSearchResults(r io.Reader, maxResults int) ([]webSearchResult, error) {
|
|
doc, err := html.Parse(r)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse search results: %w", err)
|
|
}
|
|
|
|
results := make([]webSearchResult, 0, maxResults)
|
|
var walk func(*html.Node)
|
|
walk = func(n *html.Node) {
|
|
if len(results) >= maxResults || n == nil {
|
|
return
|
|
}
|
|
if n.Type == html.ElementNode && n.Data == "div" && hasClass(n, "result") {
|
|
if result, ok := parseResultNode(n); ok {
|
|
results = append(results, result)
|
|
}
|
|
return
|
|
}
|
|
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
|
walk(child)
|
|
}
|
|
}
|
|
walk(doc)
|
|
return results, nil
|
|
}
|
|
|
|
func parseResultNode(n *html.Node) (webSearchResult, bool) {
|
|
var result webSearchResult
|
|
var walk func(*html.Node)
|
|
walk = func(node *html.Node) {
|
|
if node == nil {
|
|
return
|
|
}
|
|
if node.Type == html.ElementNode && node.Data == "a" {
|
|
if hasClass(node, "result__a") && result.URL == "" {
|
|
result.Title = cleanWebText(textContent(node))
|
|
result.URL = decodeSearchURL(attr(node, "href"))
|
|
}
|
|
if hasClass(node, "result__snippet") && result.Content == "" {
|
|
result.Content = cleanWebText(textContent(node))
|
|
}
|
|
}
|
|
if node.Type == html.ElementNode && (node.Data == "div" || node.Data == "span") && hasClass(node, "result__snippet") && result.Content == "" {
|
|
result.Content = cleanWebText(textContent(node))
|
|
}
|
|
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
|
walk(child)
|
|
}
|
|
}
|
|
walk(n)
|
|
return result, result.Title != "" && result.URL != ""
|
|
}
|
|
|
|
func decodeSearchURL(raw string) string {
|
|
raw = strings.TrimSpace(raw)
|
|
if raw == "" {
|
|
return ""
|
|
}
|
|
parsed, err := url.Parse(raw)
|
|
if err == nil {
|
|
if target := parsed.Query().Get("uddg"); target != "" {
|
|
if decoded, err := url.QueryUnescape(target); err == nil {
|
|
return decoded
|
|
}
|
|
return target
|
|
}
|
|
}
|
|
return raw
|
|
}
|
|
|
|
func htmlToMarkdown(input string, sourceURL string) (string, error) {
|
|
doc, err := html.Parse(strings.NewReader(input))
|
|
if err != nil {
|
|
return "", fmt.Errorf("parse html: %w", err)
|
|
}
|
|
|
|
var b strings.Builder
|
|
var title string
|
|
var walk func(*html.Node, int)
|
|
walk = func(n *html.Node, listDepth int) {
|
|
if n == nil || skipHTMLNode(n) {
|
|
return
|
|
}
|
|
if n.Type == html.ElementNode && n.Data == "title" && title == "" {
|
|
title = cleanWebText(textContent(n))
|
|
return
|
|
}
|
|
if n.Type == html.TextNode {
|
|
writeInlineText(&b, n.Data)
|
|
return
|
|
}
|
|
if n.Type != html.ElementNode {
|
|
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
|
walk(child, listDepth)
|
|
}
|
|
return
|
|
}
|
|
|
|
switch n.Data {
|
|
case "h1", "h2", "h3", "h4", "h5", "h6":
|
|
writeBlockBreak(&b)
|
|
level := int(n.Data[1] - '0')
|
|
b.WriteString(strings.Repeat("#", level))
|
|
b.WriteString(" ")
|
|
writeChildren(&b, n, listDepth, walk)
|
|
writeBlockBreak(&b)
|
|
case "p", "div", "section", "article", "header", "footer", "main", "tr":
|
|
writeBlockBreak(&b)
|
|
writeChildren(&b, n, listDepth, walk)
|
|
writeBlockBreak(&b)
|
|
case "br":
|
|
b.WriteString("\n")
|
|
case "li":
|
|
writeBlockBreak(&b)
|
|
b.WriteString(strings.Repeat(" ", listDepth))
|
|
b.WriteString("- ")
|
|
writeChildren(&b, n, listDepth+1, walk)
|
|
writeBlockBreak(&b)
|
|
case "a":
|
|
label := cleanWebText(textContent(n))
|
|
href := strings.TrimSpace(attr(n, "href"))
|
|
if href != "" && label != "" {
|
|
href = resolveURL(sourceURL, href)
|
|
b.WriteString(label)
|
|
b.WriteString(" (")
|
|
b.WriteString(href)
|
|
b.WriteString(")")
|
|
return
|
|
}
|
|
writeChildren(&b, n, listDepth, walk)
|
|
default:
|
|
writeChildren(&b, n, listDepth, walk)
|
|
}
|
|
}
|
|
walk(doc, 0)
|
|
|
|
content := normalizeMarkdownText(b.String())
|
|
if title != "" && !strings.Contains(content, title) {
|
|
content = "# " + title + "\n\n" + content
|
|
}
|
|
if strings.TrimSpace(content) == "" {
|
|
return "", errors.New("no readable content found")
|
|
}
|
|
return content, nil
|
|
}
|
|
|
|
func writeChildren(b *strings.Builder, n *html.Node, listDepth int, walk func(*html.Node, int)) {
|
|
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
|
walk(child, listDepth)
|
|
}
|
|
}
|
|
|
|
func writeInlineText(b *strings.Builder, text string) {
|
|
text = cleanWebText(text)
|
|
if text == "" {
|
|
return
|
|
}
|
|
if b.Len() > 0 {
|
|
last := b.String()[b.Len()-1]
|
|
if last != '\n' && last != ' ' {
|
|
b.WriteByte(' ')
|
|
}
|
|
}
|
|
b.WriteString(text)
|
|
}
|
|
|
|
func writeBlockBreak(b *strings.Builder) {
|
|
if b.Len() == 0 {
|
|
return
|
|
}
|
|
text := b.String()
|
|
if strings.HasSuffix(text, "\n\n") {
|
|
return
|
|
}
|
|
if strings.HasSuffix(text, "\n") {
|
|
b.WriteByte('\n')
|
|
return
|
|
}
|
|
b.WriteString("\n\n")
|
|
}
|
|
|
|
func skipHTMLNode(n *html.Node) bool {
|
|
if n.Type != html.ElementNode {
|
|
return false
|
|
}
|
|
switch n.Data {
|
|
case "script", "style", "noscript", "svg", "canvas", "template":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func textContent(n *html.Node) string {
|
|
var b strings.Builder
|
|
var walk func(*html.Node)
|
|
walk = func(node *html.Node) {
|
|
if node == nil || skipHTMLNode(node) {
|
|
return
|
|
}
|
|
if node.Type == html.TextNode {
|
|
b.WriteString(node.Data)
|
|
b.WriteByte(' ')
|
|
return
|
|
}
|
|
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
|
walk(child)
|
|
}
|
|
}
|
|
walk(n)
|
|
return cleanWebText(b.String())
|
|
}
|
|
|
|
func hasClass(n *html.Node, class string) bool {
|
|
for _, item := range strings.Fields(attr(n, "class")) {
|
|
if item == class {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func attr(n *html.Node, name string) string {
|
|
for _, item := range n.Attr {
|
|
if item.Key == name {
|
|
return item.Val
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func resolveURL(baseURL, raw string) string {
|
|
ref, err := url.Parse(raw)
|
|
if err != nil {
|
|
return raw
|
|
}
|
|
base, err := url.Parse(baseURL)
|
|
if err != nil {
|
|
return raw
|
|
}
|
|
return base.ResolveReference(ref).String()
|
|
}
|
|
|
|
func cleanWebText(text string) string {
|
|
return strings.Join(strings.Fields(text), " ")
|
|
}
|
|
|
|
func normalizeMarkdownText(text string) string {
|
|
lines := strings.Split(text, "\n")
|
|
out := make([]string, 0, len(lines))
|
|
blank := false
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
if !blank && len(out) > 0 {
|
|
out = append(out, "")
|
|
}
|
|
blank = true
|
|
continue
|
|
}
|
|
out = append(out, line)
|
|
blank = false
|
|
}
|
|
return strings.TrimSpace(strings.Join(out, "\n"))
|
|
}
|