Cody web: add server-side fetching for URL mentions (#64223)

Due to CORS rules, cody web cannot make requests to arbitrary URLs.
Because of this, mentions for URLs do not currently work in Cody Web.

This adds a GraphQL endpoint to the cody context resolvers that resolves
the contents of a mentioned URL.

Author: camdencheek

cmd/frontend/graphqlbackend/cody_context.go

@@ -11,6 +11,7 @@ type CodyContextResolver interface {
 	ChatContext(ctx context.Context, args ChatContextArgs) (ChatContextResolver, error)
 	RankContext(ctx context.Context, args RankContextArgs) (RankContextResolver, error)
 	RecordContext(ctx context.Context, args RecordContextArgs) (*EmptyResponse, error)
+	UrlMentionContext(ctx context.Context, args UrlMentionContextArgs) (UrlMentionContextResolver, error)
 	// GetCodyContext is the existing Cody Enterprise context endpoint
 	GetCodyContext(ctx context.Context, args GetContextArgs) ([]ContextResultResolver, error)
 }
@@ -22,6 +23,15 @@ type GetContextArgs struct {
 	TextResultsCount int32
 }
 
+type UrlMentionContextArgs struct {
+	Url string
+}
+
+type UrlMentionContextResolver interface {
+	Title() *string
+	Content() string
+}
+
 type ContextResultResolver interface {
 	ToFileChunkContext() (*FileChunkContextResolver, bool)
 }

cmd/frontend/graphqlbackend/cody_context.graphql

@@ -112,6 +112,11 @@ extend type Query {
         """
         resultsCount: Int
     ): ChatContextResult!
+
+    """
+    EXPERIMENTAL: Fetches the relevant context for a mentioned URL
+    """
+    urlMentionContext(url: String!): URLMentionContextResult!
 }
 
 """
@@ -270,3 +275,18 @@ type RetrieverContextItem {
     """
     item: CodyContextResult!
 }
+
+"""
+EXPERIMENTAL: The result of fetching context for a URL mention
+"""
+type URLMentionContextResult {
+    """
+    The extracted title of the page, if it exists
+    """
+    title: String
+    """
+    The content of the page in its processed and truncated form.
+    Not guaranteed to be in any particular format.
+    """
+    content: String!
+}

cmd/frontend/internal/context/resolvers/BUILD.bazel

@@ -24,6 +24,8 @@ go_library(
         "//schema",
         "@com_github_cohere_ai_cohere_go_v2//:cohere-go",
         "@com_github_cohere_ai_cohere_go_v2//client",
+        "@com_github_grafana_regexp//:regexp",
+        "@com_github_k3a_html2text//:html2text",
         "@com_github_sourcegraph_conc//iter",
         "@com_github_sourcegraph_conc//pool",
         "@com_github_sourcegraph_log//:log",

cmd/frontend/internal/context/resolvers/context.go

@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/grafana/regexp"
+	"github.com/k3a/html2text"
 	"github.com/sourcegraph/conc/iter"
 	"github.com/sourcegraph/conc/pool"
 	"github.com/sourcegraph/log"
@@ -427,6 +429,74 @@ func (r *Resolver) fetchZoekt(ctx context.Context, query string, repo *types.Rep
 	return res, partialErrors, nil
 }
 
+var titleRegexp = regexp.MustCompile(`<title>([^<]+)</title>`)
+
+const urlContextReadLimit = 5 * 1024 * 1024
+const urlContextOutputLimit = 14000
+
+type urlMentionContextResponse struct {
+	title   *string
+	content string
+}
+
+func (u *urlMentionContextResponse) Title() *string {
+	return u.title
+}
+
+func (u *urlMentionContextResponse) Content() string {
+	return u.content
+}
+
+func (r *Resolver) UrlMentionContext(ctx context.Context, args graphqlbackend.UrlMentionContextArgs) (graphqlbackend.UrlMentionContextResolver, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", args.Url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// 🚨 SECURITY: This endpoint allows API users to create GET requests against arbitrary URLs.
+	// To mitigate risk of SSRF, we use an the ExternalClient, which denies requests to internal targets.
+	resp, err := httpcli.UncachedExternalClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= http.StatusBadRequest {
+		return nil, errors.Errorf("request failed with status %d", resp.StatusCode)
+	}
+
+	// 🚨 SECURITY: Limit the amount of data we will read into memory.
+	content, err := io.ReadAll(io.LimitReader(resp.Body, urlContextReadLimit))
+	if err != nil {
+		return nil, err
+	}
+
+	// Attempt to extract the title
+	var title *string
+	if match := titleRegexp.FindSubmatch(content); match != nil {
+		title = pointers.Ptr(string(match[1]))
+	}
+
+	// Trim to main if it exists since that's a decent signal pointing to the important part of the page.
+	if idx := bytes.Index(content, []byte("<main")); idx > 0 {
+		content = content[idx:]
+	}
+	if idx := bytes.Index(content, []byte("</main>")); idx > 0 {
+		content = content[:idx+len("</main>")]
+	}
+
+	// Convert the HTML to text to make the ouptut higher density. The output
+	// is still pretty crude, but it does enough to capture the description and
+	// most comments from a github PR. There is significant room to improve
+	// content extraction here.
+	textified := html2text.HTML2TextWithOptions(string(content), html2text.WithUnixLineBreaks())
+	textified = textified[:min(len(textified), urlContextOutputLimit)]
+	return &urlMentionContextResponse{
+		title:   title,
+		content: textified,
+	}, nil
+}
+
 // countLines finds the number of lines corresponding to the number of runes. We 'round down'
 // to ensure that we don't return more characters than our budget.
 func countLines(content string, numRunes int) int {