[feature] Add /api/v1/admin/debug/apurl endpoint (#2359)

docs/api/swagger.yaml

@@ -921,6 +921,46 @@ definitions:
         type: object
         x-go-name: Card
         x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model
+    debugAPUrlResponse:
+        description: |-
+            DebugAPUrlResponse provides detailed debug
+            information for an AP URL dereference request.
+        properties:
+            request_headers:
+                additionalProperties:
+                    items:
+                        type: string
+                    type: array
+                description: HTTP headers used in the outgoing request.
+                type: object
+                x-go-name: RequestHeaders
+            request_url:
+                description: Remote AP URL that was requested.
+                type: string
+                x-go-name: RequestURL
+            response_body:
+                description: |-
+                    Body returned from the remote instance.
+                    Will be stringified bytes; may be JSON,
+                    may be an error, may be both!
+                type: string
+                x-go-name: ResponseBody
+            response_code:
+                description: HTTP response code returned from the remote instance.
+                format: int64
+                type: integer
+                x-go-name: ResponseCode
+            response_headers:
+                additionalProperties:
+                    items:
+                        type: string
+                    type: array
+                description: HTTP headers returned from the remote instance.
+                type: object
+                x-go-name: ResponseHeaders
+        type: object
+        x-go-name: DebugAPUrlResponse
+        x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model
     domain:
         description: Domain represents a remote domain
         properties:
@@ -4066,6 +4106,39 @@ paths:
             summary: Get a list of existing emoji categories.
             tags:
                 - admin
+    /api/v1/admin/debug/apurl:
+        get:
+            description: Only enabled / exposed if GoToSocial was built and is running with flag DEBUG=1.
+            operationId: debugAPUrl
+            parameters:
+                - description: The URL / ActivityPub ID to dereference. This should be a full URL, including protocol. Eg., `https://example.org/users/someone`
+                  in: query
+                  name: url
+                  required: true
+                  type: string
+            produces:
+                - application/json
+            responses:
+                "200":
+                    description: ""
+                    schema:
+                        $ref: '#/definitions/debugAPUrlResponse'
+                "400":
+                    description: bad request
+                "401":
+                    description: unauthorized
+                "404":
+                    description: not found
+                "406":
+                    description: not acceptable
+                "500":
+                    description: internal server error
+            security:
+                - OAuth2 Bearer:
+                    - admin
+            summary: Perform a GET to the specified ActivityPub URL and return detailed debugging information.
+            tags:
+                - debug
     /api/v1/admin/domain_allows:
         get:
             operationId: domainAllowsGet

internal/api/client/admin/admin.go

@@ -20,6 +20,7 @@
 import (
 	"net/http"
 
+	"codeberg.org/gruf/go-debug"
 	"github.com/gin-gonic/gin"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 )
@@ -46,6 +47,8 @@
 	EmailTestPath           = EmailPath + "/test"
 	InstanceRulesPath       = BasePath + "/instance/rules"
 	InstanceRulesPathWithID = InstanceRulesPath + "/:" + IDKey
+	DebugPath               = BasePath + "/debug"
+	DebugAPUrlPath          = DebugPath + "/apurl"
 
 	IDKey                 = "id"
 	FilterQueryKey        = "filter"
@@ -116,4 +119,9 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
 	attachHandler(http.MethodPost, InstanceRulesPath, m.RulePOSTHandler)
 	attachHandler(http.MethodPatch, InstanceRulesPathWithID, m.RulePATCHHandler)
 	attachHandler(http.MethodDelete, InstanceRulesPathWithID, m.RuleDELETEHandler)
+
+	// debug stuff
+	if debug.DEBUG {
+		attachHandler(http.MethodGet, DebugAPUrlPath, m.DebugAPUrlHandler)
+	}
 }

internal/api/client/admin/debug_off.go

@@ -0,0 +1,75 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !debug && !debugenv
+// +build !debug,!debugenv
+
+package admin
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// #######################################################
+// # goswagger is generated using empty / off debug by   #
+// # default, so put all the swagger documentation here! #
+// #######################################################
+
+// DebugAPUrlHandler swagger:operation GET /api/v1/admin/debug/apurl debugAPUrl
+//
+// Perform a GET to the specified ActivityPub URL and return detailed debugging information.
+//
+// Only enabled / exposed if GoToSocial was built and is running with flag DEBUG=1.
+//
+//	---
+//	tags:
+//	- debug
+//
+//	produces:
+//	- application/json
+//
+//	parameters:
+//	-
+//		name: url
+//		type: string
+//		description: >-
+//			The URL / ActivityPub ID to dereference.
+//			This should be a full URL, including protocol.
+//			Eg., `https://example.org/users/someone`
+//		in: query
+//		required: true
+//
+//	security:
+//	- OAuth2 Bearer:
+//		- admin
+//
+//	responses:
+//		'200':
+//			name: Debug response.
+//			schema:
+//				"$ref": "#/definitions/debugAPUrlResponse"
+//		'400':
+//			description: bad request
+//		'401':
+//			description: unauthorized
+//		'404':
+//			description: not found
+//		'406':
+//			description: not acceptable
+//		'500':
+//			description: internal server error
+func (m *Module) DebugAPUrlHandler(c *gin.Context) {}

internal/api/client/admin/debug_on.go

@@ -0,0 +1,58 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build debug || debugenv
+// +build debug debugenv
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+func (m *Module) DebugAPUrlHandler(c *gin.Context) {
+	authed, err := oauth.Authed(c, true, true, true, true)
+	if err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if !*authed.User.Admin {
+		err := fmt.Errorf("user %s not an admin", authed.User.ID)
+		apiutil.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	resp, errWithCode := m.processor.Admin().DebugAPUrl(c.Request.Context(), authed.Account, c.Query("url"))
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	c.JSON(http.StatusOK, resp)
+}

internal/api/model/admin.go

@@ -210,3 +210,22 @@ type AdminInstanceRule struct {
 	UpdatedAt string `json:"updated_at"` // when was item last updated
 	Text      string `json:"text"`       // text content of the rule
 }
+
+// DebugAPUrlResponse provides detailed debug
+// information for an AP URL dereference request.
+//
+// swagger:model debugAPUrlResponse
+type DebugAPUrlResponse struct {
+	// Remote AP URL that was requested.
+	RequestURL string `json:"request_url"`
+	// HTTP headers used in the outgoing request.
+	RequestHeaders map[string][]string `json:"request_headers"`
+	// HTTP headers returned from the remote instance.
+	ResponseHeaders map[string][]string `json:"response_headers"`
+	// HTTP response code returned from the remote instance.
+	ResponseCode int `json:"response_code"`
+	// Body returned from the remote instance.
+	// Will be stringified bytes; may be JSON,
+	// may be an error, may be both!
+	ResponseBody string `json:"response_body"`
+}

internal/processing/admin/debug_apurl.go

@@ -0,0 +1,126 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// DebugAPUrl performs a GET to the given url, using the
+// signature of the given admin account. The GET will
+// have Accept set to the ActivityPub content types.
+//
+// Only urls with schema http or https are allowed.
+//
+// Calls to blocked domains are not allowed, not only
+// because it's unfair to call them when they can't
+// call us, but because it probably won't work anyway
+// if they try to dereference the calling account.
+//
+// Errors returned from this function should be fairly
+// verbose, to help with debugging.
+func (p *Processor) DebugAPUrl(
+	ctx context.Context,
+	adminAcct *gtsmodel.Account,
+	urlStr string,
+) (*apimodel.DebugAPUrlResponse, gtserror.WithCode) {
+	// Validate URL.
+	if urlStr == "" {
+		err := gtserror.New("empty URL")
+		return nil, gtserror.NewErrorBadRequest(err, err.Error())
+	}
+
+	url, err := url.Parse(urlStr)
+	if err != nil {
+		err := gtserror.Newf("invalid URL: %w", err)
+		return nil, gtserror.NewErrorBadRequest(err, err.Error())
+	}
+
+	if url == nil || (url.Scheme != "http" && url.Scheme != "https") {
+		err = gtserror.New("invalid URL scheme, acceptable schemes are http or https")
+		return nil, gtserror.NewErrorBadRequest(err, err.Error())
+	}
+
+	// Ensure URL not blocked.
+	blocked, err := p.state.DB.IsDomainBlocked(ctx, url.Host)
+	if err != nil {
+		err = gtserror.Newf("db error checking for domain block: %w", err)
+		return nil, gtserror.NewErrorInternalError(err, err.Error())
+	}
+
+	if blocked {
+		err = gtserror.Newf("target domain %s is blocked", url.Host)
+		return nil, gtserror.NewErrorBadRequest(err, err.Error())
+	}
+
+	// All looks fine. Prepare the transport and (signed) GET request.
+	tsport, err := p.transportController.NewTransportForUsername(ctx, adminAcct.Username)
+	if err != nil {
+		err = gtserror.Newf("error creating transport: %w", err)
+		return nil, gtserror.NewErrorInternalError(err, err.Error())
+	}
+
+	req, err := http.NewRequestWithContext(
+		// Caller will want a snappy
+		// response so don't retry.
+		gtscontext.SetFastFail(ctx),
+		http.MethodGet, urlStr, nil,
+	)
+	if err != nil {
+		err = gtserror.Newf("error creating request: %w", err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
+	}
+
+	req.Header.Add("Accept", string(apiutil.AppActivityLDJSON)+","+string(apiutil.AppActivityJSON))
+	req.Header.Add("Accept-Charset", "utf-8")
+	req.Header.Set("Host", url.Host)
+
+	// Perform the HTTP request,
+	// and return everything.
+	rsp, err := tsport.GET(req)
+	if err != nil {
+		err = gtserror.Newf("error doing dereference: %w", err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
+	}
+	defer rsp.Body.Close()
+
+	b, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		err := gtserror.Newf("error reading response body bytes: %w", err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
+	}
+
+	debugResponse := &apimodel.DebugAPUrlResponse{
+		RequestURL:      urlStr,
+		RequestHeaders:  req.Header,
+		ResponseHeaders: rsp.Header,
+		ResponseCode:    rsp.StatusCode,
+		ResponseBody:    string(b),
+	}
+
+	return debugResponse, nil
+}

internal/transport/transport.go

@@ -46,6 +46,10 @@ type Transport interface {
 		POST functions
 	*/
 
+	// POST will perform given the http request using
+	// transport client, retrying on certain preset errors.
+	POST(*http.Request, []byte) (*http.Response, error)
+
 	// Deliver sends an ActivityStreams object.
 	Deliver(ctx context.Context, b []byte, to *url.URL) error
 
@@ -56,6 +60,10 @@ type Transport interface {
 		GET functions
 	*/
 
+	// GET will perform the given http request using
+	// transport client, retrying on certain preset errors.
+	GET(*http.Request) (*http.Response, error)
+
 	// Dereference fetches the ActivityStreams object located at this IRI with a GET request.
 	Dereference(ctx context.Context, iri *url.URL) ([]byte, error)
 
@@ -81,7 +89,6 @@ type transport struct {
 	signerMu   sync.Mutex
 }
 
-// GET will perform given http request using transport client, retrying on certain preset errors.
 func (t *transport) GET(r *http.Request) (*http.Response, error) {
 	if r.Method != http.MethodGet {
 		return nil, errors.New("must be GET request")
@@ -93,7 +100,6 @@ func (t *transport) GET(r *http.Request) (*http.Response, error) {
 	return t.controller.client.DoSigned(r, t.signGET())
 }
 
-// POST will perform given http request using transport client, retrying on certain preset errors.
 func (t *transport) POST(r *http.Request, body []byte) (*http.Response, error) {
 	if r.Method != http.MethodPost {
 		return nil, errors.New("must be POST request")