bit more hacking away at it

internal/subscriptions/subscriptions.go

@@ -19,14 +19,21 @@
 
 import (
 	"context"
+	"encoding/csv"
+	"errors"
+	"time"
 
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 type Subscriptions struct {
 	state               *state.State
+	transportController transport.Controller
 }
 
 func (s *Subscriptions) UpdateDomainPermissions(
@@ -40,22 +47,112 @@ func (s *Subscriptions) UpdateDomainPermissions(
 
 	// Get permission subscriptions in priority order (highest -> lowest).
 	permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType)
-	if err != nil {
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		// Real db error.
 		l.Error(err)
 		return
 	}
 
 	if len(permSubs) == 0 {
-		// Nothing to do.
+		// No subscriptions of this
+		// type, so nothing to do.
+		return
+	}
+
+	// Get a transport using the instance account,
+	// we can reuse this for each HTTP call.
+	tsport, err := s.transportController.NewTransportForUsername(ctx, "")
+	if err != nil {
+		l.Error(err)
 		return
 	}
 
 	for i, permSub := range permSubs {
+		// Set FetchedAt as we're
+		// going to attempt this now.
+		permSub.FetchedAt = time.Now()
+		columns := []string{"fetched_at"}
 
-		// Slice of permission subscriptions that
+		// Call the URI but don't force.
-		// have a higher priority than this one.
+		resp, err := tsport.DereferenceDomainPermissions(
-		higherPrios := permSubs[:i]
+			ctx, permSub, false,
+		)
+		if err != nil {
+			// Bollocks, couldn't get this one.
+			// Just save the error in the db
+			// for later perusal by admin.
+			permSub.Error = err.Error()
+			columns = append(columns, "error")
+			if err := s.state.DB.UpdateDomainPermissionSubscription(
+				ctx, permSub, columns...,
+			); err != nil {
+				// Real db error.
+				l.Error(err)
+				return
+			}
 
+			// Skip to the
+			// next permSub.
+			continue
+		}
+
+		// If the permissions at URI weren't modified
+		// since last time, just update some metadata
+		// and call this a successful fetch.
+		if resp.Unmodified {
+			permSub.SuccessfullyFetchedAt = time.Now()
+			columns = append(columns, "successfully_fetched_at")
+
+			if permSub.ETag == "" && resp.ETag != "" {
+				// We didn't have an ETag before but
+				// we have one now: probably the remote
+				// added ETag support in the meantime.
+				permSub.ETag = resp.ETag
+				columns = append(columns, "etag")
+			}
+
+			if err := s.state.DB.UpdateDomainPermissionSubscription(
+				ctx, permSub, columns...,
+			); err != nil {
+				// Real db error.
+				l.Error(err)
+				return
+			}
+
+			// Skip to the
+			// next permSub.
+			continue
+		}
+
+		// At this point we know we got a 200 OK from
+		// the URI, so we've got a live body. Make sure
+		// we close it when done, wrapping the close func
+		// with DoOnce for more granular control.
+		close := util.DoOnce(func() { resp.Body.Close() })
+		defer close()
+
+		// Try to parse the body as a
+		// list of domain permissions.
+		switch permSub.ContentType {
+
+		// text/csv
+		case gtsmodel.DomainPermSubContentTypeCSV:
+			records, err := csv.NewReader(resp.Body).ReadAll()
+			if err != nil {
 				
 			}
+
+		// application/json
+		case gtsmodel.DomainPermSubContentTypeJSON:
+
+		// text/plain
+		case gtsmodel.DomainPermSubContentTypePlain:
+		}
+
+		// Slice of permission subscriptions that have
+		// a higher priority than this one. We should
+		// not override perms if they already exist
+		// under a higher-priority subscription.
+		higherPrios := permSubs[:i]
+	}
 }

internal/transport/derefdomainpermlist.go

@@ -26,15 +26,26 @@
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-type DomainPermissionsRespRaw struct {
+type DereferenceDomainPermissionsResp struct {
-	body []byte
+	// Set only if response was 200 OK.
+	// It's up to the caller to close
+	// this when they're done with it.
+	Body io.ReadCloser
+
+	// True if response
+	// was 304 Not Modified.
+	Unmodified bool
+
+	// May be set
+	// if 200 or 304.
+	ETag string
 }
 
 func (t *transport) DereferenceDomainPermissions(
 	ctx context.Context,
 	permSub *gtsmodel.DomainPermissionSubscription,
-	useCacheHeaders bool,
+	force bool,
-) (*DomainPermissionsRespRaw, error) {
+) (*DereferenceDomainPermissionsResp, error) {
 	// Prepare new HTTP request to endpoint
 	req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil)
 	if err != nil {
@@ -52,7 +63,9 @@ func (t *transport) DereferenceDomainPermissions(
 	req.Header.Add("Accept-Charset", "utf-8")
 	req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*")
 
-	if useCacheHeaders {
+	// If force is true, we want to skip setting Cache
+	// headers so that we definitely don't get a 304 back.
+	if !force {
 		// If we've successfully fetched this list
 		// before, set If-Modified-Since to last
 		// success to make the request conditional.
@@ -76,24 +89,35 @@ func (t *transport) DereferenceDomainPermissions(
 	if err != nil {
 		return nil, err
 	}
-	defer rsp.Body.Close()
 
-	// Read the body regardless of response code,
+	// If we have an unexpected / error response,
-	// as we may want to store any error message.
+	// wrap + return as error. This will also drain
-	bytes, err := io.ReadAll(rsp.Body)
+	// and close the response body for us.
-	if err != nil {
+	if rsp.StatusCode != http.StatusOK &&
-		return nil, err
+		rsp.StatusCode != http.StatusNotModified {
-	}
-
-	if rsp.StatusCode == http.StatusNotModified {
-		// Nothing has changed on the remote side since
-		// we last fetched, so there's nothing to do.
-		return nil, nil
-	}
-
-	// Ensure a non-error status response.
-	if rsp.StatusCode != http.StatusOK {
 		err := gtserror.NewFromResponse(rsp)
 		return nil, err
 	}
+
+	// Check already if we were given an ETag
+	// we can use, as ETag is often returned
+	// even on 304 Not Modified responses.
+	eTag := rsp.Header.Get("ETag")
+
+	if rsp.StatusCode == http.StatusNotModified {
+		// Nothing has changed on the remote side
+		// since we last fetched, so there's nothing
+		// to do and we don't need to read the body.
+		rsp.Body.Close()
+		return &DereferenceDomainPermissionsResp{
+			Unmodified: true,
+			ETag:       eTag,
+		}, nil
+	}
+
+	// Return the body + ETag to the caller.
+	return &DereferenceDomainPermissionsResp{
+		Body: rsp.Body,
+		ETag: eTag,
+	}, nil
 }

internal/transport/transport.go

@@ -78,6 +78,20 @@ type Transport interface {
 	// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
 	DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
 
+	// DereferenceDomainPermissions dereferences the
+	// permissions list present at the given permSub's URI.
+	//
+	// If "force", then If-Modified-Since and If-None-Match
+	// headers will *NOT* be sent with the outgoing request.
+	//
+	// If err == nil and Unmodified == false, then it's up
+	// to the caller to close the returned io.ReadCloser.
+	DereferenceDomainPermissions(
+		ctx context.Context,
+		permSub *gtsmodel.DomainPermissionSubscription,
+		force bool,
+	) (*DereferenceDomainPermissionsResp, error)
+
 	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
 	Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error)
 }