diff --git a/internal/subscriptions/subscriptions.go b/internal/subscriptions/subscriptions.go index 90d5609a5..ac3d60da5 100644 --- a/internal/subscriptions/subscriptions.go +++ b/internal/subscriptions/subscriptions.go @@ -19,14 +19,21 @@ import ( "context" + "encoding/csv" + "errors" + "time" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/util" ) type Subscriptions struct { - state *state.State + state *state.State + transportController transport.Controller } func (s *Subscriptions) UpdateDomainPermissions( @@ -40,22 +47,112 @@ func (s *Subscriptions) UpdateDomainPermissions( // Get permission subscriptions in priority order (highest -> lowest). permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType) - if err != nil { + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real db error. l.Error(err) return } if len(permSubs) == 0 { - // Nothing to do. + // No subscriptions of this + // type, so nothing to do. + return + } + + // Get a transport using the instance account, + // we can reuse this for each HTTP call. + tsport, err := s.transportController.NewTransportForUsername(ctx, "") + if err != nil { + l.Error(err) return } for i, permSub := range permSubs { - - // Slice of permission subscriptions that - // have a higher priority than this one. - higherPrios := permSubs[:i] + // Set FetchedAt as we're + // going to attempt this now. + permSub.FetchedAt = time.Now() + columns := []string{"fetched_at"} - + // Call the URI but don't force. + resp, err := tsport.DereferenceDomainPermissions( + ctx, permSub, false, + ) + if err != nil { + // Bollocks, couldn't get this one. + // Just save the error in the db + // for later perusal by admin. + permSub.Error = err.Error() + columns = append(columns, "error") + if err := s.state.DB.UpdateDomainPermissionSubscription( + ctx, permSub, columns..., + ); err != nil { + // Real db error. + l.Error(err) + return + } + + // Skip to the + // next permSub. + continue + } + + // If the permissions at URI weren't modified + // since last time, just update some metadata + // and call this a successful fetch. + if resp.Unmodified { + permSub.SuccessfullyFetchedAt = time.Now() + columns = append(columns, "successfully_fetched_at") + + if permSub.ETag == "" && resp.ETag != "" { + // We didn't have an ETag before but + // we have one now: probably the remote + // added ETag support in the meantime. + permSub.ETag = resp.ETag + columns = append(columns, "etag") + } + + if err := s.state.DB.UpdateDomainPermissionSubscription( + ctx, permSub, columns..., + ); err != nil { + // Real db error. + l.Error(err) + return + } + + // Skip to the + // next permSub. + continue + } + + // At this point we know we got a 200 OK from + // the URI, so we've got a live body. Make sure + // we close it when done, wrapping the close func + // with DoOnce for more granular control. + close := util.DoOnce(func() { resp.Body.Close() }) + defer close() + + // Try to parse the body as a + // list of domain permissions. + switch permSub.ContentType { + + // text/csv + case gtsmodel.DomainPermSubContentTypeCSV: + records, err := csv.NewReader(resp.Body).ReadAll() + if err != nil { + + } + + // application/json + case gtsmodel.DomainPermSubContentTypeJSON: + + // text/plain + case gtsmodel.DomainPermSubContentTypePlain: + } + + // Slice of permission subscriptions that have + // a higher priority than this one. We should + // not override perms if they already exist + // under a higher-priority subscription. + higherPrios := permSubs[:i] } } diff --git a/internal/transport/derefdomainpermlist.go b/internal/transport/derefdomainpermlist.go index 045c69a26..b20e2bd2e 100644 --- a/internal/transport/derefdomainpermlist.go +++ b/internal/transport/derefdomainpermlist.go @@ -26,15 +26,26 @@ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -type DomainPermissionsRespRaw struct { - body []byte +type DereferenceDomainPermissionsResp struct { + // Set only if response was 200 OK. + // It's up to the caller to close + // this when they're done with it. + Body io.ReadCloser + + // True if response + // was 304 Not Modified. + Unmodified bool + + // May be set + // if 200 or 304. + ETag string } func (t *transport) DereferenceDomainPermissions( ctx context.Context, permSub *gtsmodel.DomainPermissionSubscription, - useCacheHeaders bool, -) (*DomainPermissionsRespRaw, error) { + force bool, +) (*DereferenceDomainPermissionsResp, error) { // Prepare new HTTP request to endpoint req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil) if err != nil { @@ -52,7 +63,9 @@ func (t *transport) DereferenceDomainPermissions( req.Header.Add("Accept-Charset", "utf-8") req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*") - if useCacheHeaders { + // If force is true, we want to skip setting Cache + // headers so that we definitely don't get a 304 back. + if !force { // If we've successfully fetched this list // before, set If-Modified-Since to last // success to make the request conditional. @@ -76,24 +89,35 @@ func (t *transport) DereferenceDomainPermissions( if err != nil { return nil, err } - defer rsp.Body.Close() - // Read the body regardless of response code, - // as we may want to store any error message. - bytes, err := io.ReadAll(rsp.Body) - if err != nil { - return nil, err - } - - if rsp.StatusCode == http.StatusNotModified { - // Nothing has changed on the remote side since - // we last fetched, so there's nothing to do. - return nil, nil - } - - // Ensure a non-error status response. - if rsp.StatusCode != http.StatusOK { + // If we have an unexpected / error response, + // wrap + return as error. This will also drain + // and close the response body for us. + if rsp.StatusCode != http.StatusOK && + rsp.StatusCode != http.StatusNotModified { err := gtserror.NewFromResponse(rsp) return nil, err } + + // Check already if we were given an ETag + // we can use, as ETag is often returned + // even on 304 Not Modified responses. + eTag := rsp.Header.Get("ETag") + + if rsp.StatusCode == http.StatusNotModified { + // Nothing has changed on the remote side + // since we last fetched, so there's nothing + // to do and we don't need to read the body. + rsp.Body.Close() + return &DereferenceDomainPermissionsResp{ + Unmodified: true, + ETag: eTag, + }, nil + } + + // Return the body + ETag to the caller. + return &DereferenceDomainPermissionsResp{ + Body: rsp.Body, + ETag: eTag, + }, nil } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 7f7e985fc..45d43ff18 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -78,6 +78,20 @@ type Transport interface { // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) + // DereferenceDomainPermissions dereferences the + // permissions list present at the given permSub's URI. + // + // If "force", then If-Modified-Since and If-None-Match + // headers will *NOT* be sent with the outgoing request. + // + // If err == nil and Unmodified == false, then it's up + // to the caller to close the returned io.ReadCloser. + DereferenceDomainPermissions( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + force bool, + ) (*DereferenceDomainPermissionsResp, error) + // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error) }