[feature] Let accounts set default status format, and use this when processing new statuses (#739)

* add post_format to acct & use it when making post

* update swagger docs

* add status_format updating to frontend

* fix up tests

* post_format => status_format

* add status_format to account validation

docs/api/swagger.yaml

@@ -138,6 +138,10 @@ definitions:
         description: Whether new statuses should be marked sensitive by default.
         type: boolean
         x-go-name: Sensitive
+      status_format:
+        description: The default posting format for new statuses.
+        type: string
+        x-go-name: StatusFormat
     title: Source represents display or publishing preferences of user's own account.
     type: object
     x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model
@@ -1941,6 +1945,10 @@ definitions:
         description: Mark authored statuses as sensitive by default.
         type: boolean
         x-go-name: Sensitive
+      status_format:
+        description: Default format for authored statuses (plain or markdown).
+        type: string
+        x-go-name: StatusFormat
     title: UpdateSource is to be used specifically in an UpdateCredentialsRequest.
     type: object
     x-go-name: UpdateSource
@@ -2576,6 +2584,10 @@ paths:
         in: formData
         name: source[language]
         type: string
+      - description: Default format to use for authored statuses (plain or markdown).
+        in: formData
+        name: source[status_format]
+        type: string
       produces:
       - application/json
       responses:

internal/api/client/account/accountupdate.go

@@ -88,6 +88,10 @@
 //   in: formData
 //   description: Default language to use for authored statuses (ISO 6391).
 //   type: string
+// - name: source[status_format]
+//   in: formData
+//   description: Default format to use for authored statuses (plain or markdown).
+//   type: string
 //
 // security:
 // - OAuth2 Bearer:
@@ -163,6 +167,10 @@ func parseUpdateAccountForm(c *gin.Context) (*model.UpdateCredentialsRequest, er
 		form.Source.Language = &language
 	}
 
+	if statusFormat, ok := sourceMap["status_format"]; ok {
+		form.Source.StatusFormat = &statusFormat
+	}
+
 	if form == nil ||
 		(form.Discoverable == nil &&
 			form.Bot == nil &&
@@ -174,6 +182,7 @@ func parseUpdateAccountForm(c *gin.Context) (*model.UpdateCredentialsRequest, er
 			form.Source.Privacy == nil &&
 			form.Source.Sensitive == nil &&
 			form.Source.Language == nil &&
+			form.Source.StatusFormat == nil &&
 			form.FieldsAttributes == nil) {
 		return nil, errors.New("empty form submitted")
 	}

internal/api/client/account/accountupdate_test.go

@@ -362,6 +362,81 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerUpd
 	suite.True(apimodelAccount.Locked)
 }
 
+func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerUpdateStatusFormatOK() {
+	// set up the request
+	// we're updating the language of zork
+	requestBody, w, err := testrig.CreateMultipartFormData(
+		"", "",
+		map[string]string{
+			"source[status_format]": "markdown",
+		})
+	if err != nil {
+		panic(err)
+	}
+	bodyBytes := requestBody.Bytes()
+	recorder := httptest.NewRecorder()
+	ctx := suite.newContext(recorder, http.MethodPatch, bodyBytes, account.UpdateCredentialsPath, w.FormDataContentType())
+
+	// call the handler
+	suite.accountModule.AccountUpdateCredentialsPATCHHandler(ctx)
+
+	// 1. we should have OK because our request was valid
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// 2. we should have no error message in the result body
+	result := recorder.Result()
+	defer result.Body.Close()
+
+	// check the response
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+
+	// unmarshal the returned account
+	apimodelAccount := &apimodel.Account{}
+	err = json.Unmarshal(b, apimodelAccount)
+	suite.NoError(err)
+
+	// check the returned api model account
+	// fields should be updated
+	suite.Equal("markdown", apimodelAccount.Source.StatusFormat)
+
+	dbAccount, err := suite.db.GetAccountByID(context.Background(), suite.testAccounts["local_account_1"].ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+	suite.Equal(dbAccount.StatusFormat, "markdown")
+}
+
+func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerUpdateStatusFormatBad() {
+	// set up the request
+	// we're updating the language of zork
+	requestBody, w, err := testrig.CreateMultipartFormData(
+		"", "",
+		map[string]string{
+			"source[status_format]": "peepeepoopoo",
+		})
+	if err != nil {
+		panic(err)
+	}
+	bodyBytes := requestBody.Bytes()
+	recorder := httptest.NewRecorder()
+	ctx := suite.newContext(recorder, http.MethodPatch, bodyBytes, account.UpdateCredentialsPath, w.FormDataContentType())
+
+	// call the handler
+	suite.accountModule.AccountUpdateCredentialsPATCHHandler(ctx)
+
+	suite.Equal(http.StatusBadRequest, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+
+	// check the response
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+
+	suite.Equal(`{"error":"Bad Request: status format 'peepeepoopoo' was not recognized, valid options are 'plain', 'markdown'"}`, string(b))
+}
+
 func TestAccountUpdateTestSuite(t *testing.T) {
 	suite.Run(t, new(AccountUpdateTestSuite))
 }

internal/api/client/status/statuscreate_test.go

@@ -41,13 +41,11 @@ type StatusCreateTestSuite struct {
 	StatusStandardTestSuite
 }
 
-var statusWithLinksAndTags = `#test alright, should be able to post #links with fragments in them now, let's see........
+const (
-
+	statusWithLinksAndTags = "#test alright, should be able to post #links with fragments in them now, let's see........\n\nhttps://docs.gotosocial.org/en/latest/user_guide/posts/#links\n\n#gotosocial\n\n(tobi remember to pull the docker image challenge)"
-https://docs.gotosocial.org/en/latest/user_guide/posts/#links
+	statusMarkdown         = "# Title\n\n## Smaller title\n\nThis is a post written in [markdown](https://www.markdownguide.org/)\n\n<img src=\"https://d33wubrfki0l68.cloudfront.net/f1f475a6fda1c2c4be4cac04033db5c3293032b4/513a4/assets/images/markdown-mark-white.svg\"/>"
-
+	statusMarkdownExpected = "<h1>Title</h1>\n\n<h2>Smaller title</h2>\n\n<p>This is a post written in <a href=\"https://www.markdownguide.org/\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">markdown</a></p>\n\n<p><img src=\"https://d33wubrfki0l68.cloudfront.net/f1f475a6fda1c2c4be4cac04033db5c3293032b4/513a4/assets/images/markdown-mark-white.svg\" crossorigin=\"anonymous\"/></p>\n"
-#gotosocial
+)
-
-(tobi remember to pull the docker image challenge)`
 
 // Post a new status with some custom visibility settings
 func (suite *StatusCreateTestSuite) TestPostNewStatus() {
@@ -104,6 +102,49 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 	suite.Equal(statusReply.Account.ID, gtsTag.FirstSeenFromAccountID)
 }
 
+func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() {
+	// set default post language of account 1 to markdown
+	testAccount := suite.testAccounts["local_account_1"]
+	testAccount.StatusFormat = "markdown"
+
+	a, err := suite.db.UpdateAccount(context.Background(), testAccount)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+	suite.Equal(a.StatusFormat, "markdown")
+
+	t := suite.testTokens["local_account_1"]
+	oauthToken := oauth.DBTokenToToken(t)
+
+	recorder := httptest.NewRecorder()
+	ctx, _ := testrig.CreateGinTestContext(recorder, nil)
+	ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+	ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
+	ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
+	ctx.Set(oauth.SessionAuthorizedAccount, a)
+
+	ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil)
+	ctx.Request.Header.Set("accept", "application/json")
+	ctx.Request.Form = url.Values{
+		"status":     {statusMarkdown},
+		"visibility": {string(model.VisibilityPublic)},
+	}
+	suite.statusModule.StatusCreatePOSTHandler(ctx)
+
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+
+	statusReply := &model.Status{}
+	err = json.Unmarshal(b, statusReply)
+	suite.NoError(err)
+
+	suite.Equal(statusMarkdownExpected, statusReply.Content)
+}
+
 // mention an account that is not yet known to the instance -- it should be looked up and put in the db
 func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
 	// first remove remote account 1 from the database so it gets looked up again

internal/api/model/account.go

@@ -163,6 +163,8 @@ type UpdateSource struct {
 	Sensitive *bool `form:"sensitive" json:"sensitive" xml:"sensitive"`
 	// Default language to use for authored statuses. (ISO 6391)
 	Language *string `form:"language" json:"language" xml:"language"`
+	// Default format for authored statuses (plain or markdown).
+	StatusFormat *string `form:"status_format" json:"status_format" xml:"status_format"`
 }
 
 // UpdateField is to be used specifically in an UpdateCredentialsRequest.

internal/api/model/source.go

@@ -31,6 +31,8 @@ type Source struct {
 	Sensitive bool `json:"sensitive,omitempty"`
 	// The default posting language for new statuses.
 	Language string `json:"language,omitempty"`
+	// The default posting format for new statuses.
+	StatusFormat string `json:"status_format"`
 	// Profile bio.
 	Note string `json:"note"`
 	// Metadata about the account.

internal/api/model/status.go

@@ -181,8 +181,8 @@ type StatusCreateRequest struct {
 	Language string `form:"language" json:"language" xml:"language"`
 	// Format to use when parsing this status.
 	// enum:
-	// - markdown
 	// - plain
+	// - markdown
 	// in: formData
 	Format StatusFormat `form:"format" json:"format" xml:"format"`
 }
@@ -245,11 +245,9 @@ type AdvancedVisibilityFlagsForm struct {
 // example: plain
 type StatusFormat string
 
-// StatusFormatPlain expects a plaintext status which will then be formatted into html.
+// Format to use when parsing submitted status into an html-formatted status
-const StatusFormatPlain StatusFormat = "plain"
+const (
-
+	StatusFormatPlain    StatusFormat = "plain"
-// StatusFormatMarkdown expects a markdown formatted status, which will then be formatted into html.
+	StatusFormatMarkdown StatusFormat = "markdown"
-const StatusFormatMarkdown StatusFormat = "markdown"
+	StatusFormatDefault  StatusFormat = StatusFormatPlain
-
+)
-// StatusFormatDefault is the format that should be used when nothing else is specified.
-const StatusFormatDefault StatusFormat = StatusFormatPlain

internal/cache/account.go

@@ -114,6 +114,7 @@ func copyAccount(account *gtsmodel.Account) *gtsmodel.Account {
 		Privacy:                 account.Privacy,
 		Sensitive:               account.Sensitive,
 		Language:                account.Language,
+		StatusFormat:            account.StatusFormat,
 		URI:                     account.URI,
 		URL:                     account.URL,
 		LastWebfingeredAt:       account.LastWebfingeredAt,

internal/db/bundb/migrations/20220804120132_account_default_post_format.go

@@ -0,0 +1,46 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package migrations
+
+import (
+	"context"
+	"strings"
+
+	"github.com/uptrace/bun"
+)
+
+func init() {
+	up := func(ctx context.Context, db *bun.DB) error {
+		_, err := db.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? TEXT", bun.Ident("accounts"), bun.Ident("status_format"))
+		if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
+			return err
+		}
+		return nil
+	}
+
+	down := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			return nil
+		})
+	}
+
+	if err := Migrations.Register(up, down); err != nil {
+		panic(err)
+	}
+}

internal/gtsmodel/account.go

@@ -54,6 +54,7 @@ type Account struct {
 	Privacy                 Visibility       `validate:"required_without=Domain,omitempty,oneof=public unlocked followers_only mutuals_only direct" bun:",nullzero"` // Default post privacy for this account
 	Sensitive               bool             `validate:"-" bun:",default:false"`                                                                                     // Set posts from this account to sensitive by default?
 	Language                string           `validate:"omitempty,bcp47_language_tag" bun:",nullzero,notnull,default:'en'"`                                          // What language does this account post in?
+	StatusFormat            string           `validate:"required_without=Domain,omitempty,oneof=plain markdown" bun:",nullzero"`                                     // What is the default format for statuses posted by this account (only for local accounts).
 	URI                     string           `validate:"required,url" bun:",nullzero,notnull,unique"`                                                                // ActivityPub URI for this account.
 	URL                     string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // Web URL for this account's profile
 	LastWebfingeredAt       time.Time        `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"`                                                       // Last time this account was refreshed/located with webfinger.

internal/processing/account/update.go

@@ -114,6 +114,14 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form
 			privacy := p.tc.APIVisToVis(apimodel.Visibility(*form.Source.Privacy))
 			account.Privacy = privacy
 		}
+
+		if form.Source.StatusFormat != nil {
+			if err := validate.StatusFormat(*form.Source.StatusFormat); err != nil {
+				return nil, gtserror.NewErrorBadRequest(err, err.Error())
+			}
+
+			account.StatusFormat = *form.Source.StatusFormat
+		}
 	}
 
 	updatedAccount, err := p.db.UpdateAccount(ctx, account)

internal/processing/status/util.go

@@ -23,6 +23,7 @@
 	"errors"
 	"fmt"
 
+	"github.com/superseriousbusiness/gotosocial/internal/api/model"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
@@ -269,9 +270,21 @@ func (p *processor) ProcessContent(ctx context.Context, form *apimodel.AdvancedS
 		return nil
 	}
 
-	// if format wasn't specified we should set the default
+	// if format wasn't specified we should try to figure out what format this user prefers
 	if form.Format == "" {
-		form.Format = apimodel.StatusFormatDefault
+		acct, err := p.db.GetAccountByID(ctx, accountID)
+		if err != nil {
+			return fmt.Errorf("error processing new content: couldn't retrieve account from db to check post format: %s", err)
+		}
+
+		switch acct.StatusFormat {
+		case "plain":
+			form.Format = model.StatusFormatPlain
+		case "markdown":
+			form.Format = model.StatusFormatMarkdown
+		default:
+			form.Format = model.StatusFormatDefault
+		}
 	}
 
 	// parse content out of the status depending on what format has been submitted

internal/text/formatter.go

@@ -27,10 +27,10 @@
 
 // Formatter wraps some logic and functions for parsing statuses and other text input into nice html.
 type Formatter interface {
-	// FromMarkdown parses an HTML text from a markdown-formatted text.
-	FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
 	// FromPlain parses an HTML text from a plaintext.
 	FromPlain(ctx context.Context, plain string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
+	// FromMarkdown parses an HTML text from a markdown-formatted text.
+	FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
 
 	// ReplaceTags takes a piece of text and a slice of tags, and returns the same text with the tags nicely formatted as hrefs.
 	ReplaceTags(ctx context.Context, in string, tags []*gtsmodel.Tag) string

internal/typeutils/internaltofrontend.go

@@ -53,10 +53,16 @@ func (c *converter) AccountToAPIAccountSensitive(ctx context.Context, a *gtsmode
 		frc = len(frs)
 	}
 
+	statusFormat := string(model.StatusFormatDefault)
+	if a.StatusFormat != "" {
+		statusFormat = a.StatusFormat
+	}
+
 	apiAccount.Source = &model.Source{
 		Privacy:             c.VisToAPIVis(ctx, a.Privacy),
 		Sensitive:           a.Sensitive,
 		Language:            a.Language,
+		StatusFormat:        statusFormat,
 		Note:                a.NoteRaw,
 		Fields:              apiAccount.Fields,
 		FollowRequestsCount: frc,

internal/typeutils/internaltofrontend_test.go

@@ -43,6 +43,17 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontend() {
 	suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[]}`, string(b))
 }
 
+func (suite *InternalToFrontendTestSuite) TestAccountToFrontendSensitive() {
+	testAccount := suite.testAccounts["local_account_1"] // take zork for this test
+	apiAccount, err := suite.typeconverter.AccountToAPIAccountSensitive(context.Background(), testAccount)
+	suite.NoError(err)
+	suite.NotNil(apiAccount)
+
+	b, err := json.Marshal(apiAccount)
+	suite.NoError(err)
+	suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"source":{"privacy":"public","language":"en","status_format":"plain","note":"hey yo this is my profile!","fields":[]}}`, string(b))
+}
+
 func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
 	testStatus := suite.testStatuses["admin_account_status_1"]
 	requestingAccount := suite.testAccounts["local_account_1"]

internal/validate/account_test.go

@@ -62,6 +62,7 @@ func happyAccount() *gtsmodel.Account {
 		Privacy:                 gtsmodel.VisibilityPublic,
 		Sensitive:               false,
 		Language:                "en",
+		StatusFormat:            "plain",
 		URI:                     "http://localhost:8080/users/the_mighty_zork",
 		URL:                     "http://localhost:8080/@the_mighty_zork",
 		LastWebfingeredAt:       time.Time{},

internal/validate/formvalidation.go

@@ -144,7 +144,19 @@ func Privacy(privacy string) error {
 	case apimodel.VisibilityDirect, apimodel.VisibilityMutualsOnly, apimodel.VisibilityPrivate, apimodel.VisibilityPublic, apimodel.VisibilityUnlisted:
 		return nil
 	}
-	return fmt.Errorf("privacy %s was not recognized", privacy)
+	return fmt.Errorf("privacy '%s' was not recognized, valid options are 'direct', 'mutuals_only', 'private', 'public', 'unlisted'", privacy)
+}
+
+// StatusFormat checks that the desired status format setting is valid.
+func StatusFormat(statusFormat string) error {
+	if statusFormat == "" {
+		return fmt.Errorf("empty string for status format not allowed")
+	}
+	switch apimodel.StatusFormat(statusFormat) {
+	case apimodel.StatusFormatPlain, apimodel.StatusFormatMarkdown:
+		return nil
+	}
+	return fmt.Errorf("status format '%s' was not recognized, valid options are 'plain', 'markdown'", statusFormat)
 }
 
 // EmojiShortcode just runs the given shortcode through the regular expression