mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-12-23 02:32:12 +00:00
[bugfix] Fix emphasis being added to emoji shortcodes with markdown parsing (#856)
* fix underscored emoji shortcodes being emphasized * remove footnote parsing from md
This commit is contained in:
parent
429bb770e2
commit
00d38855d4
|
@ -24,6 +24,8 @@
|
||||||
|
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
"github.com/superseriousbusiness/gotosocial/internal/api/model"
|
"github.com/superseriousbusiness/gotosocial/internal/api/model"
|
||||||
|
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||||
|
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||||
)
|
)
|
||||||
|
|
||||||
type StatusCreateTestSuite struct {
|
type StatusCreateTestSuite struct {
|
||||||
|
@ -98,6 +100,45 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithHTMLEscapedQuot
|
||||||
suite.Equal("\"test\"", apiStatus.SpoilerText)
|
suite.Equal("\"test\"", apiStatus.SpoilerText)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithUnderscoreEmoji() {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// update the shortcode of the rainbow emoji to surround it in underscores
|
||||||
|
if err := suite.db.UpdateWhere(ctx, []db.Where{{Key: "shortcode", Value: "rainbow"}}, "shortcode", "_rainbow_", >smodel.Emoji{}); err != nil {
|
||||||
|
suite.FailNow(err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
creatingAccount := suite.testAccounts["local_account_1"]
|
||||||
|
creatingApplication := suite.testApplications["application_1"]
|
||||||
|
|
||||||
|
statusCreateForm := &model.AdvancedStatusCreateForm{
|
||||||
|
StatusCreateRequest: model.StatusCreateRequest{
|
||||||
|
Status: "poopoo peepee :_rainbow_:",
|
||||||
|
MediaIDs: []string{},
|
||||||
|
Poll: nil,
|
||||||
|
InReplyToID: "",
|
||||||
|
Sensitive: false,
|
||||||
|
Visibility: model.VisibilityPublic,
|
||||||
|
ScheduledAt: "",
|
||||||
|
Language: "en",
|
||||||
|
Format: model.StatusFormatMarkdown,
|
||||||
|
},
|
||||||
|
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
|
||||||
|
Federated: nil,
|
||||||
|
Boostable: nil,
|
||||||
|
Replyable: nil,
|
||||||
|
Likeable: nil,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm)
|
||||||
|
suite.NoError(err)
|
||||||
|
suite.NotNil(apiStatus)
|
||||||
|
|
||||||
|
suite.Equal("<p>poopoo peepee :_rainbow_:</p>", apiStatus.Content)
|
||||||
|
suite.NotEmpty(apiStatus.Emojis)
|
||||||
|
}
|
||||||
|
|
||||||
func TestStatusCreateTestSuite(t *testing.T) {
|
func TestStatusCreateTestSuite(t *testing.T) {
|
||||||
suite.Run(t, new(StatusCreateTestSuite))
|
suite.Run(t, new(StatusCreateTestSuite))
|
||||||
}
|
}
|
||||||
|
|
|
@ -302,7 +302,7 @@ func (p *processor) ProcessContent(ctx context.Context, form *apimodel.AdvancedS
|
||||||
case apimodel.StatusFormatPlain:
|
case apimodel.StatusFormatPlain:
|
||||||
formatted = p.formatter.FromPlain(ctx, form.Status, status.Mentions, status.Tags)
|
formatted = p.formatter.FromPlain(ctx, form.Status, status.Mentions, status.Tags)
|
||||||
case apimodel.StatusFormatMarkdown:
|
case apimodel.StatusFormatMarkdown:
|
||||||
formatted = p.formatter.FromMarkdown(ctx, form.Status, status.Mentions, status.Tags)
|
formatted = p.formatter.FromMarkdown(ctx, form.Status, status.Mentions, status.Tags, status.Emojis)
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("format %s not recognised as a valid status format", form.Format)
|
return fmt.Errorf("format %s not recognised as a valid status format", form.Format)
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ type Formatter interface {
|
||||||
// FromPlain parses an HTML text from a plaintext.
|
// FromPlain parses an HTML text from a plaintext.
|
||||||
FromPlain(ctx context.Context, plain string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
|
FromPlain(ctx context.Context, plain string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
|
||||||
// FromMarkdown parses an HTML text from a markdown-formatted text.
|
// FromMarkdown parses an HTML text from a markdown-formatted text.
|
||||||
FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
|
FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag, emojis []*gtsmodel.Emoji) string
|
||||||
|
|
||||||
// ReplaceTags takes a piece of text and a slice of tags, and returns the same text with the tags nicely formatted as hrefs.
|
// ReplaceTags takes a piece of text and a slice of tags, and returns the same text with the tags nicely formatted as hrefs.
|
||||||
ReplaceTags(ctx context.Context, in string, tags []*gtsmodel.Tag) string
|
ReplaceTags(ctx context.Context, in string, tags []*gtsmodel.Tag) string
|
||||||
|
|
|
@ -22,6 +22,7 @@
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/russross/blackfriday/v2"
|
"github.com/russross/blackfriday/v2"
|
||||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||||
|
@ -31,7 +32,7 @@
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
bfExtensions = blackfriday.CommonExtensions | blackfriday.HardLineBreak | blackfriday.Footnotes
|
bfExtensions = blackfriday.NoIntraEmphasis | blackfriday.FencedCode | blackfriday.Autolink | blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak
|
||||||
m *minify.M
|
m *minify.M
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -54,8 +55,7 @@ func (r *renderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool
|
||||||
html = r.f.ReplaceMentions(r.ctx, html, r.mentions)
|
html = r.f.ReplaceMentions(r.ctx, html, r.mentions)
|
||||||
|
|
||||||
// we don't have much recourse if this fails
|
// we don't have much recourse if this fails
|
||||||
_, err := io.WriteString(w, html)
|
if _, err := io.WriteString(w, html); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Errorf("error outputting markdown text: %s", err)
|
log.Errorf("error outputting markdown text: %s", err)
|
||||||
}
|
}
|
||||||
return status
|
return status
|
||||||
|
@ -63,7 +63,7 @@ func (r *renderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool
|
||||||
return r.HTMLRenderer.RenderNode(w, node, entering)
|
return r.HTMLRenderer.RenderNode(w, node, entering)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *formatter) FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string {
|
func (f *formatter) FromMarkdown(ctx context.Context, markdownText string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag, emojis []*gtsmodel.Emoji) string {
|
||||||
|
|
||||||
renderer := &renderer{
|
renderer := &renderer{
|
||||||
f: f,
|
f: f,
|
||||||
|
@ -75,11 +75,28 @@ func (f *formatter) FromMarkdown(ctx context.Context, md string, mentions []*gts
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
// parse markdown, use custom renderer to add hashtag/mention links
|
// Temporarily replace all found emoji shortcodes in the markdown text with
|
||||||
contentBytes := blackfriday.Run([]byte(md), blackfriday.WithExtensions(bfExtensions), blackfriday.WithRenderer(renderer))
|
// their ID so that they're not parsed as anything by the markdown parser -
|
||||||
|
// this fixes cases where emojis with some underscores in them are parsed as
|
||||||
|
// words with emphasis, eg `:_some_emoji:` becomes `:<em>some</em>emoji:`
|
||||||
|
//
|
||||||
|
// Since the IDs of the emojis are just uppercase letters + numbers they should
|
||||||
|
// be safe to pass through the markdown parser without unexpected effects.
|
||||||
|
for _, e := range emojis {
|
||||||
|
markdownText = strings.ReplaceAll(markdownText, ":"+e.Shortcode+":", ":"+e.ID+":")
|
||||||
|
}
|
||||||
|
|
||||||
// clean anything dangerous out of it
|
// parse markdown text into html, using custom renderer to add hashtag/mention links
|
||||||
content := SanitizeHTML(string(contentBytes))
|
htmlContentBytes := blackfriday.Run([]byte(markdownText), blackfriday.WithExtensions(bfExtensions), blackfriday.WithRenderer(renderer))
|
||||||
|
htmlContent := string(htmlContentBytes)
|
||||||
|
|
||||||
|
// Replace emoji IDs in the parsed html content with their shortcodes again
|
||||||
|
for _, e := range emojis {
|
||||||
|
htmlContent = strings.ReplaceAll(htmlContent, ":"+e.ID+":", ":"+e.Shortcode+":")
|
||||||
|
}
|
||||||
|
|
||||||
|
// clean anything dangerous out of the html
|
||||||
|
htmlContent = SanitizeHTML(htmlContent)
|
||||||
|
|
||||||
if m == nil {
|
if m == nil {
|
||||||
m = minify.New()
|
m = minify.New()
|
||||||
|
@ -89,7 +106,7 @@ func (f *formatter) FromMarkdown(ctx context.Context, md string, mentions []*gts
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
minified, err := m.String("text/html", content)
|
minified, err := m.String("text/html", htmlContent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("error minifying markdown text: %s", err)
|
log.Errorf("error minifying markdown text: %s", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -62,7 +62,7 @@
|
||||||
mdCodeBlockWithNewlines = "some code coming up\n\n```\n\n\n\n```\nthat was some code"
|
mdCodeBlockWithNewlines = "some code coming up\n\n```\n\n\n\n```\nthat was some code"
|
||||||
mdCodeBlockWithNewlinesExpected = "<p>some code coming up</p><pre><code>\n\n\n</code></pre><p>that was some code</p>"
|
mdCodeBlockWithNewlinesExpected = "<p>some code coming up</p><pre><code>\n\n\n</code></pre><p>that was some code</p>"
|
||||||
mdWithFootnote = "fox mulder,fbi.[^1]\n\n[^1]: federated bureau of investigation"
|
mdWithFootnote = "fox mulder,fbi.[^1]\n\n[^1]: federated bureau of investigation"
|
||||||
mdWithFootnoteExpected = "<p>fox mulder,fbi.<sup id=\"fnref:1\"><a href=\"#fn:1\" rel=\"nofollow noreferrer\">1</a></sup></p><div><hr><ol><li id=\"fn:1\">federated bureau of investigation<br></li></ol></div>"
|
mdWithFootnoteExpected = "<p>fox mulder,fbi.[^1]</p><p>[^1]: federated bureau of investigation</p>"
|
||||||
mdWithBlockQuote = "get ready, there's a block quote coming:\n\n>line1\n>line2\n>\n>line3\n\n"
|
mdWithBlockQuote = "get ready, there's a block quote coming:\n\n>line1\n>line2\n>\n>line3\n\n"
|
||||||
mdWithBlockQuoteExpected = "<p>get ready, there’s a block quote coming:</p><blockquote><p>line1<br>line2</p><p>line3</p></blockquote>"
|
mdWithBlockQuoteExpected = "<p>get ready, there’s a block quote coming:</p><blockquote><p>line1<br>line2</p><p>line3</p></blockquote>"
|
||||||
mdHashtagAndCodeBlock = "#Hashtag\n\n```\n#Hashtag\n```"
|
mdHashtagAndCodeBlock = "#Hashtag\n\n```\n#Hashtag\n```"
|
||||||
|
@ -76,22 +76,22 @@ type MarkdownTestSuite struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseSimple() {
|
func (suite *MarkdownTestSuite) TestParseSimple() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), simpleMarkdown, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), simpleMarkdown, nil, nil, nil)
|
||||||
suite.Equal(simpleMarkdownExpected, s)
|
suite.Equal(simpleMarkdownExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseWithCodeBlock() {
|
func (suite *MarkdownTestSuite) TestParseWithCodeBlock() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), withCodeBlock, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), withCodeBlock, nil, nil, nil)
|
||||||
suite.Equal(withCodeBlockExpected, s)
|
suite.Equal(withCodeBlockExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseWithInlineCode() {
|
func (suite *MarkdownTestSuite) TestParseWithInlineCode() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode, nil, nil, nil)
|
||||||
suite.Equal(withInlineCodeExpected, s)
|
suite.Equal(withInlineCodeExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseWithInlineCode2() {
|
func (suite *MarkdownTestSuite) TestParseWithInlineCode2() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode2, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode2, nil, nil, nil)
|
||||||
suite.Equal(withInlineCode2Expected, s)
|
suite.Equal(withInlineCode2Expected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -100,17 +100,17 @@ func (suite *MarkdownTestSuite) TestParseWithHashtag() {
|
||||||
suite.testTags["Hashtag"],
|
suite.testTags["Hashtag"],
|
||||||
}
|
}
|
||||||
|
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), withHashtag, nil, foundTags)
|
s := suite.formatter.FromMarkdown(context.Background(), withHashtag, nil, foundTags, nil)
|
||||||
suite.Equal(withHashtagExpected, s)
|
suite.Equal(withHashtagExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseWithHTML() {
|
func (suite *MarkdownTestSuite) TestParseWithHTML() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdWithHTML, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), mdWithHTML, nil, nil, nil)
|
||||||
suite.Equal(mdWithHTMLExpected, s)
|
suite.Equal(mdWithHTMLExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseWithCheekyHTML() {
|
func (suite *MarkdownTestSuite) TestParseWithCheekyHTML() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdWithCheekyHTML, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), mdWithCheekyHTML, nil, nil, nil)
|
||||||
suite.Equal(mdWithCheekyHTMLExpected, s)
|
suite.Equal(mdWithCheekyHTMLExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -118,36 +118,36 @@ func (suite *MarkdownTestSuite) TestParseWithHashtagInitial() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdWithHashtagInitial, nil, []*gtsmodel.Tag{
|
s := suite.formatter.FromMarkdown(context.Background(), mdWithHashtagInitial, nil, []*gtsmodel.Tag{
|
||||||
suite.testTags["Hashtag"],
|
suite.testTags["Hashtag"],
|
||||||
suite.testTags["welcome"],
|
suite.testTags["welcome"],
|
||||||
})
|
}, nil)
|
||||||
suite.Equal(mdWithHashtagInitialExpected, s)
|
suite.Equal(mdWithHashtagInitialExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseCodeBlockWithNewlines() {
|
func (suite *MarkdownTestSuite) TestParseCodeBlockWithNewlines() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdCodeBlockWithNewlines, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), mdCodeBlockWithNewlines, nil, nil, nil)
|
||||||
suite.Equal(mdCodeBlockWithNewlinesExpected, s)
|
suite.Equal(mdCodeBlockWithNewlinesExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseWithFootnote() {
|
func (suite *MarkdownTestSuite) TestParseWithFootnote() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdWithFootnote, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), mdWithFootnote, nil, nil, nil)
|
||||||
suite.Equal(mdWithFootnoteExpected, s)
|
suite.Equal(mdWithFootnoteExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseWithBlockquote() {
|
func (suite *MarkdownTestSuite) TestParseWithBlockquote() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdWithBlockQuote, nil, nil)
|
s := suite.formatter.FromMarkdown(context.Background(), mdWithBlockQuote, nil, nil, nil)
|
||||||
suite.Equal(mdWithBlockQuoteExpected, s)
|
suite.Equal(mdWithBlockQuoteExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseHashtagWithCodeBlock() {
|
func (suite *MarkdownTestSuite) TestParseHashtagWithCodeBlock() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdHashtagAndCodeBlock, nil, []*gtsmodel.Tag{
|
s := suite.formatter.FromMarkdown(context.Background(), mdHashtagAndCodeBlock, nil, []*gtsmodel.Tag{
|
||||||
suite.testTags["Hashtag"],
|
suite.testTags["Hashtag"],
|
||||||
})
|
}, nil)
|
||||||
suite.Equal(mdHashtagAndCodeBlockExpected, s)
|
suite.Equal(mdHashtagAndCodeBlockExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *MarkdownTestSuite) TestParseMentionWithCodeBlock() {
|
func (suite *MarkdownTestSuite) TestParseMentionWithCodeBlock() {
|
||||||
s := suite.formatter.FromMarkdown(context.Background(), mdMentionAndCodeBlock, []*gtsmodel.Mention{
|
s := suite.formatter.FromMarkdown(context.Background(), mdMentionAndCodeBlock, []*gtsmodel.Mention{
|
||||||
suite.testMentions["local_user_2_mention_zork"],
|
suite.testMentions["local_user_2_mention_zork"],
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
suite.Equal(mdMentionAndCodeBlockExpected, s)
|
suite.Equal(mdMentionAndCodeBlockExpected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue