From c84384e6608368a13a774d6d33a8cc32da7cf209 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:21:17 +0200 Subject: [PATCH] [bugfix] html escape special characters in text instead of totally removing them (#719) * remove minify dependency * tidy up some tests * remove pre + postformat funcs * rework sanitization + formatting * update tests * add some more markdown tests --- README.md | 1 - go.mod | 2 - go.sum | 10 - .../api/client/status/statuscreate_test.go | 6 +- internal/processing/account/update_test.go | 13 +- internal/processing/status/util_test.go | 116 +- internal/text/common.go | 33 - internal/text/common_test.go | 46 +- internal/text/link_test.go | 20 +- internal/text/markdown.go | 6 +- internal/text/markdown_test.go | 32 +- internal/text/minify.go | 39 - internal/text/plain.go | 10 +- internal/text/plain_test.go | 36 +- internal/text/sanitize.go | 7 +- .../tdewolff/minify/v2/.gitattributes | 2 - .../github.com/tdewolff/minify/v2/.gitignore | 24 - .../tdewolff/minify/v2/.golangci.yml | 16 - .../github.com/tdewolff/minify/v2/Dockerfile | 14 - vendor/github.com/tdewolff/minify/v2/LICENSE | 22 - vendor/github.com/tdewolff/minify/v2/Makefile | 58 - .../github.com/tdewolff/minify/v2/README.md | 724 --------- .../github.com/tdewolff/minify/v2/common.go | 515 ------- .../tdewolff/minify/v2/html/buffer.go | 137 -- .../tdewolff/minify/v2/html/hash.go | 543 ------- .../tdewolff/minify/v2/html/html.go | 511 ------- .../tdewolff/minify/v2/html/table.go | 1346 ----------------- .../github.com/tdewolff/minify/v2/minify.go | 371 ----- .../tdewolff/parse/v2/.gitattributes | 1 - .../github.com/tdewolff/parse/v2/.gitignore | 5 - .../tdewolff/parse/v2/.golangci.yml | 16 - .../github.com/tdewolff/parse/v2/LICENSE.md | 22 - vendor/github.com/tdewolff/parse/v2/README.md | 64 - .../tdewolff/parse/v2/buffer/buffer.go | 12 - .../tdewolff/parse/v2/buffer/lexer.go | 164 -- .../tdewolff/parse/v2/buffer/reader.go | 44 - .../tdewolff/parse/v2/buffer/streamlexer.go | 223 --- .../tdewolff/parse/v2/buffer/writer.go | 65 - vendor/github.com/tdewolff/parse/v2/common.go | 237 --- vendor/github.com/tdewolff/parse/v2/error.go | 47 - .../tdewolff/parse/v2/html/README.md | 98 -- .../github.com/tdewolff/parse/v2/html/hash.go | 81 - .../github.com/tdewolff/parse/v2/html/lex.go | 494 ------ .../github.com/tdewolff/parse/v2/html/util.go | 103 -- vendor/github.com/tdewolff/parse/v2/input.go | 173 --- .../github.com/tdewolff/parse/v2/position.go | 95 -- .../tdewolff/parse/v2/strconv/float.go | 257 ---- .../tdewolff/parse/v2/strconv/int.go | 108 -- .../tdewolff/parse/v2/strconv/price.go | 83 - vendor/github.com/tdewolff/parse/v2/util.go | 486 ------ vendor/modules.txt | 10 - 51 files changed, 129 insertions(+), 7419 deletions(-) delete mode 100644 internal/text/minify.go delete mode 100644 vendor/github.com/tdewolff/minify/v2/.gitattributes delete mode 100644 vendor/github.com/tdewolff/minify/v2/.gitignore delete mode 100644 vendor/github.com/tdewolff/minify/v2/.golangci.yml delete mode 100644 vendor/github.com/tdewolff/minify/v2/Dockerfile delete mode 100644 vendor/github.com/tdewolff/minify/v2/LICENSE delete mode 100644 vendor/github.com/tdewolff/minify/v2/Makefile delete mode 100644 vendor/github.com/tdewolff/minify/v2/README.md delete mode 100644 vendor/github.com/tdewolff/minify/v2/common.go delete mode 100644 vendor/github.com/tdewolff/minify/v2/html/buffer.go delete mode 100644 vendor/github.com/tdewolff/minify/v2/html/hash.go delete mode 100644 vendor/github.com/tdewolff/minify/v2/html/html.go delete mode 100644 vendor/github.com/tdewolff/minify/v2/html/table.go delete mode 100644 vendor/github.com/tdewolff/minify/v2/minify.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/.gitattributes delete mode 100644 vendor/github.com/tdewolff/parse/v2/.gitignore delete mode 100644 vendor/github.com/tdewolff/parse/v2/.golangci.yml delete mode 100644 vendor/github.com/tdewolff/parse/v2/LICENSE.md delete mode 100644 vendor/github.com/tdewolff/parse/v2/README.md delete mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/buffer.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/lexer.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/reader.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/streamlexer.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/writer.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/common.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/error.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/html/README.md delete mode 100644 vendor/github.com/tdewolff/parse/v2/html/hash.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/html/lex.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/html/util.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/input.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/position.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/strconv/float.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/strconv/int.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/strconv/price.go delete mode 100644 vendor/github.com/tdewolff/parse/v2/util.go diff --git a/README.md b/README.md index c0107eaac..e3b5bb70d 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,6 @@ The following libraries and frameworks are used by GoToSocial, with gratitude - [superseriousbusiness/activity](https://github.com/superseriousbusiness/activity) forked from [go-fed/activity](https://github.com/go-fed/activity); Golang ActivityPub/ActivityStreams library. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). - [superseriousbusiness/oauth2](https://github.com/superseriousbusiness/oauth2) forked from [go-oauth2/oauth2](https://github.com/go-oauth2/oauth2); oauth server framework and token handling. [MIT License](https://spdx.org/licenses/MIT.html). - [go-swagger/go-swagger](https://github.com/go-swagger/go-swagger); Swagger OpenAPI spec generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). -- [tdewolff/minify](https://github.com/tdewolff/minify); HTML minification. [MIT License](https://spdx.org/licenses/MIT.html). - [uptrace/bun](https://github.com/uptrace/bun); database ORM. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html). - [wagslane/go-password-validator](https://github.com/wagslane/go-password-validator); password strength validation. [MIT License](https://spdx.org/licenses/MIT.html). diff --git a/go.mod b/go.mod index 3d963019d..445445bad 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,6 @@ require ( github.com/superseriousbusiness/activity v1.1.0-gts github.com/superseriousbusiness/exif-terminator v0.3.0 github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB - github.com/tdewolff/minify/v2 v2.11.2 github.com/uptrace/bun v1.1.3 github.com/uptrace/bun/dialect/pgdialect v1.1.3 github.com/uptrace/bun/dialect/sqlitedialect v1.1.3 @@ -121,7 +120,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe // indirect - github.com/tdewolff/parse/v2 v2.5.29 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/ugorji/go/codec v1.2.7 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect diff --git a/go.sum b/go.sum index dda597dca..e1e18ba73 100644 --- a/go.sum +++ b/go.sum @@ -101,7 +101,6 @@ github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd3 github.com/buckket/go-blurhash v1.1.0 h1:X5M6r0LIvwdvKiUtiNcRL2YlmOfMzYobI3VCKCZc9Do= github.com/buckket/go-blurhash v1.1.0/go.mod h1:aT2iqo5W9vu9GpyoLErKfTHwgODsZp3bQfXjXJUxNb8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -121,7 +120,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE= github.com/dsoprea/go-exif/v2 v2.0.0-20200321225314-640175a69fe4/go.mod h1:Lm2lMM2zx8p4a34ZemkaUV95AnMl4ZvLbCUbwOvLC2E= github.com/dsoprea/go-exif/v3 v3.0.0-20200717053412-08f1b6708903/go.mod h1:0nsO1ce0mh5czxGeLo4+OCZ/C6Eo6ZlMWsz7rH/Gxv8= github.com/dsoprea/go-exif/v3 v3.0.0-20210428042052-dca55bf8ca15/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk= @@ -154,7 +152,6 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.3/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8= @@ -398,7 +395,6 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -518,12 +514,6 @@ github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430 github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe/go.mod h1:gH4P6gN1V+wmIw5o97KGaa1RgXB/tVpC2UNzijhg3E4= github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB h1:PtW2w6budTvRV2J5QAoSvThTHBuvh8t/+BXIZFAaBSc= github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB/go.mod h1:uYC/W92oVRJ49Vh1GcvTqpeFqHi+Ovrl2sMllQWRAEo= -github.com/tdewolff/minify/v2 v2.11.2 h1:PpaPWhNlMVjkAKaOj0bbPv6KCVnrm8jbVwG7OtSdAqw= -github.com/tdewolff/minify/v2 v2.11.2/go.mod h1:NxozhBtgUVypPLzQdV96wkIu9J9vAiVmBcKhfC2zMfg= -github.com/tdewolff/parse/v2 v2.5.29 h1:Uf0OtZL9YaUXTuHEOitdo9lD90P0XTwCjZi+KbGChuM= -github.com/tdewolff/parse/v2 v2.5.29/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= -github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4= -github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E= github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo= diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go index c6c9b4dab..f83ac8e8f 100644 --- a/internal/api/client/status/statuscreate_test.go +++ b/internal/api/client/status/statuscreate_test.go @@ -178,7 +178,7 @@ func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() { err = json.Unmarshal(b, statusReply) suite.NoError(err) - suite.Equal("
#test alright, should be able to post #links with fragments in them now, let's see........
docs.gotosocial.org/en/latest/user_guide/posts/#links
#gotosocial
(tobi remember to pull the docker image challenge)
#test alright, should be able to post #links with fragments in them now, let's see........
docs.gotosocial.org/en/latest/user_guide/posts/#links
#gotosocial
(tobi remember to pull the docker image challenge)
here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:
here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:
here's an image attachment
", statusResponse.Content) + suite.Equal("here's an image attachment
", statusResponse.Content) suite.False(statusResponse.Sensitive) suite.Equal(model.VisibilityPublic, statusResponse.Visibility) diff --git a/internal/processing/account/update_test.go b/internal/processing/account/update_test.go index 582dc82e9..7e4ca818e 100644 --- a/internal/processing/account/update_test.go +++ b/internal/processing/account/update_test.go @@ -73,13 +73,12 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateSimple() { func (suite *AccountUpdateTestSuite) TestAccountUpdateWithMention() { testAccount := suite.testAccounts["local_account_1"] - locked := true - displayName := "new display name" - note := `#hello here i am! - -go check out @1happyturtle, they have a cool account! -` - noteExpected := `#hello here i am!
go check out @1happyturtle, they have a cool account!
#hello here i am!
go check out @1happyturtle, they have a cool account!
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
#hashTAG
Another test @foss_satan
#Hashtag
#hashTAG
Another test @foss_satan
#Hashtag
#hashTAG
Here’s a simple text in markdown.
\n\nHere’s a link.
\n" - withCodeBlockExpected = "Below is some JSON.
\n\n{\n "key": "value",\n "another_key": [\n "value1",\n "value2"\n ]\n}\n
\n\nthat was some JSON :)
\n" - withInlineCode = "`Nobody tells you about theSECRET CODE
, do they?`"
- withInlineCodeExpected = "Nobody tells you about the <code><del>SECRET CODE</del></code>, do they?
, do they?`"
- withInlineCode2Expected = "Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?
\n"
- withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
- withHashtagExpected = "Title
\n\nhere’s a simple status that uses hashtag #Hashtag!
\n"
+ simpleMarkdown = "# Title\n\nHere's a simple text in markdown.\n\nHere's a [link](https://example.org)."
+ simpleMarkdownExpected = "Title
\n\nHere’s a simple text in markdown.
\n\nHere’s a link.
\n"
+ withCodeBlockExpected = "Title
\n\nBelow is some JSON.
\n\n{\n "key": "value",\n "another_key": [\n "value1",\n "value2"\n ]\n}\n
\n\nthat was some JSON :)
\n"
+ withInlineCode = "`Nobody tells you about the SECRET CODE
, do they?`"
+ withInlineCodeExpected = "Nobody tells you about the <code><del>SECRET CODE</del></code>, do they?
\n"
+ withInlineCode2 = "`Nobody tells you about the
, do they?`"
+ withInlineCode2Expected = "Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?
\n"
+ withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
+ withHashtagExpected = "Title
\n\nhere’s a simple status that uses hashtag #Hashtag!
\n"
+ mdWithHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a link.\n\nHere's an image: "
+ mdWithHTMLExpected = "Title
\n\nHere’s a simple text in markdown.
\n\nHere’s a link.
\n\nHere’s an image:
\n"
+ mdWithCheekyHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a cheeky little script: "
+ mdWithCheekyHTMLExpected = "Title
\n\nHere’s a simple text in markdown.
\n\nHere’s a cheeky little script:
\n"
)
type MarkdownTestSuite struct {
@@ -88,6 +92,16 @@ func (suite *MarkdownTestSuite) TestParseWithHashtag() {
suite.Equal(withHashtagExpected, s)
}
+func (suite *MarkdownTestSuite) TestParseWithHTML() {
+ s := suite.formatter.FromMarkdown(context.Background(), mdWithHTML, nil, nil)
+ suite.Equal(mdWithHTMLExpected, s)
+}
+
+func (suite *MarkdownTestSuite) TestParseWithCheekyHTML() {
+ s := suite.formatter.FromMarkdown(context.Background(), mdWithCheekyHTML, nil, nil)
+ suite.Equal(mdWithCheekyHTMLExpected, s)
+}
+
func TestMarkdownTestSuite(t *testing.T) {
suite.Run(t, new(MarkdownTestSuite))
}
diff --git a/internal/text/minify.go b/internal/text/minify.go
deleted file mode 100644
index e2515b9a4..000000000
--- a/internal/text/minify.go
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- GoToSocial
- Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with this program. If not, see .
-*/
-
-package text
-
-import (
- "github.com/tdewolff/minify/v2"
- "github.com/tdewolff/minify/v2/html"
-)
-
-var m *minify.M
-
-// MinifyHTML runs html through a minifier, reducing it in size.
-func MinifyHTML(in string) (string, error) {
- if m == nil {
- m = minify.New()
- m.Add("text/html", &html.Minifier{
- KeepQuotes: true,
- KeepEndTags: true,
- KeepDocumentTags: true,
- })
- }
- return m.String("text/html", in)
-}
diff --git a/internal/text/plain.go b/internal/text/plain.go
index bc10d1b67..3daea5686 100644
--- a/internal/text/plain.go
+++ b/internal/text/plain.go
@@ -20,6 +20,7 @@
import (
"context"
+ "html"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -32,10 +33,11 @@
)
func (f *formatter) FromPlain(ctx context.Context, plain string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string {
- content := preformat(plain)
+ // trim any crap
+ content := strings.TrimSpace(plain)
- // sanitize any html elements
- content = removeHTML(content)
+ // clean 'er up
+ content = html.EscapeString(content)
// format links nicely
content = f.ReplaceLinks(ctx, content)
@@ -52,5 +54,5 @@ func (f *formatter) FromPlain(ctx context.Context, plain string, mentions []*gts
// wrap the whole thing in a pee
content = `` + content + `
`
- return postformat(content)
+ return SanitizeHTML(content)
}
diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go
index 2b7b50d5e..cd82e0d1b 100644
--- a/internal/text/plain_test.go
+++ b/internal/text/plain_test.go
@@ -20,27 +20,21 @@
import (
"context"
- "fmt"
"testing"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
const (
- simple = "this is a plain and simple status"
- simpleExpected = "this is a plain and simple status
"
-
- withTag = "here's a simple status that uses hashtag #welcome!"
- withTagExpected = "here's a simple status that uses hashtag #welcome!
"
-
- moreComplex = `Another test @foss_satan@fossbros-anonymous.io
-
-#Hashtag
-
-Text`
- moreComplexFull = "Another test @foss_satan
#Hashtag
Text
"
+ simple = "this is a plain and simple status"
+ simpleExpected = "this is a plain and simple status
"
+ withTag = "here's a simple status that uses hashtag #welcome!"
+ withTagExpected = "here's a simple status that uses hashtag #welcome!
"
+ withHTML = "blah this should just be html escaped blah"
+ withHTMLExpected = "<div>blah this should just be html escaped blah</div>
"
+ moreComplex = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\nText"
+ moreComplexFull = "Another test @foss_satan
#Hashtag
Text
"
)
type PlainTestSuite struct {
@@ -49,7 +43,7 @@ type PlainTestSuite struct {
func (suite *PlainTestSuite) TestParseSimple() {
f := suite.formatter.FromPlain(context.Background(), simple, nil, nil)
- assert.Equal(suite.T(), simpleExpected, f)
+ suite.Equal(simpleExpected, f)
}
func (suite *PlainTestSuite) TestParseWithTag() {
@@ -58,7 +52,12 @@ func (suite *PlainTestSuite) TestParseWithTag() {
}
f := suite.formatter.FromPlain(context.Background(), withTag, nil, foundTags)
- assert.Equal(suite.T(), withTagExpected, f)
+ suite.Equal(withTagExpected, f)
+}
+
+func (suite *PlainTestSuite) TestParseWithHTML() {
+ f := suite.formatter.FromPlain(context.Background(), withHTML, nil, nil)
+ suite.Equal(withHTMLExpected, f)
}
func (suite *PlainTestSuite) TestParseMoreComplex() {
@@ -71,10 +70,7 @@ func (suite *PlainTestSuite) TestParseMoreComplex() {
}
f := suite.formatter.FromPlain(context.Background(), moreComplex, foundMentions, foundTags)
-
- fmt.Println(f)
-
- assert.Equal(suite.T(), moreComplexFull, f)
+ suite.Equal(moreComplexFull, f)
}
func TestPlainTestSuite(t *testing.T) {
diff --git a/internal/text/sanitize.go b/internal/text/sanitize.go
index d4faabbb1..96b7ef994 100644
--- a/internal/text/sanitize.go
+++ b/internal/text/sanitize.go
@@ -19,7 +19,9 @@
package text
import (
+ "html"
"regexp"
+ "strings"
"github.com/microcosm-cc/bluemonday"
)
@@ -59,7 +61,8 @@ func SanitizeHTML(in string) string {
// SanitizePlaintext runs text through basic sanitization. This removes
// any html elements that were in the string, and returns clean plaintext.
func SanitizePlaintext(in string) string {
- content := preformat(in)
+ content := html.UnescapeString(in)
content = removeHTML(content)
- return postformat(content)
+ content = html.UnescapeString(content)
+ return strings.TrimSpace(content)
}
diff --git a/vendor/github.com/tdewolff/minify/v2/.gitattributes b/vendor/github.com/tdewolff/minify/v2/.gitattributes
deleted file mode 100644
index 16a3a8b06..000000000
--- a/vendor/github.com/tdewolff/minify/v2/.gitattributes
+++ /dev/null
@@ -1,2 +0,0 @@
-benchmarks/sample_* linguist-generated
-tests/*/corpus/* linguist-generated
diff --git a/vendor/github.com/tdewolff/minify/v2/.gitignore b/vendor/github.com/tdewolff/minify/v2/.gitignore
deleted file mode 100644
index cd340e43e..000000000
--- a/vendor/github.com/tdewolff/minify/v2/.gitignore
+++ /dev/null
@@ -1,24 +0,0 @@
-release.sh
-dist/
-benchmarks/*
-!benchmarks/*.go
-!benchmarks/sample_*
-tests/*/fuzz-fuzz.zip
-tests/*/crashers
-tests/*/suppressions
-tests/*/corpus/*
-!tests/*/corpus/*.*
-parse/tests/*/fuzz-fuzz.zip
-parse/tests/*/crashers
-parse/tests/*/suppressions
-parse/tests/*/corpus/*
-!parse/tests/*/corpus/*.*
-bindings/js/build
-bindings/js/minify.h
-bindings/js/minify.a
-bindings/js/test.min.html
-bindings/js/node_modules
-bindings/py/minify.h
-bindings/py/minify.so
-bindings/py/test.min.html
-bindings/py/tdewolff_minify.egg-info
diff --git a/vendor/github.com/tdewolff/minify/v2/.golangci.yml b/vendor/github.com/tdewolff/minify/v2/.golangci.yml
deleted file mode 100644
index 7009f9201..000000000
--- a/vendor/github.com/tdewolff/minify/v2/.golangci.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-linters:
- enable:
- - depguard
- - dogsled
- - gofmt
- - goimports
- - golint
- - gosec
- - govet
- - megacheck
- - misspell
- - nakedret
- - prealloc
- - unconvert
- - unparam
- - wastedassign
diff --git a/vendor/github.com/tdewolff/minify/v2/Dockerfile b/vendor/github.com/tdewolff/minify/v2/Dockerfile
deleted file mode 100644
index fe3676b06..000000000
--- a/vendor/github.com/tdewolff/minify/v2/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-# Use this image to build the executable
-FROM golang:1.16-alpine AS compiler
-
-RUN apk add --no-cache git ca-certificates make
-
-WORKDIR $GOPATH/src/minify
-COPY . .
-
-RUN /usr/bin/env bash -c make install
-
-# Final image containing the executable from the previous step
-FROM alpine:3
-
-COPY --from=compiler /bin/minify /bin/minify
diff --git a/vendor/github.com/tdewolff/minify/v2/LICENSE b/vendor/github.com/tdewolff/minify/v2/LICENSE
deleted file mode 100644
index 41677de41..000000000
--- a/vendor/github.com/tdewolff/minify/v2/LICENSE
+++ /dev/null
@@ -1,22 +0,0 @@
-Copyright (c) 2015 Taco de Wolff
-
- Permission is hereby granted, free of charge, to any person
- obtaining a copy of this software and associated documentation
- files (the "Software"), to deal in the Software without
- restriction, including without limitation the rights to use,
- copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following
- conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/vendor/github.com/tdewolff/minify/v2/Makefile b/vendor/github.com/tdewolff/minify/v2/Makefile
deleted file mode 100644
index 1c908800e..000000000
--- a/vendor/github.com/tdewolff/minify/v2/Makefile
+++ /dev/null
@@ -1,58 +0,0 @@
-SHELL=/usr/bin/env bash
-NAME=minify
-CMD=./cmd/minify
-TARGETS=linux_amd64 darwin_amd64 freebsd_amd64 netbsd_amd64 openbsd_amd64 windows_amd64
-VERSION=`git describe --tags`
-FLAGS=-ldflags "-s -w -X 'main.Version=${VERSION}'" -trimpath
-ENVS=GO111MODULES=on CGO_ENABLED=0
-
-all: install
-
-install:
- echo "Installing ${VERSION}"
- ${ENVS} go install ${FLAGS} ./cmd/minify
- . cmd/minify/bash_completion
-
-release:
- TAG=$(shell git describe --tags --exact-match 2> /dev/null);
- if [ "${.SHELLSTATUS}" -eq 0 ]; then \
- echo "Releasing ${VERSION}"; \
- else \
- echo "ERROR: commit is not tagged with a version"; \
- echo ""; \
- exit 1; \
- fi
- rm -rf dist
- mkdir -p dist
- for t in ${TARGETS}; do \
- echo Building $$t...; \
- mkdir dist/$$t; \
- os=$$(echo $$t | cut -f1 -d_); \
- arch=$$(echo $$t | cut -f2 -d_); \
- ${ENVS} GOOS=$$os GOARCH=$$arch go build ${FLAGS} -o dist/$$t/${NAME} ${CMD}; \
- \
- cp LICENSE dist/$$t/.; \
- cp cmd/minify/README.md dist/$$t/.; \
- if [ "$$os" == "windows" ]; then \
- mv dist/$$t/${NAME} dist/$$t/${NAME}.exe; \
- zip -jq dist/${NAME}_$$t.zip dist/$$t/*; \
- cd dist; \
- sha256sum ${NAME}_$$t.zip >> checksums.txt; \
- cd ..; \
- else \
- cp cmd/minify/bash_completion dist/$$t/.; \
- cd dist/$$t; \
- tar -cf - * | gzip -9 > ../${NAME}_$$t.tar.gz; \
- cd ..; \
- sha256sum ${NAME}_$$t.tar.gz >> checksums.txt; \
- cd ..; \
- fi; \
- rm -rf dist/$$t; \
- done
-
-clean:
- echo "Cleaning dist/"
- rm -rf dist
-
-.PHONY: install release clean
-.SILENT: install release clean
diff --git a/vendor/github.com/tdewolff/minify/v2/README.md b/vendor/github.com/tdewolff/minify/v2/README.md
deleted file mode 100644
index 0f1679ba0..000000000
--- a/vendor/github.com/tdewolff/minify/v2/README.md
+++ /dev/null
@@ -1,724 +0,0 @@
-# Minify [![API reference](https://img.shields.io/badge/godoc-reference-5272B4)](https://pkg.go.dev/github.com/tdewolff/minify/v2?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/tdewolff/minify)](https://goreportcard.com/report/github.com/tdewolff/minify) [![codecov](https://codecov.io/gh/tdewolff/minify/branch/master/graph/badge.svg?token=Cr7r2EKPj2)](https://codecov.io/gh/tdewolff/minify) [![Donate](https://img.shields.io/badge/patreon-donate-DFB317)](https://www.patreon.com/tdewolff)
-
-**[Online demo](https://go.tacodewolff.nl/minify)** if you need to minify files *now*.
-
-**[Binaries](https://github.com/tdewolff/minify/releases) of CLI for various platforms.** See [CLI](https://github.com/tdewolff/minify/tree/master/cmd/minify) for more installation instructions.
-
-**[Python bindings](https://pypi.org/project/tdewolff-minify/)** install with `pip install tdewolff-minify`
-
-**[JavaScript bindings](https://www.npmjs.com/package/tdewolff-minify)**
-
----
-
-*Did you know that the shortest valid piece of HTML5 is `x `? See for yourself at the [W3C Validator](http://validator.w3.org/)!*
-
-Minify is a minifier package written in [Go][1]. It provides HTML5, CSS3, JS, JSON, SVG and XML minifiers and an interface to implement any other minifier. Minification is the process of removing bytes from a file (such as whitespace) without changing its output and therefore shrinking its size and speeding up transmission over the internet and possibly parsing. The implemented minifiers are designed for high performance.
-
-The core functionality associates mimetypes with minification functions, allowing embedded resources (like CSS or JS within HTML files) to be minified as well. Users can add new implementations that are triggered based on a mimetype (or pattern), or redirect to an external command (like ClosureCompiler, UglifyCSS, ...).
-
-### Sponsors
-
-[![SiteGround](https://www.siteground.com/img/downloads/siteground-logo-black-transparent-vector.svg)](https://www.siteground.com/)
-
-Please see https://www.patreon.com/tdewolff for ways to contribute, otherwise please contact me directly!
-
-#### Table of Contents
-
-- [Minify](#minify)
- - [Prologue](#prologue)
- - [Installation](#installation)
- - [API stability](#api-stability)
- - [Testing](#testing)
- - [Performance](#performance)
- - [HTML](#html)
- - [Whitespace removal](#whitespace-removal)
- - [CSS](#css)
- - [JS](#js)
- - [Comparison with other tools](#comparison-with-other-tools)
- - [Compression ratio (lower is better)](#compression-ratio-lower-is-better)
- - [Time (lower is better)](#time-lower-is-better)
- - [JSON](#json)
- - [SVG](#svg)
- - [XML](#xml)
- - [Usage](#usage)
- - [New](#new)
- - [From reader](#from-reader)
- - [From bytes](#from-bytes)
- - [From string](#from-string)
- - [To reader](#to-reader)
- - [To writer](#to-writer)
- - [Middleware](#middleware)
- - [Custom minifier](#custom-minifier)
- - [Mediatypes](#mediatypes)
- - [Examples](#examples)
- - [Common minifiers](#common-minifiers)
- - [External minifiers](#external-minifiers)
- - [Closure Compiler](#closure-compiler)
- - [UglifyJS](#uglifyjs)
- - [esbuild](#esbuild)
- - [Custom minifier](#custom-minifier-example)
- - [ResponseWriter](#responsewriter)
- - [Templates](#templates)
- - [License](#license)
-
-### Roadmap
-
-- [ ] Use ASM/SSE to further speed-up core parts of the parsers/minifiers
-- [x] Improve JS minifiers by shortening variables and proper semicolon omission
-- [ ] Speed-up SVG minifier, it is very slow
-- [x] Proper parser error reporting and line number + column information
-- [ ] Generation of source maps (uncertain, might slow down parsers too much if it cannot run separately nicely)
-- [ ] Create a cmd to pack webfiles (much like webpack), ie. merging CSS and JS files, inlining small external files, minification and gzipping. This would work on HTML files.
-
-## Prologue
-Minifiers or bindings to minifiers exist in almost all programming languages. Some implementations are merely using several regular expressions to trim whitespace and comments (even though regex for parsing HTML/XML is ill-advised, for a good read see [Regular Expressions: Now You Have Two Problems](http://blog.codinghorror.com/regular-expressions-now-you-have-two-problems/)). Some implementations are much more profound, such as the [YUI Compressor](http://yui.github.io/yuicompressor/) and [Google Closure Compiler](https://github.com/google/closure-compiler) for JS. As most existing implementations either use JavaScript, use regexes, and don't focus on performance, they are pretty slow.
-
-This minifier proves to be that fast and extensive minifier that can handle HTML and any other filetype it may contain (CSS, JS, ...). It is usually orders of magnitude faster than existing minifiers.
-
-## Installation
-Make sure you have [Git](https://git-scm.com/) and [Go](https://golang.org/dl/) (1.13 or higher) installed, run
-```
-mkdir Project
-cd Project
-go mod init
-go get -u github.com/tdewolff/minify/v2
-```
-
-Then add the following imports to be able to use the various minifiers
-``` go
-import (
- "github.com/tdewolff/minify/v2"
- "github.com/tdewolff/minify/v2/css"
- "github.com/tdewolff/minify/v2/html"
- "github.com/tdewolff/minify/v2/js"
- "github.com/tdewolff/minify/v2/json"
- "github.com/tdewolff/minify/v2/svg"
- "github.com/tdewolff/minify/v2/xml"
-)
-```
-
-You can optionally run `go mod tidy` to clean up the `go.mod` and `go.sum` files.
-
-See [CLI tool](https://github.com/tdewolff/minify/tree/master/cmd/minify) for installation instructions of the binary.
-
-### Docker
-
-If you want to use Docker, please see https://hub.docker.com/r/tdewolff/minify.
-
-```bash
-$ docker run -it tdewolff/minify
-/ # minify --version
-```
-
-## API stability
-There is no guarantee for absolute stability, but I take issues and bugs seriously and don't take API changes lightly. The library will be maintained in a compatible way unless vital bugs prevent me from doing so. There has been one API change after v1 which added options support and I took the opportunity to push through some more API clean up as well. There are no plans whatsoever for future API changes.
-
-## Testing
-For all subpackages and the imported `parse` package, test coverage of 100% is pursued. Besides full coverage, the minifiers are [fuzz tested](https://github.com/tdewolff/fuzz) using [github.com/dvyukov/go-fuzz](http://www.github.com/dvyukov/go-fuzz), see [the wiki](https://github.com/tdewolff/minify/wiki) for the most important bugs found by fuzz testing. These tests ensure that everything works as intended and that the code does not crash (whatever the input). If you still encounter a bug, please file a [bug report](https://github.com/tdewolff/minify/issues)!
-
-## Performance
-The benchmarks directory contains a number of standardized samples used to compare performance between changes. To give an indication of the speed of this library, I've ran the tests on my Thinkpad T460 (i5-6300U quad-core 2.4GHz running Arch Linux) using Go 1.15.
-
-```
-name time/op
-CSS/sample_bootstrap.css-4 2.70ms ± 0%
-CSS/sample_gumby.css-4 3.57ms ± 0%
-CSS/sample_fontawesome.css-4 767µs ± 0%
-CSS/sample_normalize.css-4 85.5µs ± 0%
-HTML/sample_amazon.html-4 15.2ms ± 0%
-HTML/sample_bbc.html-4 3.90ms ± 0%
-HTML/sample_blogpost.html-4 420µs ± 0%
-HTML/sample_es6.html-4 15.6ms ± 0%
-HTML/sample_stackoverflow.html-4 3.73ms ± 0%
-HTML/sample_wikipedia.html-4 6.60ms ± 0%
-JS/sample_ace.js-4 28.7ms ± 0%
-JS/sample_dot.js-4 357µs ± 0%
-JS/sample_jquery.js-4 10.0ms ± 0%
-JS/sample_jqueryui.js-4 20.4ms ± 0%
-JS/sample_moment.js-4 3.47ms ± 0%
-JSON/sample_large.json-4 3.25ms ± 0%
-JSON/sample_testsuite.json-4 1.74ms ± 0%
-JSON/sample_twitter.json-4 24.2µs ± 0%
-SVG/sample_arctic.svg-4 34.7ms ± 0%
-SVG/sample_gopher.svg-4 307µs ± 0%
-SVG/sample_usa.svg-4 57.4ms ± 0%
-SVG/sample_car.svg-4 18.0ms ± 0%
-SVG/sample_tiger.svg-4 5.61ms ± 0%
-XML/sample_books.xml-4 54.7µs ± 0%
-XML/sample_catalog.xml-4 33.0µs ± 0%
-XML/sample_omg.xml-4 7.17ms ± 0%
-
-name speed
-CSS/sample_bootstrap.css-4 50.7MB/s ± 0%
-CSS/sample_gumby.css-4 52.1MB/s ± 0%
-CSS/sample_fontawesome.css-4 61.2MB/s ± 0%
-CSS/sample_normalize.css-4 70.8MB/s ± 0%
-HTML/sample_amazon.html-4 31.1MB/s ± 0%
-HTML/sample_bbc.html-4 29.5MB/s ± 0%
-HTML/sample_blogpost.html-4 49.8MB/s ± 0%
-HTML/sample_es6.html-4 65.6MB/s ± 0%
-HTML/sample_stackoverflow.html-4 55.0MB/s ± 0%
-HTML/sample_wikipedia.html-4 67.5MB/s ± 0%
-JS/sample_ace.js-4 22.4MB/s ± 0%
-JS/sample_dot.js-4 14.5MB/s ± 0%
-JS/sample_jquery.js-4 24.8MB/s ± 0%
-JS/sample_jqueryui.js-4 23.0MB/s ± 0%
-JS/sample_moment.js-4 28.6MB/s ± 0%
-JSON/sample_large.json-4 234MB/s ± 0%
-JSON/sample_testsuite.json-4 394MB/s ± 0%
-JSON/sample_twitter.json-4 63.0MB/s ± 0%
-SVG/sample_arctic.svg-4 42.4MB/s ± 0%
-SVG/sample_gopher.svg-4 19.0MB/s ± 0%
-SVG/sample_usa.svg-4 17.8MB/s ± 0%
-SVG/sample_car.svg-4 29.3MB/s ± 0%
-SVG/sample_tiger.svg-4 12.2MB/s ± 0%
-XML/sample_books.xml-4 81.0MB/s ± 0%
-XML/sample_catalog.xml-4 58.6MB/s ± 0%
-XML/sample_omg.xml-4 159MB/s ± 0%
-```
-
-## HTML
-
-HTML (with JS and CSS) minification typically shaves off about 10%.
-
-The HTML5 minifier uses these minifications:
-
-- strip unnecessary whitespace and otherwise collapse it to one space (or newline if it originally contained a newline)
-- strip superfluous quotes, or uses single/double quotes whichever requires fewer escapes
-- strip default attribute values and attribute boolean values
-- strip some empty attributes
-- strip unrequired tags (`html`, `head`, `body`, ...)
-- strip unrequired end tags (`tr`, `td`, `li`, ... and often `p`)
-- strip default protocols (`http:`, `https:` and `javascript:`)
-- strip all comments (including conditional comments, old IE versions are not supported anymore by Microsoft)
-- shorten `doctype` and `meta` charset
-- lowercase tags, attributes and some values to enhance gzip compression
-
-Options:
-
-- `KeepConditionalComments` preserve all IE conditional comments such as `` and ``, see https://msdn.microsoft.com/en-us/library/ms537512(v=vs.85).aspx#syntax
-- `KeepDefaultAttrVals` preserve default attribute values such as `` // Faulty JS
- req := httptest.NewRequest(http.MethodGet, "/", nil)
- rec := httptest.NewRecorder()
- m.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "text/html")
- _, _ = w.Write([]byte(input))
-
- if err = w.(io.Closer).Close(); err != nil {
- panic(err)
- }
- })).ServeHTTP(rec, req)
-}
-```
-
-#### ResponseWriter
-``` go
-func Serve(w http.ResponseWriter, r *http.Request) {
- mw := m.ResponseWriter(w, r)
- defer mw.Close()
- w = mw
-
- http.ServeFile(w, r, path.Join("www", r.URL.Path))
-}
-```
-
-#### Custom response writer
-ResponseWriter example which returns a ResponseWriter that minifies the content and then writes to the original ResponseWriter. Any write after applying this filter will be minified.
-``` go
-type MinifyResponseWriter struct {
- http.ResponseWriter
- io.WriteCloser
-}
-
-func (m MinifyResponseWriter) Write(b []byte) (int, error) {
- return m.WriteCloser.Write(b)
-}
-
-// MinifyResponseWriter must be closed explicitly by calling site.
-func MinifyFilter(mediatype string, res http.ResponseWriter) MinifyResponseWriter {
- m := minify.New()
- // add minfiers
-
- mw := m.Writer(mediatype, res)
- return MinifyResponseWriter{res, mw}
-}
-```
-
-``` go
-// Usage
-func(w http.ResponseWriter, req *http.Request) {
- w = MinifyFilter("text/html", w)
- if _, err := io.WriteString(w, " "); err != nil {
- panic(err)
- }
- if err := w.Close(); err != nil {
- panic(err)
- }
- // Output: