diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index eb708219e..da25d29c5 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -2003,8 +2003,8 @@ definitions: type: array x-go-name: Accounts hashtags: - items: - $ref: '#/definitions/tag' + description: Slice of strings if api v1, slice of tags if api v2. + items: {} type: array x-go-name: Hashtags statuses: @@ -2483,6 +2483,14 @@ definitions: x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users tag: properties: + history: + description: |- + History of this hashtag's usage. + Currently just a stub, if provided will always be an empty array. + example: [] + items: {} + type: array + x-go-name: History name: description: 'The value of the hashtag after the # sign.' example: helloworld @@ -2666,6 +2674,98 @@ paths: summary: Upload a new media attachment. tags: - media + /api/{api_version}/search: + get: + description: If statuses are in the result, they will be returned in descending chronological order (newest first), with sequential IDs (bigger = newer). + operationId: searchGet + parameters: + - description: Version of the API to use. Must be either `v1` or `v2`. If v1 is used, Hashtag results will be a slice of strings. If v2 is used, Hashtag results will be a slice of apimodel tags. + in: path + name: api_version + required: true + type: string + - description: Return only items *OLDER* than the given max ID. The item with the specified ID will not be included in the response. Currently only used if 'type' is set to a specific type. + in: query + name: max_id + type: string + - description: Return only items *immediately newer* than the given min ID. The item with the specified ID will not be included in the response. Currently only used if 'type' is set to a specific type. + in: query + name: min_id + type: string + - default: 20 + description: Number of each type of item to return. + in: query + maximum: 40 + minimum: 1 + name: limit + type: integer + - default: 0 + description: Page number of results to return (starts at 0). This parameter is currently not used, page by selecting a specific query type and using maxID and minID instead. + in: query + maximum: 10 + minimum: 0 + name: offset + type: integer + - description: |- + Query string to search for. This can be in the following forms: + - `@[username]` -- search for an account with the given username on any domain. Can return multiple results. + - @[username]@[domain]` -- search for a remote account with exact username and domain. Will only ever return 1 result at most. + - `https://example.org/some/arbitrary/url` -- search for an account OR a status with the given URL. Will only ever return 1 result at most. + - `#[hashtag_name]` -- search for a hashtag with the given hashtag name, or starting with the given hashtag name. Case insensitive. Can return multiple results. + - any arbitrary string -- search for accounts or statuses containing the given string. Can return multiple results. + in: query + name: q + required: true + type: string + - description: |- + Type of item to return. One of: + - `` -- empty string; return any/all results. + - `accounts` -- return only account(s). + - `statuses` -- return only status(es). + - `hashtags` -- return only hashtag(s). + If `type` is specified, paging can be performed using max_id and min_id parameters. + If `type` is not specified, see the `offset` parameter for paging. + in: query + name: type + type: string + - default: false + description: If searching query is for `@[username]@[domain]`, or a URL, allow the GoToSocial instance to resolve the search by making calls to remote instances (webfinger, ActivityPub, etc). + in: query + name: resolve + type: boolean + - default: false + description: If search type includes accounts, and search query is an arbitrary string, show only accounts that the requesting account follows. If this is set to `true`, then the GoToSocial instance will enhance the search by also searching within account notes, not just in usernames and display names. + in: query + name: following + type: boolean + - default: false + description: If searching for hashtags, exclude those not yet approved by instance admin. Currently this parameter is unused. + in: query + name: exclude_unreviewed + type: boolean + produces: + - application/json + responses: + "200": + description: Results of the search. + schema: + $ref: '#/definitions/searchResult' + "400": + description: bad request + "401": + description: unauthorized + "404": + description: not found + "406": + description: not acceptable + "500": + description: internal server error + security: + - OAuth2 Bearer: + - read:search + summary: Search for statuses, accounts, or hashtags, on this instance or elsewhere. + tags: + - search /api/v1/accounts: post: consumes: @@ -5474,94 +5574,6 @@ paths: summary: Get one report with the given id. tags: - reports - /api/v1/search: - get: - description: If statuses are in the result, they will be returned in descending chronological order (newest first), with sequential IDs (bigger = newer). - operationId: searchGet - parameters: - - description: Return only items *OLDER* than the given max ID. The item with the specified ID will not be included in the response. Currently only used if 'type' is set to a specific type. - in: query - name: max_id - type: string - - description: Return only items *immediately newer* than the given min ID. The item with the specified ID will not be included in the response. Currently only used if 'type' is set to a specific type. - in: query - name: min_id - type: string - - default: 20 - description: Number of each type of item to return. - in: query - maximum: 40 - minimum: 1 - name: limit - type: integer - - default: 0 - description: Page number of results to return (starts at 0). This parameter is currently not used, page by selecting a specific query type and using maxID and minID instead. - in: query - maximum: 10 - minimum: 0 - name: offset - type: integer - - description: |- - Query string to search for. This can be in the following forms: - - `@[username]` -- search for an account with the given username on any domain. Can return multiple results. - - @[username]@[domain]` -- search for a remote account with exact username and domain. Will only ever return 1 result at most. - - `https://example.org/some/arbitrary/url` -- search for an account OR a status with the given URL. Will only ever return 1 result at most. - - any arbitrary string -- search for accounts or statuses containing the given string. Can return multiple results. - in: query - name: q - required: true - type: string - - description: |- - Type of item to return. One of: - - `` -- empty string; return any/all results. - - `accounts` -- return account(s). - - `statuses` -- return status(es). - - `hashtags` -- return hashtag(s). - If `type` is specified, paging can be performed using max_id and min_id parameters. - If `type` is not specified, see the `offset` parameter for paging. - in: query - name: type - type: string - - default: false - description: If searching query is for `@[username]@[domain]`, or a URL, allow the GoToSocial instance to resolve the search by making calls to remote instances (webfinger, ActivityPub, etc). - in: query - name: resolve - type: boolean - - default: false - description: If search type includes accounts, and search query is an arbitrary string, show only accounts that the requesting account follows. If this is set to `true`, then the GoToSocial instance will enhance the search by also searching within account notes, not just in usernames and display names. - in: query - name: following - type: boolean - - default: false - description: If searching for hashtags, exclude those not yet approved by instance admin. Currently this parameter is unused. - in: query - name: exclude_unreviewed - type: boolean - produces: - - application/json - responses: - "200": - description: Results of the search. - schema: - items: - $ref: '#/definitions/searchResult' - type: array - "400": - description: bad request - "401": - description: unauthorized - "404": - description: not found - "406": - description: not acceptable - "500": - description: internal server error - security: - - OAuth2 Bearer: - - read:search - summary: Search for statuses, accounts, or hashtags, on this instance or elsewhere. - tags: - - search /api/v1/statuses: post: consumes: @@ -6413,6 +6425,62 @@ paths: summary: See public statuses/posts that your instance is aware of. tags: - timelines + /api/v1/timelines/tag/{tag_name}: + get: + description: |- + The statuses will be returned in descending chronological order (newest first), with sequential IDs (bigger = newer). + + The returned Link header can be used to generate the previous and next queries when scrolling up or down a timeline. + + Example: + + ``` + ; rel="next", ; rel="prev" + ```` + operationId: tagTimeline + parameters: + - description: Return only statuses *OLDER* than the given max status ID. The status with the specified ID will not be included in the response. + in: query + name: max_id + type: string + - description: Return only statuses *newer* than the given since status ID. The status with the specified ID will not be included in the response. + in: query + name: since_id + type: string + - description: Return only statuses *immediately newer* than the given since status ID. The status with the specified ID will not be included in the response. + in: query + name: min_id + type: string + - default: 20 + description: Number of statuses to return. + in: query + maximum: 40 + minimum: 1 + name: limit + type: integer + produces: + - application/json + responses: + "200": + description: Array of statuses. + headers: + Link: + description: Links to the next and previous queries. + type: string + schema: + items: + $ref: '#/definitions/status' + type: array + "400": + description: bad request + "401": + description: unauthorized + security: + - OAuth2 Bearer: + - read:statuses + summary: See public statuses that use the given hashtag (case insensitive). + tags: + - timelines /api/v1/user/password_change: post: consumes: diff --git a/docs/federation/federating_with_gotosocial.md b/docs/federation/federating_with_gotosocial.md index 9723ce2c6..f5796b58e 100644 --- a/docs/federation/federating_with_gotosocial.md +++ b/docs/federation/federating_with_gotosocial.md @@ -380,3 +380,50 @@ While `attachment` is not technically an ordered collection, GoToSocial--again, GoToSocial will also parse PropertyValue fields from remote `actor`s discovered by the GoToSocial instance, to allow them to be displayed to users on the GoToSocial instance. GoToSocial allows up to 6 `PropertyValue` fields by default, as opposed to Mastodon's default 4. + +## Hashtags + +GoToSocial users can include hashtags in their posts, which indicate to other instances that that user wishes their post to be grouped together with other posts using the same hashtag, for discovery purposes. + +In line with other ActivityPub server implementations, GoToSocial implicitly expects that only public-addressed posts will be grouped by hashtag. + +To federate hashtags in and out, GoToSocial uses the widely-adopted [ActivityStreams `Hashtag` type extension](https://www.w3.org/wiki/Activity_Streams_extensions#as:Hashtag_type) in the `tag` property of objects. + +Here's what the `tag` property might look like on an outgoing message that uses one custom emoji, and one tag: + +```json +"tag": [ + { + "icon": { + "mediaType": "image/png", + "type": "Image", + "url": "https://example.org/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png" + }, + "id": "https://example.org/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ", + "name": ":rainbow:", + "type": "Emoji", + "updated": "2021-09-20T10:40:37Z" + }, + { + "href": "https://example.org/tags/welcome", + "name": "#welcome", + "type": "Hashtag" + } +] +``` + +With just one tag, the `tag` property will be an object rather than an array, which will look like this: + +```json +"tag": { + "href": "https://example.org/tags/welcome", + "name": "#welcome", + "type": "Hashtag" +} +``` + +### Hashtag `href` property + +The `href` URL provided by GoToSocial in outgoing tags points to a web URL that serves `text/html`. + +GoToSocial makes no guarantees whatsoever about what the content of the given `text/html` will be, and remote servers should not interpret the URL as a canonical ActivityPub ID/URI property. The `href` URL is provided merely as an endpoint which *might* contain more information about the given hashtag. diff --git a/docs/user_guide/posts.md b/docs/user_guide/posts.md index 43eaaf454..cf217b06a 100644 --- a/docs/user_guide/posts.md +++ b/docs/user_guide/posts.md @@ -241,6 +241,26 @@ which will be rendered as: > hey @local_account_person you're my neighbour +### Hashtags + +You can use one or more hashtags in your post to indicate subject matter, and to allow the post to be grouped together with other posts using the same hashtag in order to aid discoverability of your posts. + +Most ActivityPub server implementations like Mastodon and similar only group together **Public** posts by the hashtags they use, but there is no guarantee about that. Generally speaking then, it is better to only use hashtags for Public visibility posts where you want the post to be able to spread more widely than it would otherwise. A good example of this is the `#introduction` hashtag, which tends to be used by new accounts who want to introduce themselves to the fediverse! + +Including hashtags in your post works like most other social media software: just add a `#` symbol before the word you want to use as a hashtag. + +Some examples: + +* `#introduction` +* `#Mosstodon` +* `#LichenSubscribe` + +Hashtags in GoToSocial are case-insensitive, so it doesn't matter if you use uppercase, lowercase, or a mixture of both when writing your hashtag, it will still count as the same hashtag. For example, `#Introduction` and `#introduction` are treated exactly the same. + +For accessibility reasons, it is considerate to use upper camel case when you're writing hashtags. In other words: capitalize the first letter of every word in the hashtag. So rather than writing `#thisisahashtag`, which is difficult to read visually, and difficult for screenreaders to read out loud, consider writing `#ThisIsAHashtag` instead. + +You can include as many hashtags as you like within a GoToSocial post, and each hashtag has a length limit of 100 characters. + ## Input Sanitization In order not to spread scripts, vulnerabilities, and glitchy HTML all over the place, GoToSocial performs the following types of input sanitization: diff --git a/example/config.yaml b/example/config.yaml index ce471ffed..ceef6dc95 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -321,6 +321,10 @@ cache: status-fave-ttl: "30m" status-fave-sweep-freq: "1m" + tag-max-size: 2000 + tag-ttl: "30m" + tag-sweep-freq: "1m" + tombstone-max-size: 500 tombstone-ttl: "30m" tombstone-sweep-freq: "1m" diff --git a/internal/ap/ap_test.go b/internal/ap/ap_test.go index 105bc1fcf..6a5073c63 100644 --- a/internal/ap/ap_test.go +++ b/internal/ap/ap_test.go @@ -98,6 +98,97 @@ func noteWithMentions1() vocab.ActivityStreamsNote { return note } +func (suite *APTestSuite) noteWithHashtags1() ap.Statusable { + noteJson := []byte(` +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "votersCount": "toot:votersCount", + "Hashtag": "as:Hashtag" + } + ], + "id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319", + "type": "Note", + "summary": null, + "inReplyTo": null, + "published": "2023-06-26T09:01:56Z", + "url": "https://mastodon.social/@pixelfed/110609702372389319", + "attributedTo": "https://mastodon.social/users/pixelfed", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.social/users/pixelfed/followers", + "https://gts.superseriousbusiness.org/users/gotosocial" + ], + "sensitive": false, + "atomUri": "https://mastodon.social/users/pixelfed/statuses/110609702372389319", + "inReplyToAtomUri": null, + "conversation": "tag:mastodon.social,2023-06-26:objectId=474977189:objectType=Conversation", + "content": "

⚡ Heard of @gotosocial ?

GoToSocial provides a lightweight, customizable, and safety-focused entryway into the #fediverse, you can keep in touch with your friends, post, read, and share images and articles.

Consider #GoToSocial instead of Pixelfed if you'd like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!

We ❤️ GtS, check them out!

🌍 https://gotosocial.org/

🔍 https://fedidb.org/software/gotosocial

", + "contentMap": { + "en": "

⚡ Heard of @gotosocial ?

GoToSocial provides a lightweight, customizable, and safety-focused entryway into the #fediverse, you can keep in touch with your friends, post, read, and share images and articles.

Consider #GoToSocial instead of Pixelfed if you'd like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!

We ❤️ GtS, check them out!

🌍 https://gotosocial.org/

🔍 https://fedidb.org/software/gotosocial

" + }, + "attachment": [], + "tag": [ + { + "type": "Mention", + "href": "https://gts.superseriousbusiness.org/users/gotosocial", + "name": "@gotosocial@superseriousbusiness.org" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/fediverse", + "name": "#fediverse" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/gotosocial", + "name": "#gotosocial" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/this_hashtag_will_be_ignored_since_it_cant_be_normalized", + "name": "#b̴̛͇̒̌͑̓̐̑͗̏̐̇͗̎̕͝O̵̧̧͎̟̰̭̊͌͒́̊̑̄̐͐͗Ọ̷̧̡̰̟̪̫̹͖͇̱͕̺̦̲̀̐̽̓̇̚͠b̶̨̖͍͙͈̹͉̯͕̯̯̯̞̼̞̏͊͂̐̔͛s̴̢̞̺͈͇̘͚͉͔̥̔͛͆͑͑̍̄̌̚͜͜ͅ" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/this_hashtag_will_be_included_correctly", + "name": "#Grüvy" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/this_hashtag_will_be_squashed_into_a_single_character", + "name": "#` + `ᄀ` + `ᅡ` + `ᆨ` + `" + } + ], + "replies": { + "id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "next": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies?only_other_accounts=true&page=true", + "partOf": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies", + "items": [] + } + } +}`) + + statusable, err := ap.ResolveStatusable(context.Background(), noteJson) + if err != nil { + suite.FailNow(err.Error()) + } + + return statusable +} + func addressable1() ap.Addressable { // make a note addressed to public with followers in cc note := streams.NewActivityStreamsNote() diff --git a/internal/ap/extract.go b/internal/ap/extract.go index 9a1b6aa4f..21ff20235 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -30,6 +30,7 @@ "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -529,7 +530,7 @@ func ExtractBlurhash(i WithBlurhash) string { // ExtractHashtags extracts a slice of minimal gtsmodel.Tags // from a WithTag. If an entry in the WithTag is not a hashtag, -// it will be quietly ignored. +// or has a name that cannot be normalized, it will be ignored. // // TODO: find a better heuristic for determining if something // is a hashtag or not, since looking for type name "Hashtag" @@ -562,18 +563,29 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { continue } - tag, err := ExtractHashtag(hashtaggable) + tag, err := extractHashtag(hashtaggable) if err != nil { continue } + // "Normalize" this tag by combining diacritics + + // unicode chars. If this returns false, it means + // we couldn't normalize it well enough to make it + // valid on our instance, so just ignore it. + normalized, ok := text.NormalizeHashtag(tag.Name) + if !ok { + continue + } + + // We store tag names lowercased, might + // as well change case here already. + tag.Name = strings.ToLower(normalized) + // Only append this tag if we haven't // seen it already, to avoid duplicates // in the slice. - if _, set := keys[tag.URL]; !set { - keys[tag.URL] = nil // Value doesn't matter. - tags = append(tags, tag) - tags = append(tags, tag) + if _, set := keys[tag.Name]; !set { + keys[tag.Name] = nil // Value doesn't matter. tags = append(tags, tag) } } @@ -581,16 +593,9 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { return tags, nil } -// ExtractEmoji extracts a minimal gtsmodel.Tag -// from the given Hashtaggable. -func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { - // Extract href/link for this tag. - hrefProp := i.GetActivityStreamsHref() - if hrefProp == nil || !hrefProp.IsIRI() { - return nil, gtserror.New("no href prop") - } - tagURL := hrefProp.GetIRI().String() - +// extractHashtag extracts a minimal gtsmodel.Tag from the given +// Hashtaggable, without yet doing any normalization on it. +func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { // Extract name for the tag; trim leading hash // character, so '#example' becomes 'example'. name := ExtractName(i) @@ -599,9 +604,11 @@ func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { } tagName := strings.TrimPrefix(name, "#") + yeah := func() *bool { t := true; return &t } return >smodel.Tag{ - URL: tagURL, - Name: tagName, + Name: tagName, + Useable: yeah(), // Assume true by default. + Listable: yeah(), // Assume true by default. }, nil } diff --git a/internal/ap/extracthashtags_test.go b/internal/ap/extracthashtags_test.go new file mode 100644 index 000000000..1d4fbcf6f --- /dev/null +++ b/internal/ap/extracthashtags_test.go @@ -0,0 +1,66 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ap_test + +import ( + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" +) + +type ExtractHashtagsTestSuite struct { + APTestSuite +} + +func (suite *ExtractHashtagsTestSuite) TestExtractHashtags1() { + note := suite.noteWithHashtags1() + + hashtags, err := ap.ExtractHashtags(note) + if err != nil { + suite.FailNow(err.Error()) + } + + if l := len(hashtags); l != 4 { + suite.FailNow("", "expected 4 hashtags, got %d", l) + } + + hashtagFediverse := hashtags[0] + suite.Equal("fediverse", hashtagFediverse.Name) + suite.Equal(true, *hashtagFediverse.Useable) + suite.Equal(true, *hashtagFediverse.Listable) + + hashtagGoToSocial := hashtags[1] + suite.Equal("gotosocial", hashtagGoToSocial.Name) + suite.Equal(true, *hashtagGoToSocial.Useable) + suite.Equal(true, *hashtagGoToSocial.Listable) + + hashtagGrüvy := hashtags[2] + suite.Equal("grüvy", hashtagGrüvy.Name) + suite.Equal(true, *hashtagGrüvy.Useable) + suite.Equal(true, *hashtagGrüvy.Listable) + + hashtagAngle := hashtags[3] + suite.Equal("각", hashtagAngle.Name) + suite.Equal(true, *hashtagAngle.Useable) + suite.Equal(true, *hashtagAngle.Listable) +} + +func TestExtractHashtagsTestSuite(t *testing.T) { + suite.Run(t, &ExtractHashtagsTestSuite{}) +} diff --git a/internal/api/client/media/media.go b/internal/api/client/media/media.go index 833cba0a2..dc640d380 100644 --- a/internal/api/client/media/media.go +++ b/internal/api/client/media/media.go @@ -21,16 +21,14 @@ "net/http" "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/processing" ) const ( - IDKey = "id" // IDKey is the key for media attachment IDs - APIVersionKey = "api_version" // APIVersionKey is the key for which version of the API to use (v1 or v2) - APIv1 = "v1" // APIV1 corresponds to version 1 of the api - APIv2 = "v2" // APIV2 corresponds to version 2 of the api - BasePath = "/:" + APIVersionKey + "/media" // BasePath is the base API path for making media requests through v1 or v2 of the api (for mastodon API compatibility) - AttachmentWithID = BasePath + "/:" + IDKey // BasePathWithID corresponds to a media attachment with the given ID + IDKey = "id" // IDKey is the key for media attachment IDs + BasePath = "/:" + apiutil.APIVersionKey + "/media" // BasePath is the base API path for making media requests through v1 or v2 of the api (for mastodon API compatibility) + AttachmentWithID = BasePath + "/:" + IDKey // BasePathWithID corresponds to a media attachment with the given ID ) type Module struct { diff --git a/internal/api/client/media/mediacreate.go b/internal/api/client/media/mediacreate.go index 0ae3ff70d..d2264bb0d 100644 --- a/internal/api/client/media/mediacreate.go +++ b/internal/api/client/media/mediacreate.go @@ -93,10 +93,12 @@ // '500': // description: internal server error func (m *Module) MediaCreatePOSTHandler(c *gin.Context) { - apiVersion := c.Param(APIVersionKey) - if apiVersion != APIv1 && apiVersion != APIv2 { - err := errors.New("api version must be one of v1 or v2 for this path") - apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGetV1) + apiVersion, errWithCode := apiutil.ParseAPIVersion( + c.Param(apiutil.APIVersionKey), + []string{apiutil.APIv1, apiutil.APIv2}..., + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return } @@ -128,7 +130,7 @@ func (m *Module) MediaCreatePOSTHandler(c *gin.Context) { return } - if apiVersion == APIv2 { + if apiVersion == apiutil.APIv2 { // the mastodon v2 media API specifies that the URL should be null // and that the client should call /api/v1/media/:id to get the URL // diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 27e77f12f..471be8557 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -32,6 +32,7 @@ "github.com/stretchr/testify/suite" mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -169,7 +170,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() { ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) ctx.Request.Header.Set("accept", "application/json") - ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1) + ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1) // do the actual request suite.mediaModule.MediaCreatePOSTHandler(ctx) @@ -254,7 +255,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessfulV2() { ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v2/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) ctx.Request.Header.Set("accept", "application/json") - ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv2) + ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv2) // do the actual request suite.mediaModule.MediaCreatePOSTHandler(ctx) @@ -337,7 +338,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateLongDescription() { ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) ctx.Request.Header.Set("accept", "application/json") - ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1) + ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1) // do the actual request suite.mediaModule.MediaCreatePOSTHandler(ctx) @@ -378,7 +379,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateTooShortDescription() { ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) ctx.Request.Header.Set("accept", "application/json") - ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1) + ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1) // do the actual request suite.mediaModule.MediaCreatePOSTHandler(ctx) diff --git a/internal/api/client/media/mediaget.go b/internal/api/client/media/mediaget.go index f06991e7f..431f73d65 100644 --- a/internal/api/client/media/mediaget.go +++ b/internal/api/client/media/mediaget.go @@ -66,9 +66,11 @@ // '500': // description: internal server error func (m *Module) MediaGETHandler(c *gin.Context) { - if apiVersion := c.Param(APIVersionKey); apiVersion != APIv1 { - err := errors.New("api version must be one v1 for this path") - apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGetV1) + if _, errWithCode := apiutil.ParseAPIVersion( + c.Param(apiutil.APIVersionKey), + []string{apiutil.APIv1, apiutil.APIv2}..., + ); errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return } diff --git a/internal/api/client/media/mediaupdate.go b/internal/api/client/media/mediaupdate.go index 23e416c20..032cfd705 100644 --- a/internal/api/client/media/mediaupdate.go +++ b/internal/api/client/media/mediaupdate.go @@ -98,9 +98,11 @@ // '500': // description: internal server error func (m *Module) MediaPUTHandler(c *gin.Context) { - if apiVersion := c.Param(APIVersionKey); apiVersion != APIv1 { - err := errors.New("api version must be one v1 for this path") - apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGetV1) + if _, errWithCode := apiutil.ParseAPIVersion( + c.Param(apiutil.APIVersionKey), + []string{apiutil.APIv1, apiutil.APIv2}..., + ); errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return } diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go index 5fd3e2856..1af3bcf06 100644 --- a/internal/api/client/media/mediaupdate_test.go +++ b/internal/api/client/media/mediaupdate_test.go @@ -30,6 +30,7 @@ "github.com/stretchr/testify/suite" mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -160,7 +161,7 @@ func (suite *MediaUpdateTestSuite) TestUpdateImage() { ctx.Request = httptest.NewRequest(http.MethodPut, fmt.Sprintf("http://localhost:8080/api/v1/media/%s", toUpdate.ID), bytes.NewReader(buf.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) ctx.Request.Header.Set("accept", "application/json") - ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1) + ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1) ctx.AddParam(mediamodule.IDKey, toUpdate.ID) // do the actual request @@ -221,7 +222,7 @@ func (suite *MediaUpdateTestSuite) TestUpdateImageShortDescription() { ctx.Request = httptest.NewRequest(http.MethodPut, fmt.Sprintf("http://localhost:8080/api/v1/media/%s", toUpdate.ID), bytes.NewReader(buf.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) ctx.Request.Header.Set("accept", "application/json") - ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1) + ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1) ctx.AddParam(mediamodule.IDKey, toUpdate.ID) // do the actual request diff --git a/internal/api/client/search/search.go b/internal/api/client/search/search.go index 219e30280..d413aff91 100644 --- a/internal/api/client/search/search.go +++ b/internal/api/client/search/search.go @@ -21,12 +21,12 @@ "net/http" "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/processing" ) const ( - BasePathV1 = "/v1/search" // Base path for serving v1 of the search API, minus the 'api' prefix. - BasePathV2 = "/v2/search" // Base path for serving v2 of the search API, minus the 'api' prefix. + BasePath = "/:" + apiutil.APIVersionKey + "/search" ) type Module struct { @@ -40,6 +40,5 @@ func New(processor *processing.Processor) *Module { } func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) { - attachHandler(http.MethodGet, BasePathV1, m.SearchGETHandler) - attachHandler(http.MethodGet, BasePathV2, m.SearchGETHandler) + attachHandler(http.MethodGet, BasePath, m.SearchGETHandler) } diff --git a/internal/api/client/search/searchget.go b/internal/api/client/search/searchget.go index 33a90e078..2759feb5b 100644 --- a/internal/api/client/search/searchget.go +++ b/internal/api/client/search/searchget.go @@ -27,7 +27,7 @@ "github.com/superseriousbusiness/gotosocial/internal/oauth" ) -// SearchGETHandler swagger:operation GET /api/v1/search searchGet +// SearchGETHandler swagger:operation GET /api/{api_version}/search searchGet // // Search for statuses, accounts, or hashtags, on this instance or elsewhere. // @@ -42,6 +42,15 @@ // // parameters: // - +// name: api_version +// type: string +// in: path +// description: >- +// Version of the API to use. Must be either `v1` or `v2`. +// If v1 is used, Hashtag results will be a slice of strings. +// If v2 is used, Hashtag results will be a slice of apimodel tags. +// required: true +// - // name: max_id // type: string // description: >- @@ -88,6 +97,7 @@ // - `@[username]` -- search for an account with the given username on any domain. Can return multiple results. // - @[username]@[domain]` -- search for a remote account with exact username and domain. Will only ever return 1 result at most. // - `https://example.org/some/arbitrary/url` -- search for an account OR a status with the given URL. Will only ever return 1 result at most. +// - `#[hashtag_name]` -- search for a hashtag with the given hashtag name, or starting with the given hashtag name. Case insensitive. Can return multiple results. // - any arbitrary string -- search for accounts or statuses containing the given string. Can return multiple results. // in: query // required: true @@ -97,9 +107,9 @@ // description: |- // Type of item to return. One of: // - `` -- empty string; return any/all results. -// - `accounts` -- return account(s). -// - `statuses` -- return status(es). -// - `hashtags` -- return hashtag(s). +// - `accounts` -- return only account(s). +// - `statuses` -- return only status(es). +// - `hashtags` -- return only hashtag(s). // If `type` is specified, paging can be performed using max_id and min_id parameters. // If `type` is not specified, see the `offset` parameter for paging. // in: query @@ -138,9 +148,7 @@ // name: search results // description: Results of the search. // schema: -// type: array -// items: -// "$ref": "#/definitions/searchResult" +// "$ref": "#/definitions/searchResult" // '400': // description: bad request // '401': @@ -152,6 +160,15 @@ // '500': // description: internal server error func (m *Module) SearchGETHandler(c *gin.Context) { + apiVersion, errWithCode := apiutil.ParseAPIVersion( + c.Param(apiutil.APIVersionKey), + []string{apiutil.APIv1, apiutil.APIv2}..., + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + authed, err := oauth.Authed(c, true, true, true, true) if err != nil { apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1) @@ -209,6 +226,7 @@ func (m *Module) SearchGETHandler(c *gin.Context) { Resolve: resolve, Following: following, ExcludeUnreviewed: excludeUnreviewed, + APIv1: apiVersion == apiutil.APIv1, } results, errWithCode := m.processor.Search().Get(c.Request.Context(), authed.Account, searchRequest) diff --git a/internal/api/client/search/searchget_test.go b/internal/api/client/search/searchget_test.go index f6a2db70a..edaac2fc1 100644 --- a/internal/api/client/search/searchget_test.go +++ b/internal/api/client/search/searchget_test.go @@ -47,6 +47,7 @@ type SearchGetTestSuite struct { func (suite *SearchGetTestSuite) getSearch( requestingAccount *gtsmodel.Account, token *gtsmodel.Token, + apiVersion string, user *gtsmodel.User, maxID *string, minID *string, @@ -62,11 +63,13 @@ func (suite *SearchGetTestSuite) getSearch( var ( recorder = httptest.NewRecorder() ctx, _ = testrig.CreateGinTestContext(recorder, nil) - requestURL = testrig.URLMustParse("/api" + search.BasePathV1) + requestURL = testrig.URLMustParse("/api" + search.BasePath) queryParts []string ) // Put the request together. + ctx.AddParam(apiutil.APIVersionKey, apiVersion) + if maxID != nil { queryParts = append(queryParts, apiutil.MaxIDKey+"="+url.QueryEscape(*maxID)) } @@ -175,6 +178,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByURI() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -218,6 +222,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestring() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -261,6 +266,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringUppercase() searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -304,6 +310,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringNoLeadingAt( searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -347,6 +354,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringNoResolve() searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -385,6 +393,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringSpecialChars searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -426,6 +435,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringSpecialChars searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -467,6 +477,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByNamestring() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -510,6 +521,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByNamestringWithDomain() searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -553,6 +565,7 @@ func (suite *SearchGetTestSuite) TestSearchNonexistingLocalAccountByNamestringRe searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -591,6 +604,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByURI() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -634,6 +648,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByURL() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -677,6 +692,7 @@ func (suite *SearchGetTestSuite) TestSearchNonexistingLocalAccountByURL() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -715,6 +731,7 @@ func (suite *SearchGetTestSuite) TestSearchStatusByURL() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -758,6 +775,7 @@ func (suite *SearchGetTestSuite) TestSearchBlockedDomainURL() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -798,6 +816,7 @@ func (suite *SearchGetTestSuite) TestSearchBlockedDomainNamestring() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -838,6 +857,7 @@ func (suite *SearchGetTestSuite) TestSearchAAny() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -878,6 +898,7 @@ func (suite *SearchGetTestSuite) TestSearchAAnyFollowingOnly() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -918,6 +939,7 @@ func (suite *SearchGetTestSuite) TestSearchAStatuses() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -958,6 +980,7 @@ func (suite *SearchGetTestSuite) TestSearchAAccounts() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -998,6 +1021,7 @@ func (suite *SearchGetTestSuite) TestSearchAAccountsLimit1() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -1038,6 +1062,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalInstanceAccountByURI() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -1084,6 +1109,7 @@ func (suite *SearchGetTestSuite) TestSearchInstanceAccountFull() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -1130,6 +1156,7 @@ func (suite *SearchGetTestSuite) TestSearchInstanceAccountPartial() { searchResult, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -1170,6 +1197,7 @@ func (suite *SearchGetTestSuite) TestSearchBadQueryType() { _, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -1206,6 +1234,7 @@ func (suite *SearchGetTestSuite) TestSearchEmptyQuery() { _, err := suite.getSearch( requestingAccount, token, + apiutil.APIv2, user, maxID, minID, @@ -1222,6 +1251,170 @@ func (suite *SearchGetTestSuite) TestSearchEmptyQuery() { } } +func (suite *SearchGetTestSuite) TestSearchHashtagV1() { + var ( + requestingAccount = suite.testAccounts["local_account_1"] + token = suite.testTokens["local_account_1"] + user = suite.testUsers["local_account_1"] + maxID *string = nil + minID *string = nil + limit *int = nil + offset *int = nil + resolve *bool = nil + query = "#welcome" + queryType *string = func() *string { i := "hashtags"; return &i }() + following *bool = nil + expectedHTTPStatus = http.StatusOK + expectedBody = `{"accounts":[],"statuses":[],"hashtags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome","history":[]}]}` + ) + + searchResult, err := suite.getSearch( + requestingAccount, + token, + apiutil.APIv2, + user, + maxID, + minID, + limit, + offset, + query, + queryType, + resolve, + following, + expectedHTTPStatus, + expectedBody) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Len(searchResult.Accounts, 0) + suite.Len(searchResult.Statuses, 0) + suite.Len(searchResult.Hashtags, 1) +} + +func (suite *SearchGetTestSuite) TestSearchHashtagV2() { + var ( + requestingAccount = suite.testAccounts["local_account_1"] + token = suite.testTokens["local_account_1"] + user = suite.testUsers["local_account_1"] + maxID *string = nil + minID *string = nil + limit *int = nil + offset *int = nil + resolve *bool = nil + query = "#welcome" + queryType *string = func() *string { i := "hashtags"; return &i }() + following *bool = nil + expectedHTTPStatus = http.StatusOK + expectedBody = `{"accounts":[],"statuses":[],"hashtags":["welcome"]}` + ) + + searchResult, err := suite.getSearch( + requestingAccount, + token, + apiutil.APIv1, + user, + maxID, + minID, + limit, + offset, + query, + queryType, + resolve, + following, + expectedHTTPStatus, + expectedBody) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Len(searchResult.Accounts, 0) + suite.Len(searchResult.Statuses, 0) + suite.Len(searchResult.Hashtags, 1) +} + +func (suite *SearchGetTestSuite) TestSearchHashtagButWithAccountSearch() { + var ( + requestingAccount = suite.testAccounts["local_account_1"] + token = suite.testTokens["local_account_1"] + user = suite.testUsers["local_account_1"] + maxID *string = nil + minID *string = nil + limit *int = nil + offset *int = nil + resolve *bool = nil + query = "#welcome" + queryType *string = func() *string { i := "accounts"; return &i }() + following *bool = nil + expectedHTTPStatus = http.StatusOK + expectedBody = `` + ) + + searchResult, err := suite.getSearch( + requestingAccount, + token, + apiutil.APIv2, + user, + maxID, + minID, + limit, + offset, + query, + queryType, + resolve, + following, + expectedHTTPStatus, + expectedBody) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Len(searchResult.Accounts, 0) + suite.Len(searchResult.Statuses, 0) + suite.Len(searchResult.Hashtags, 0) +} + +func (suite *SearchGetTestSuite) TestSearchNotHashtagButWithTypeHashtag() { + var ( + requestingAccount = suite.testAccounts["local_account_1"] + token = suite.testTokens["local_account_1"] + user = suite.testUsers["local_account_1"] + maxID *string = nil + minID *string = nil + limit *int = nil + offset *int = nil + resolve *bool = nil + query = "welco" + queryType *string = func() *string { i := "hashtags"; return &i }() + following *bool = nil + expectedHTTPStatus = http.StatusOK + expectedBody = `` + ) + + searchResult, err := suite.getSearch( + requestingAccount, + token, + apiutil.APIv2, + user, + maxID, + minID, + limit, + offset, + query, + queryType, + resolve, + following, + expectedHTTPStatus, + expectedBody) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Len(searchResult.Accounts, 0) + suite.Len(searchResult.Statuses, 0) + suite.Len(searchResult.Hashtags, 1) +} + func TestSearchGetTestSuite(t *testing.T) { suite.Run(t, &SearchGetTestSuite{}) } diff --git a/internal/api/client/statuses/statuscreate_test.go b/internal/api/client/statuses/statuscreate_test.go index e84bcd816..05f24c24c 100644 --- a/internal/api/client/statuses/statuscreate_test.go +++ b/internal/api/client/statuses/statuscreate_test.go @@ -98,7 +98,6 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() { gtsTag := >smodel.Tag{} err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "helloworld"}}, gtsTag) suite.NoError(err) - suite.Equal(statusReply.Account.ID, gtsTag.FirstSeenFromAccountID) } func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() { diff --git a/internal/api/client/timelines/home.go b/internal/api/client/timelines/home.go index c3f075d5e..963096f59 100644 --- a/internal/api/client/timelines/home.go +++ b/internal/api/client/timelines/home.go @@ -133,9 +133,9 @@ func (m *Module) HomeTimelineGETHandler(c *gin.Context) { resp, errWithCode := m.processor.Timeline().HomeTimelineGet( c.Request.Context(), authed, - c.Query(MaxIDKey), - c.Query(SinceIDKey), - c.Query(MinIDKey), + c.Query(apiutil.MaxIDKey), + c.Query(apiutil.SinceIDKey), + c.Query(apiutil.MinIDKey), limit, local, ) diff --git a/internal/api/client/timelines/list.go b/internal/api/client/timelines/list.go index 8b4f7fad9..2e13e32cd 100644 --- a/internal/api/client/timelines/list.go +++ b/internal/api/client/timelines/list.go @@ -18,7 +18,6 @@ package timelines import ( - "errors" "net/http" "github.com/gin-gonic/gin" @@ -118,11 +117,9 @@ func (m *Module) ListTimelineGETHandler(c *gin.Context) { return } - targetListID := c.Param(IDKey) - if targetListID == "" { - err := errors.New("no list id specified") - apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1) - return + targetListID, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey)) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) } limit, errWithCode := apiutil.ParseLimit(c.Query(apiutil.LimitKey), 20, 40, 1) @@ -135,9 +132,9 @@ func (m *Module) ListTimelineGETHandler(c *gin.Context) { c.Request.Context(), authed, targetListID, - c.Query(MaxIDKey), - c.Query(SinceIDKey), - c.Query(MinIDKey), + c.Query(apiutil.MaxIDKey), + c.Query(apiutil.SinceIDKey), + c.Query(apiutil.MinIDKey), limit, ) if errWithCode != nil { diff --git a/internal/api/client/timelines/public.go b/internal/api/client/timelines/public.go index 96958e6a4..7b8acf1ca 100644 --- a/internal/api/client/timelines/public.go +++ b/internal/api/client/timelines/public.go @@ -144,9 +144,9 @@ func (m *Module) PublicTimelineGETHandler(c *gin.Context) { resp, errWithCode := m.processor.Timeline().PublicTimelineGet( c.Request.Context(), authed, - c.Query(MaxIDKey), - c.Query(SinceIDKey), - c.Query(MinIDKey), + c.Query(apiutil.MaxIDKey), + c.Query(apiutil.SinceIDKey), + c.Query(apiutil.MinIDKey), limit, local, ) diff --git a/internal/api/client/timelines/tag.go b/internal/api/client/timelines/tag.go new file mode 100644 index 000000000..58754705b --- /dev/null +++ b/internal/api/client/timelines/tag.go @@ -0,0 +1,146 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package timelines + +import ( + "net/http" + + "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// HomeTimelineGETHandler swagger:operation GET /api/v1/timelines/tag/{tag_name} tagTimeline +// +// See public statuses that use the given hashtag (case insensitive). +// +// The statuses will be returned in descending chronological order (newest first), with sequential IDs (bigger = newer). +// +// The returned Link header can be used to generate the previous and next queries when scrolling up or down a timeline. +// +// Example: +// +// ``` +// ; rel="next", ; rel="prev" +// ```` +// +// --- +// tags: +// - timelines +// +// produces: +// - application/json +// +// parameters: +// - +// name: max_id +// type: string +// description: >- +// Return only statuses *OLDER* than the given max status ID. +// The status with the specified ID will not be included in the response. +// in: query +// required: false +// - +// name: since_id +// type: string +// description: >- +// Return only statuses *newer* than the given since status ID. +// The status with the specified ID will not be included in the response. +// in: query +// - +// name: min_id +// type: string +// description: >- +// Return only statuses *immediately newer* than the given since status ID. +// The status with the specified ID will not be included in the response. +// in: query +// required: false +// - +// name: limit +// type: integer +// description: Number of statuses to return. +// default: 20 +// minimum: 1 +// maximum: 40 +// in: query +// required: false +// +// security: +// - OAuth2 Bearer: +// - read:statuses +// +// responses: +// '200': +// name: statuses +// description: Array of statuses. +// schema: +// type: array +// items: +// "$ref": "#/definitions/status" +// headers: +// Link: +// type: string +// description: Links to the next and previous queries. +// '401': +// description: unauthorized +// '400': +// description: bad request +func (m *Module) TagTimelineGETHandler(c *gin.Context) { + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1) + return + } + + if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1) + return + } + + limit, errWithCode := apiutil.ParseLimit(c.Query(apiutil.LimitKey), 20, 40, 1) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + tagName, errWithCode := apiutil.ParseTagName(c.Param(apiutil.TagNameKey)) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + resp, errWithCode := m.processor.Timeline().TagTimelineGet( + c.Request.Context(), + authed.Account, + tagName, + c.Query(apiutil.MaxIDKey), + c.Query(apiutil.SinceIDKey), + c.Query(apiutil.MinIDKey), + limit, + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + if resp.LinkHeader != "" { + c.Header("Link", resp.LinkHeader) + } + c.JSON(http.StatusOK, resp.Items) +} diff --git a/internal/api/client/timelines/timeline.go b/internal/api/client/timelines/timeline.go index 2580333d9..2362ca47e 100644 --- a/internal/api/client/timelines/timeline.go +++ b/internal/api/client/timelines/timeline.go @@ -21,28 +21,16 @@ "net/http" "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/processing" ) const ( - // BasePath is the base URI path for serving timelines, minus the 'api' prefix. - BasePath = "/v1/timelines" - IDKey = "id" - // HomeTimeline is the path for the home timeline - HomeTimeline = BasePath + "/home" - // PublicTimeline is the path for the public (and public local) timeline + BasePath = "/v1/timelines" + HomeTimeline = BasePath + "/home" PublicTimeline = BasePath + "/public" - ListTimeline = BasePath + "/list/:" + IDKey - // MaxIDKey is the url query for setting a max status ID to return - MaxIDKey = "max_id" - // SinceIDKey is the url query for returning results newer than the given ID - SinceIDKey = "since_id" - // MinIDKey is the url query for returning results immediately newer than the given ID - MinIDKey = "min_id" - // LimitKey is for specifying maximum number of results to return. - LimitKey = "limit" - // LocalKey is for specifying whether only local statuses should be returned - LocalKey = "local" + ListTimeline = BasePath + "/list/:" + apiutil.IDKey + TagTimeline = BasePath + "/tag/:" + apiutil.TagNameKey ) type Module struct { @@ -59,4 +47,5 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H attachHandler(http.MethodGet, HomeTimeline, m.HomeTimelineGETHandler) attachHandler(http.MethodGet, PublicTimeline, m.PublicTimelineGETHandler) attachHandler(http.MethodGet, ListTimeline, m.ListTimelineGETHandler) + attachHandler(http.MethodGet, TagTimeline, m.TagTimelineGETHandler) } diff --git a/internal/api/model/search.go b/internal/api/model/search.go index 664bf7b26..738c5911f 100644 --- a/internal/api/model/search.go +++ b/internal/api/model/search.go @@ -28,6 +28,7 @@ type SearchRequest struct { Resolve bool Following bool ExcludeUnreviewed bool + APIv1 bool // Set to 'true' if using version 1 of the search API. } // SearchResult models a search result. @@ -36,5 +37,6 @@ type SearchRequest struct { type SearchResult struct { Accounts []*Account `json:"accounts"` Statuses []*Status `json:"statuses"` - Hashtags []*Tag `json:"hashtags"` + // Slice of strings if api v1, slice of tags if api v2. + Hashtags []any `json:"hashtags"` } diff --git a/internal/api/model/tag.go b/internal/api/model/tag.go index 66b54d7f8..ebc12e2d4 100644 --- a/internal/api/model/tag.go +++ b/internal/api/model/tag.go @@ -27,4 +27,8 @@ type Tag struct { // Web link to the hashtag. // example: https://example.org/tags/helloworld URL string `json:"url"` + // History of this hashtag's usage. + // Currently just a stub, if provided will always be an empty array. + // example: [] + History *[]any `json:"history,omitempty"` } diff --git a/internal/api/util/parsequery.go b/internal/api/util/parsequery.go index 662870910..a87c77aeb 100644 --- a/internal/api/util/parsequery.go +++ b/internal/api/util/parsequery.go @@ -20,11 +20,18 @@ import ( "fmt" "strconv" + "strings" "github.com/superseriousbusiness/gotosocial/internal/gtserror" ) const ( + /* API version keys */ + + APIVersionKey = "api_version" + APIv1 = "v1" + APIv2 = "v2" + /* Common keys */ IDKey = "id" @@ -44,6 +51,10 @@ SearchResolveKey = "resolve" SearchTypeKey = "type" + /* Tag keys */ + + TagNameKey = "tag_name" + /* Web endpoint keys */ WebUsernameKey = "username" @@ -122,6 +133,26 @@ func ParseDomainBlockImport(value string, defaultValue bool) (bool, gtserror.Wit Parse functions for *REQUIRED* parameters. */ +func ParseAPIVersion(value string, availableVersion ...string) (string, gtserror.WithCode) { + key := APIVersionKey + + if value == "" { + return "", requiredError(key) + } + + for _, av := range availableVersion { + if value == av { + return value, nil + } + } + + err := fmt.Errorf( + "invalid API version, valid versions for this path are [%s]", + strings.Join(availableVersion, ", "), + ) + return "", gtserror.NewErrorBadRequest(err, err.Error()) +} + func ParseID(value string) (string, gtserror.WithCode) { key := IDKey @@ -152,6 +183,16 @@ func ParseSearchQuery(value string) (string, gtserror.WithCode) { return value, nil } +func ParseTagName(value string) (string, gtserror.WithCode) { + key := TagNameKey + + if value == "" { + return "", requiredError(key) + } + + return value, nil +} + func ParseWebUsername(value string) (string, gtserror.WithCode) { key := WebUsernameKey diff --git a/internal/cache/gts.go b/internal/cache/gts.go index fefd02fff..6014d13d4 100644 --- a/internal/cache/gts.go +++ b/internal/cache/gts.go @@ -47,6 +47,7 @@ type GTSCaches struct { report *result.Cache[*gtsmodel.Report] status *result.Cache[*gtsmodel.Status] statusFave *result.Cache[*gtsmodel.StatusFave] + tag *result.Cache[*gtsmodel.Tag] tombstone *result.Cache[*gtsmodel.Tombstone] user *result.Cache[*gtsmodel.User] @@ -78,6 +79,7 @@ func (c *GTSCaches) Init() { c.initReport() c.initStatus() c.initStatusFave() + c.initTag() c.initTombstone() c.initUser() c.initWebfinger() @@ -120,6 +122,7 @@ func (c *GTSCaches) Start() { tryStart(c.report, config.GetCacheGTSReportSweepFreq()) tryStart(c.status, config.GetCacheGTSStatusSweepFreq()) tryStart(c.statusFave, config.GetCacheGTSStatusFaveSweepFreq()) + tryStart(c.tag, config.GetCacheGTSTagSweepFreq()) tryStart(c.tombstone, config.GetCacheGTSTombstoneSweepFreq()) tryStart(c.user, config.GetCacheGTSUserSweepFreq()) tryUntil("starting *gtsmodel.Webfinger cache", 5, func() bool { @@ -167,6 +170,7 @@ func (c *GTSCaches) Stop() { tryStop(c.report, config.GetCacheGTSReportSweepFreq()) tryStop(c.status, config.GetCacheGTSStatusSweepFreq()) tryStop(c.statusFave, config.GetCacheGTSStatusFaveSweepFreq()) + tryStop(c.tag, config.GetCacheGTSTagSweepFreq()) tryStop(c.tombstone, config.GetCacheGTSTombstoneSweepFreq()) tryStop(c.user, config.GetCacheGTSUserSweepFreq()) tryUntil("stopping *gtsmodel.Webfinger cache", 5, func() bool { @@ -290,6 +294,11 @@ func (c *GTSCaches) StatusFave() *result.Cache[*gtsmodel.StatusFave] { return c.statusFave } +// Tag provides access to the gtsmodel Tag database cache. +func (c *GTSCaches) Tag() *result.Cache[*gtsmodel.Tag] { + return c.tag +} + // Tombstone provides access to the gtsmodel Tombstone database cache. func (c *GTSCaches) Tombstone() *result.Cache[*gtsmodel.Tombstone] { return c.tombstone @@ -568,6 +577,19 @@ func (c *GTSCaches) initStatusFave() { c.status.IgnoreErrors(ignoreErrors) } +func (c *GTSCaches) initTag() { + c.tag = result.New([]result.Lookup{ + {Name: "ID"}, + {Name: "Name"}, + }, func(m1 *gtsmodel.Tag) *gtsmodel.Tag { + m2 := new(gtsmodel.Tag) + *m2 = *m1 + return m2 + }, config.GetCacheGTSTagMaxSize()) + c.tag.SetTTL(config.GetCacheGTSTagTTL(), true) + c.tag.IgnoreErrors(ignoreErrors) +} + func (c *GTSCaches) initTombstone() { c.tombstone = result.New([]result.Lookup{ {Name: "ID"}, diff --git a/internal/config/config.go b/internal/config/config.go index 99b07358e..9397379b8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -266,6 +266,10 @@ type GTSCacheConfiguration struct { StatusFaveTTL time.Duration `name:"status-fave-ttl"` StatusFaveSweepFreq time.Duration `name:"status-fave-sweep-freq"` + TagMaxSize int `name:"tag-max-size"` + TagTTL time.Duration `name:"tag-ttl"` + TagSweepFreq time.Duration `name:"tag-sweep-freq"` + TombstoneMaxSize int `name:"tombstone-max-size"` TombstoneTTL time.Duration `name:"tombstone-ttl"` TombstoneSweepFreq time.Duration `name:"tombstone-sweep-freq"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index cb37838c1..7729840f0 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -211,6 +211,10 @@ StatusFaveTTL: time.Minute * 30, StatusFaveSweepFreq: time.Minute, + TagMaxSize: 2000, + TagTTL: time.Minute * 30, + TagSweepFreq: time.Minute, + TombstoneMaxSize: 500, TombstoneTTL: time.Minute * 30, TombstoneSweepFreq: time.Minute, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 1bf8ec2bc..e4b82edd5 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -3984,6 +3984,81 @@ func GetCacheGTSStatusFaveSweepFreq() time.Duration { return global.GetCacheGTSS // SetCacheGTSStatusFaveSweepFreq safely sets the value for global configuration 'Cache.GTS.StatusFaveSweepFreq' field func SetCacheGTSStatusFaveSweepFreq(v time.Duration) { global.SetCacheGTSStatusFaveSweepFreq(v) } +// GetCacheGTSTagMaxSize safely fetches the Configuration value for state's 'Cache.GTS.TagMaxSize' field +func (st *ConfigState) GetCacheGTSTagMaxSize() (v int) { + st.mutex.RLock() + v = st.config.Cache.GTS.TagMaxSize + st.mutex.RUnlock() + return +} + +// SetCacheGTSTagMaxSize safely sets the Configuration value for state's 'Cache.GTS.TagMaxSize' field +func (st *ConfigState) SetCacheGTSTagMaxSize(v int) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.GTS.TagMaxSize = v + st.reloadToViper() +} + +// CacheGTSTagMaxSizeFlag returns the flag name for the 'Cache.GTS.TagMaxSize' field +func CacheGTSTagMaxSizeFlag() string { return "cache-gts-tag-max-size" } + +// GetCacheGTSTagMaxSize safely fetches the value for global configuration 'Cache.GTS.TagMaxSize' field +func GetCacheGTSTagMaxSize() int { return global.GetCacheGTSTagMaxSize() } + +// SetCacheGTSTagMaxSize safely sets the value for global configuration 'Cache.GTS.TagMaxSize' field +func SetCacheGTSTagMaxSize(v int) { global.SetCacheGTSTagMaxSize(v) } + +// GetCacheGTSTagTTL safely fetches the Configuration value for state's 'Cache.GTS.TagTTL' field +func (st *ConfigState) GetCacheGTSTagTTL() (v time.Duration) { + st.mutex.RLock() + v = st.config.Cache.GTS.TagTTL + st.mutex.RUnlock() + return +} + +// SetCacheGTSTagTTL safely sets the Configuration value for state's 'Cache.GTS.TagTTL' field +func (st *ConfigState) SetCacheGTSTagTTL(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.GTS.TagTTL = v + st.reloadToViper() +} + +// CacheGTSTagTTLFlag returns the flag name for the 'Cache.GTS.TagTTL' field +func CacheGTSTagTTLFlag() string { return "cache-gts-tag-ttl" } + +// GetCacheGTSTagTTL safely fetches the value for global configuration 'Cache.GTS.TagTTL' field +func GetCacheGTSTagTTL() time.Duration { return global.GetCacheGTSTagTTL() } + +// SetCacheGTSTagTTL safely sets the value for global configuration 'Cache.GTS.TagTTL' field +func SetCacheGTSTagTTL(v time.Duration) { global.SetCacheGTSTagTTL(v) } + +// GetCacheGTSTagSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.TagSweepFreq' field +func (st *ConfigState) GetCacheGTSTagSweepFreq() (v time.Duration) { + st.mutex.RLock() + v = st.config.Cache.GTS.TagSweepFreq + st.mutex.RUnlock() + return +} + +// SetCacheGTSTagSweepFreq safely sets the Configuration value for state's 'Cache.GTS.TagSweepFreq' field +func (st *ConfigState) SetCacheGTSTagSweepFreq(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.GTS.TagSweepFreq = v + st.reloadToViper() +} + +// CacheGTSTagSweepFreqFlag returns the flag name for the 'Cache.GTS.TagSweepFreq' field +func CacheGTSTagSweepFreqFlag() string { return "cache-gts-tag-sweep-freq" } + +// GetCacheGTSTagSweepFreq safely fetches the value for global configuration 'Cache.GTS.TagSweepFreq' field +func GetCacheGTSTagSweepFreq() time.Duration { return global.GetCacheGTSTagSweepFreq() } + +// SetCacheGTSTagSweepFreq safely sets the value for global configuration 'Cache.GTS.TagSweepFreq' field +func SetCacheGTSTagSweepFreq(v time.Duration) { global.SetCacheGTSTagSweepFreq(v) } + // GetCacheGTSTombstoneMaxSize safely fetches the Configuration value for state's 'Cache.GTS.TombstoneMaxSize' field func (st *ConfigState) GetCacheGTSTombstoneMaxSize() (v int) { st.mutex.RLock() diff --git a/internal/db/bundb/basic.go b/internal/db/bundb/basic.go index 4991dcf69..33d6c6cb5 100644 --- a/internal/db/bundb/basic.go +++ b/internal/db/bundb/basic.go @@ -133,7 +133,6 @@ func (b *basicDB) CreateAllTables(ctx context.Context) error { >smodel.Mention{}, >smodel.Status{}, >smodel.StatusToEmoji{}, - >smodel.StatusToTag{}, >smodel.StatusFave{}, >smodel.StatusBookmark{}, >smodel.StatusMute{}, diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index 6a6ff2224..8387bb8d1 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -39,7 +39,6 @@ "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/tracing" @@ -77,6 +76,7 @@ type DBService struct { db.Status db.StatusBookmark db.StatusFave + db.Tag db.Timeline db.User db.Tombstone @@ -230,6 +230,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { db: db, state: state, }, + Tag: &tagDB{ + conn: db, + state: state, + }, Timeline: &timelineDB{ db: db, state: state, @@ -494,45 +498,3 @@ func sqlitePragmas(ctx context.Context, db *WrappedDB) error { return nil } - -/* - CONVERSION FUNCTIONS -*/ - -func (dbService *DBService) TagStringToTag(ctx context.Context, t string, originAccountID string) (*gtsmodel.Tag, error) { - protocol := config.GetProtocol() - host := config.GetHost() - now := time.Now() - - tag := >smodel.Tag{} - // we can use selectorinsert here to create the new tag if it doesn't exist already - // inserted will be true if this is a new tag we just created - if err := dbService.db.NewSelect().Model(tag).Where("LOWER(?) = LOWER(?)", bun.Ident("name"), t).Scan(ctx); err != nil && err != sql.ErrNoRows { - return nil, fmt.Errorf("error getting tag with name %s: %s", t, err) - } - - if tag.ID == "" { - // tag doesn't exist yet so populate it - newID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - tag.ID = newID - tag.URL = protocol + "://" + host + "/tags/" + t - tag.Name = t - tag.FirstSeenFromAccountID = originAccountID - tag.CreatedAt = now - tag.UpdatedAt = now - useable := true - tag.Useable = &useable - listable := true - tag.Listable = &listable - } - - // bail already if the tag isn't useable - if !*tag.Useable { - return nil, fmt.Errorf("tag %s is not useable", t) - } - tag.LastStatusAt = now - return tag, nil -} diff --git a/internal/db/bundb/bundb_test.go b/internal/db/bundb/bundb_test.go index d608f7bc4..0cdbb5cce 100644 --- a/internal/db/bundb/bundb_test.go +++ b/internal/db/bundb/bundb_test.go @@ -84,5 +84,7 @@ func (suite *BunDBStandardTestSuite) SetupTest() { } func (suite *BunDBStandardTestSuite) TearDownTest() { - testrig.StandardDBTeardown(suite.db) + if suite.db != nil { + testrig.StandardDBTeardown(suite.db) + } } diff --git a/internal/db/bundb/migrations/20230718161520_hashtaggery.go b/internal/db/bundb/migrations/20230718161520_hashtaggery.go new file mode 100644 index 000000000..1b2c8edc9 --- /dev/null +++ b/internal/db/bundb/migrations/20230718161520_hashtaggery.go @@ -0,0 +1,76 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package migrations + +import ( + "context" + + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Drop now unused columns from tags table. + for _, column := range []string{ + "url", + "first_seen_from_account_id", + "last_status_at", + } { + if _, err := tx. + NewDropColumn(). + Table("tags"). + Column(column). + Exec(ctx); err != nil { + return err + } + } + + // Index status_to_tags table properly. + for index, columns := range map[string][]string{ + // Index for tag timeline paging. + "status_to_tags_tag_timeline_idx": {"tag_id", "status_id"}, + // These indexes were only implicit + // before, make them explicit now. + "status_to_tags_tag_id_idx": {"tag_id"}, + "status_to_tags_status_id_idx": {"status_id"}, + } { + if _, err := tx. + NewCreateIndex(). + Table("status_to_tags"). + Index(index). + Column(columns...). + Exec(ctx); err != nil { + return err + } + } + + return nil + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/search.go b/internal/db/bundb/search.go index f4e41d0f4..755f60e7d 100644 --- a/internal/db/bundb/search.go +++ b/internal/db/bundb/search.go @@ -19,6 +19,7 @@ import ( "context" + "strings" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" @@ -385,3 +386,101 @@ func (s *searchDB) statusText() *bun.SelectQuery { return statusText } + +// Query example (SQLite): +// +// SELECT "tag"."id" FROM "tags" AS "tag" +// WHERE ("tag"."id" < 'ZZZZZZZZZZZZZZZZZZZZZZZZZZ') +// AND (("tag"."name") LIKE 'welcome%' ESCAPE '\') +// ORDER BY "tag"."id" DESC LIMIT 10 +func (s *searchDB) SearchForTags( + ctx context.Context, + query string, + maxID string, + minID string, + limit int, + offset int, +) ([]*gtsmodel.Tag, error) { + // Ensure reasonable + if limit < 0 { + limit = 0 + } + + // Make educated guess for slice size + var ( + tagIDs = make([]string, 0, limit) + frontToBack = true + ) + + q := s.db. + NewSelect(). + TableExpr("? AS ?", bun.Ident("tags"), bun.Ident("tag")). + // Select only IDs from table + Column("tag.id") + + // Return only items with a LOWER id than maxID. + if maxID == "" { + maxID = id.Highest + } + q = q.Where("? < ?", bun.Ident("tag.id"), maxID) + + if minID != "" { + // return only tags HIGHER (ie., newer) than minID + q = q.Where("? > ?", bun.Ident("tag.id"), minID) + + // page up + frontToBack = false + } + + // Normalize tag 'name' string. + name := strings.TrimSpace(query) + name = strings.ToLower(name) + + // Search using LIKE for tags that start with `name`. + q = whereStartsLike(q, bun.Ident("tag.name"), name) + + if limit > 0 { + // Limit amount of tags returned. + q = q.Limit(limit) + } + + if frontToBack { + // Page down. + q = q.Order("tag.id DESC") + } else { + // Page up. + q = q.Order("tag.id ASC") + } + + if err := q.Scan(ctx, &tagIDs); err != nil { + return nil, s.db.ProcessError(err) + } + + if len(tagIDs) == 0 { + return nil, nil + } + + // If we're paging up, we still want tags + // to be sorted by ID desc, so reverse slice. + // https://zchee.github.io/golang-wiki/SliceTricks/#reversing + if !frontToBack { + for l, r := 0, len(tagIDs)-1; l < r; l, r = l+1, r-1 { + tagIDs[l], tagIDs[r] = tagIDs[r], tagIDs[l] + } + } + + tags := make([]*gtsmodel.Tag, 0, len(tagIDs)) + for _, id := range tagIDs { + // Fetch tag from db for ID + tag, err := s.state.DB.GetTag(ctx, id) + if err != nil { + log.Errorf(ctx, "error fetching tag %q: %v", id, err) + continue + } + + // Append status to slice + tags = append(tags, tag) + } + + return tags, nil +} diff --git a/internal/db/bundb/search_test.go b/internal/db/bundb/search_test.go index d670c90d6..f84704df2 100644 --- a/internal/db/bundb/search_test.go +++ b/internal/db/bundb/search_test.go @@ -77,6 +77,23 @@ func (suite *SearchTestSuite) TestSearchStatuses() { suite.Len(statuses, 1) } +func (suite *SearchTestSuite) TestSearchTags() { + // Search with full tag string. + tags, err := suite.db.SearchForTags(context.Background(), "welcome", "", "", 10, 0) + suite.NoError(err) + suite.Len(tags, 1) + + // Search with partial tag string. + tags, err = suite.db.SearchForTags(context.Background(), "wel", "", "", 10, 0) + suite.NoError(err) + suite.Len(tags, 1) + + // Search with end of tag string. + tags, err = suite.db.SearchForTags(context.Background(), "come", "", "", 10, 0) + suite.NoError(err) + suite.Len(tags, 0) +} + func TestSearchTestSuite(t *testing.T) { suite.Run(t, new(SearchTestSuite)) } diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 4dc7d8468..0fef01736 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -214,9 +214,16 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) } } - // TODO: once we don't fetch using relations. - // if !status.TagsPopulated() { - // } + if !status.TagsPopulated() { + // Status tags are out-of-date with IDs, repopulate. + status.Tags, err = s.state.DB.GetTags( + ctx, + status.TagIDs, + ) + if err != nil { + errs.Append(fmt.Errorf("error populating status tags: %w", err)) + } + } if !status.MentionsPopulated() { // Status mentions are out-of-date with IDs, repopulate. diff --git a/internal/db/bundb/tag.go b/internal/db/bundb/tag.go new file mode 100644 index 000000000..043af5728 --- /dev/null +++ b/internal/db/bundb/tag.go @@ -0,0 +1,119 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb + +import ( + "context" + "strings" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/uptrace/bun" +) + +type tagDB struct { + conn *WrappedDB + state *state.State +} + +func (m *tagDB) GetTag(ctx context.Context, id string) (*gtsmodel.Tag, error) { + return m.state.Caches.GTS.Tag().Load("ID", func() (*gtsmodel.Tag, error) { + var tag gtsmodel.Tag + + q := m.conn. + NewSelect(). + Model(&tag). + Where("? = ?", bun.Ident("tag.id"), id) + + if err := q.Scan(ctx); err != nil { + return nil, m.conn.ProcessError(err) + } + + return &tag, nil + }, id) +} + +func (m *tagDB) GetTagByName(ctx context.Context, name string) (*gtsmodel.Tag, error) { + // Normalize 'name' string. + name = strings.TrimSpace(name) + name = strings.ToLower(name) + + return m.state.Caches.GTS.Tag().Load("Name", func() (*gtsmodel.Tag, error) { + var tag gtsmodel.Tag + + q := m.conn. + NewSelect(). + Model(&tag). + Where("? = ?", bun.Ident("tag.name"), name) + + if err := q.Scan(ctx); err != nil { + return nil, m.conn.ProcessError(err) + } + + return &tag, nil + }, name) +} + +func (m *tagDB) GetTags(ctx context.Context, ids []string) ([]*gtsmodel.Tag, error) { + tags := make([]*gtsmodel.Tag, 0, len(ids)) + + for _, id := range ids { + // Attempt fetch from DB + tag, err := m.GetTag(ctx, id) + if err != nil { + log.Errorf(ctx, "error getting tag %q: %v", id, err) + continue + } + + // Append tag + tags = append(tags, tag) + } + + return tags, nil +} + +func (m *tagDB) PutTag(ctx context.Context, tag *gtsmodel.Tag) error { + // Normalize 'name' string before it enters + // the db, without changing tag we were given. + // + // First copy tag to new pointer. + t2 := new(gtsmodel.Tag) + *t2 = *tag + + // Normalize name on new pointer. + t2.Name = strings.TrimSpace(t2.Name) + t2.Name = strings.ToLower(t2.Name) + + // Insert the copy. + if err := m.state.Caches.GTS.Tag().Store(t2, func() error { + _, err := m.conn.NewInsert().Model(t2).Exec(ctx) + return m.conn.ProcessError(err) + }); err != nil { + return err // err already processed + } + + // Update original tag with + // field values populated by db. + tag.CreatedAt = t2.CreatedAt + tag.UpdatedAt = t2.UpdatedAt + tag.Useable = t2.Useable + tag.Listable = t2.Listable + + return nil +} diff --git a/internal/db/bundb/tag_test.go b/internal/db/bundb/tag_test.go new file mode 100644 index 000000000..324398d27 --- /dev/null +++ b/internal/db/bundb/tag_test.go @@ -0,0 +1,91 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" +) + +type TagTestSuite struct { + BunDBStandardTestSuite +} + +func (suite *TagTestSuite) TestGetTag() { + testTag := suite.testTags["welcome"] + + dbTag, err := suite.db.GetTag(context.Background(), testTag.ID) + suite.NoError(err) + suite.NotNil(dbTag) + suite.Equal(testTag.ID, dbTag.ID) +} + +func (suite *TagTestSuite) TestGetTagByName() { + testTag := suite.testTags["welcome"] + + // Name is normalized when doing + // selects from the db, so these + // should all yield the same result. + for _, name := range []string{ + "WELCOME", + "welcome", + "Welcome", + "WELCoME ", + } { + dbTag, err := suite.db.GetTagByName(context.Background(), name) + suite.NoError(err) + suite.NotNil(dbTag) + suite.Equal(testTag.ID, dbTag.ID) + } +} + +func (suite *TagTestSuite) TestPutTag() { + // Name is normalized when doing + // inserts to the db, so these + // should all yield the same result. + for i, name := range []string{ + "NewTag", + "newtag", + "NEWtag", + "NEWTAG ", + } { + err := suite.db.PutTag(context.Background(), >smodel.Tag{ + ID: id.NewULID(), + Name: name, + }) + if i == 0 { + // This is the first one, so it + // should have just been created. + suite.NoError(err) + continue + } + + // Subsequent inserts should fail + // since all these tags are equivalent. + suite.ErrorIs(err, db.ErrAlreadyExists) + } +} + +func TestTagTestSuite(t *testing.T) { + suite.Run(t, new(TagTestSuite)) +} diff --git a/internal/db/bundb/timeline.go b/internal/db/bundb/timeline.go index 6aa4989d9..62f1f642d 100644 --- a/internal/db/bundb/timeline.go +++ b/internal/db/bundb/timeline.go @@ -410,3 +410,111 @@ func (t *timelineDB) GetListTimeline( return statuses, nil } + +func (t *timelineDB) GetTagTimeline( + ctx context.Context, + tagID string, + maxID string, + sinceID string, + minID string, + limit int, +) ([]*gtsmodel.Status, error) { + // Ensure reasonable + if limit < 0 { + limit = 0 + } + + // Make educated guess for slice size + var ( + statusIDs = make([]string, 0, limit) + frontToBack = true + ) + + q := t.db. + NewSelect(). + TableExpr("? AS ?", bun.Ident("status_to_tags"), bun.Ident("status_to_tag")). + Column("status_to_tag.status_id"). + // Join with statuses for filtering. + Join( + "INNER JOIN ? AS ? ON ? = ?", + bun.Ident("statuses"), bun.Ident("status"), + bun.Ident("status.id"), bun.Ident("status_to_tag.status_id"), + ). + // Public only. + Where("? = ?", bun.Ident("status.visibility"), gtsmodel.VisibilityPublic). + // This tag only. + Where("? = ?", bun.Ident("status_to_tag.tag_id"), tagID) + + if maxID == "" || maxID >= id.Highest { + const future = 24 * time.Hour + + var err error + + // don't return statuses more than 24hr in the future + maxID, err = id.NewULIDFromTime(time.Now().Add(future)) + if err != nil { + return nil, err + } + } + + // return only statuses LOWER (ie., older) than maxID + q = q.Where("? < ?", bun.Ident("status_to_tag.status_id"), maxID) + + if sinceID != "" { + // return only statuses HIGHER (ie., newer) than sinceID + q = q.Where("? > ?", bun.Ident("status_to_tag.status_id"), sinceID) + } + + if minID != "" { + // return only statuses HIGHER (ie., newer) than minID + q = q.Where("? > ?", bun.Ident("status_to_tag.status_id"), minID) + + // page up + frontToBack = false + } + + if limit > 0 { + // limit amount of statuses returned + q = q.Limit(limit) + } + + if frontToBack { + // Page down. + q = q.Order("status_to_tag.status_id DESC") + } else { + // Page up. + q = q.Order("status_to_tag.status_id ASC") + } + + if err := q.Scan(ctx, &statusIDs); err != nil { + return nil, t.db.ProcessError(err) + } + + if len(statusIDs) == 0 { + return nil, nil + } + + // If we're paging up, we still want statuses + // to be sorted by ID desc, so reverse ids slice. + // https://zchee.github.io/golang-wiki/SliceTricks/#reversing + if !frontToBack { + for l, r := 0, len(statusIDs)-1; l < r; l, r = l+1, r-1 { + statusIDs[l], statusIDs[r] = statusIDs[r], statusIDs[l] + } + } + + statuses := make([]*gtsmodel.Status, 0, len(statusIDs)) + for _, id := range statusIDs { + // Fetch status from db for ID + status, err := t.state.DB.GetStatusByID(ctx, id) + if err != nil { + log.Errorf(ctx, "error fetching status %q: %v", id, err) + continue + } + + // Append status to slice + statuses = append(statuses, status) + } + + return statuses, nil +} diff --git a/internal/db/bundb/timeline_test.go b/internal/db/bundb/timeline_test.go index 7e8fd0838..43407bc69 100644 --- a/internal/db/bundb/timeline_test.go +++ b/internal/db/bundb/timeline_test.go @@ -272,6 +272,21 @@ func (suite *TimelineTestSuite) TestGetListTimelineMinIDPagingUp() { suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", s[len(s)-1].ID) } +func (suite *TimelineTestSuite) TestGetTagTimelineNoParams() { + var ( + ctx = context.Background() + tag = suite.testTags["welcome"] + ) + + s, err := suite.db.GetTagTimeline(ctx, tag.ID, "", "", "", 1) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.checkStatuses(s, id.Highest, id.Lowest, 1) + suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", s[0].ID) +} + func TestTimelineTestSuite(t *testing.T) { suite.Run(t, new(TimelineTestSuite)) } diff --git a/internal/db/bundb/util.go b/internal/db/bundb/util.go index bdd45d1e7..3c3249daf 100644 --- a/internal/db/bundb/util.go +++ b/internal/db/bundb/util.go @@ -34,9 +34,10 @@ `_`, `\_`, // Exactly one char. ) -// whereSubqueryLike appends a WHERE clause to the -// given SelectQuery, which searches for matches -// of `search` in the given subQuery using LIKE. +// whereLike appends a WHERE clause to the +// given SelectQuery, which searches for +// matches of `search` in the given subQuery +// using LIKE. func whereLike( query *bun.SelectQuery, subject interface{}, @@ -58,6 +59,30 @@ func whereLike( ) } +// whereStartsLike is like whereLike, +// but only searches for strings that +// START WITH `search`. +func whereStartsLike( + query *bun.SelectQuery, + subject interface{}, + search string, +) *bun.SelectQuery { + // Escape existing wildcard + escape + // chars in the search query string. + search = likeEscaper.Replace(search) + + // Add our own wildcards back in; search + // zero or more chars after the query. + search += `%` + + // Append resulting WHERE + // clause to the main query. + return query.Where( + "(?) LIKE ? ESCAPE ?", + subject, search, `\`, + ) +} + // updateWhere parses []db.Where and adds it to the given update query. func updateWhere(q *bun.UpdateQuery, where []db.Where) { for _, w := range where { diff --git a/internal/db/db.go b/internal/db/db.go index 370dab38b..7c00050ff 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -17,12 +17,6 @@ package db -import ( - "context" - - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - const ( // DBTypePostgres represents an underlying POSTGRES database type. DBTypePostgres string = "POSTGRES" @@ -48,20 +42,8 @@ type DB interface { Status StatusBookmark StatusFave + Tag Timeline User Tombstone - - /* - USEFUL CONVERSION FUNCTIONS - */ - - // TagStringToTag takes a lowercase tag in the form "somehashtag", which has been - // used in a status. It takes the id of the account that wrote the status, and the id of the status itself, and then - // returns an *apimodel.Tag corresponding to the given tags. If the tag already exists in database, that tag - // will be returned. Otherwise a pointer to a new tag struct will be created and returned. - // - // Note: this func doesn't/shouldn't do any manipulation of tags in the DB, it's just for checking - // if they exist in the db already, and conveniently returning them, or creating new tag structs. - TagStringToTag(ctx context.Context, tag string, originAccountID string) (*gtsmodel.Tag, error) } diff --git a/internal/db/search.go b/internal/db/search.go index b2ade0cfe..d2ffe4ad5 100644 --- a/internal/db/search.go +++ b/internal/db/search.go @@ -29,4 +29,7 @@ type Search interface { // SearchForStatuses uses the given query text to search for statuses created by accountID, or in reply to accountID. SearchForStatuses(ctx context.Context, accountID string, query string, maxID string, minID string, limit int, offset int) ([]*gtsmodel.Status, error) + + // SearchForTags searches for tags that start with the given query text (case insensitive). + SearchForTags(ctx context.Context, query string, maxID string, minID string, limit int, offset int) ([]*gtsmodel.Tag, error) } diff --git a/internal/db/tag.go b/internal/db/tag.go new file mode 100644 index 000000000..c0642f5a4 --- /dev/null +++ b/internal/db/tag.go @@ -0,0 +1,39 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package db + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// Tag contains functions for getting/creating tags in the database. +type Tag interface { + // GetTag gets a single tag by ID + GetTag(ctx context.Context, id string) (*gtsmodel.Tag, error) + + // GetTagByName gets a single tag using the given name. + GetTagByName(ctx context.Context, name string) (*gtsmodel.Tag, error) + + // PutTag inserts the given tag in the database. + PutTag(ctx context.Context, tag *gtsmodel.Tag) error + + // GetTags gets multiple tags. + GetTags(ctx context.Context, ids []string) ([]*gtsmodel.Tag, error) +} diff --git a/internal/db/timeline.go b/internal/db/timeline.go index 40d5b8015..43ac655d0 100644 --- a/internal/db/timeline.go +++ b/internal/db/timeline.go @@ -48,4 +48,8 @@ type Timeline interface { // GetListTimeline returns a slice of statuses from followed accounts collected within the list with the given listID. // Statuses should be returned in descending order of when they were created (newest first). GetListTimeline(ctx context.Context, listID string, maxID string, sinceID string, minID string, limit int) ([]*gtsmodel.Status, error) + + // GetTagTimeline returns a slice of public-visibility statuses that use the given tagID. + // Statuses should be returned in descending order of when they were created (newest first). + GetTagTimeline(ctx context.Context, tagID string, maxID string, sinceID string, minID string, limit int) ([]*gtsmodel.Status, error) } diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 4525f64a9..8586884eb 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -288,7 +288,10 @@ func (d *deref) enrichStatus( return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err) } - // TODO: populateStatusTags() + // Ensure the status' tags are populated. + if err := d.fetchStatusTags(ctx, requestUser, latestStatus); err != nil { + return nil, nil, gtserror.Newf("error populating tags for status %s: %w", uri, err) + } // Ensure the status' media attachments are populated, passing in existing to check for changes. if err := d.fetchStatusAttachments(ctx, tsport, status, latestStatus); err != nil { @@ -400,6 +403,55 @@ func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, exi return nil } +func (d *deref) fetchStatusTags(ctx context.Context, requestUser string, status *gtsmodel.Status) error { + // Allocate new slice to take the yet-to-be determined tag IDs. + status.TagIDs = make([]string, len(status.Tags)) + + for i := range status.Tags { + placeholder := status.Tags[i] + + // Look for existing tag with this name first. + tag, err := d.state.DB.GetTagByName(ctx, placeholder.Name) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + log.Errorf(ctx, "db error getting tag %s: %v", tag.Name, err) + continue + } + + // No tag with this name yet, create it. + if tag == nil { + tag = >smodel.Tag{ + ID: id.NewULID(), + Name: placeholder.Name, + } + + if err := d.state.DB.PutTag(ctx, tag); err != nil { + log.Errorf(ctx, "db error putting tag %s: %v", tag.Name, err) + continue + } + } + + // Set the *new* tag and ID. + status.Tags[i] = tag + status.TagIDs[i] = tag.ID + } + + // Remove any tag we couldn't get or create. + for i := 0; i < len(status.TagIDs); { + if status.TagIDs[i] == "" { + // This is a failed tag population, likely due + // to some database peculiarity / race condition. + copy(status.Tags[i:], status.Tags[i+1:]) + copy(status.TagIDs[i:], status.TagIDs[i+1:]) + status.Tags = status.Tags[:len(status.Tags)-1] + status.TagIDs = status.TagIDs[:len(status.TagIDs)-1] + continue + } + i++ + } + + return nil +} + func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing, status *gtsmodel.Status) error { // Allocate new slice to take the yet-to-be fetched attachment IDs. status.AttachmentIDs = make([]string, len(status.Attachments)) diff --git a/internal/federation/dereferencing/status_test.go b/internal/federation/dereferencing/status_test.go index 9ec77fbcc..e9cdbcff5 100644 --- a/internal/federation/dereferencing/status_test.go +++ b/internal/federation/dereferencing/status_test.go @@ -123,6 +123,56 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() { suite.False(*m.Silent) } +func (suite *StatusTestSuite) TestDereferenceStatusWithTag() { + fetchingAccount := suite.testAccounts["local_account_1"] + + statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7") + status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL) + suite.NoError(err) + suite.NotNil(status) + + // status values should be set + suite.Equal("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7", status.URI) + suite.Equal("https://unknown-instance.com/users/@brand_new_person/01H641QSRS3TCXSVC10X4GPKW7", status.URL) + suite.Equal("

Babe are you okay, you've hardly touched your #piss

", status.Content) + suite.Equal("https://unknown-instance.com/users/brand_new_person", status.AccountURI) + suite.False(*status.Local) + suite.Empty(status.ContentWarning) + suite.Equal(gtsmodel.VisibilityPublic, status.Visibility) + suite.Equal(ap.ObjectNote, status.ActivityStreamsType) + + // Ensure tags set + ID'd. + suite.Len(status.Tags, 1) + suite.Len(status.TagIDs, 1) + + // status should be in the database + dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI) + suite.NoError(err) + suite.Equal(status.ID, dbStatus.ID) + suite.True(*dbStatus.Federated) + suite.True(*dbStatus.Boostable) + suite.True(*dbStatus.Replyable) + suite.True(*dbStatus.Likeable) + + // account should be in the database now too + account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI) + suite.NoError(err) + suite.NotNil(account) + suite.True(*account.Discoverable) + suite.Equal("https://unknown-instance.com/users/brand_new_person", account.URI) + suite.Equal("hey I'm a new person, your instance hasn't seen me yet uwu", account.Note) + suite.Equal("Geoff Brando New Personson", account.DisplayName) + suite.Equal("brand_new_person", account.Username) + suite.NotNil(account.PublicKey) + suite.Nil(account.PrivateKey) + + // we should have a tag in the database + t := >smodel.Tag{} + err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "piss"}}, t) + suite.NoError(err) + suite.NotNil(t) +} + func (suite *StatusTestSuite) TestDereferenceStatusWithImageAndNoContent() { fetchingAccount := suite.testAccounts["local_account_1"] diff --git a/internal/federation/federatingactor_test.go b/internal/federation/federatingactor_test.go index cdc265d82..f0010152f 100644 --- a/internal/federation/federatingactor_test.go +++ b/internal/federation/federatingactor_test.go @@ -50,6 +50,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() { false, nil, nil, + nil, ) testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), time.Now(), testNote) @@ -97,6 +98,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { false, nil, nil, + nil, ) testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), testNote) diff --git a/internal/gtsmodel/tag.go b/internal/gtsmodel/tag.go index f8aff1dc1..a43c4a5ec 100644 --- a/internal/gtsmodel/tag.go +++ b/internal/gtsmodel/tag.go @@ -21,13 +21,10 @@ // Tag represents a hashtag for gathering public statuses together. type Tag struct { - ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database - CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated - URL string `validate:"required,url" bun:",nullzero,notnull"` // Href/web address of this tag, eg https://example.org/tags/somehashtag - Name string `validate:"required" bun:",unique,nullzero,notnull"` // name of this tag -- the tag without the hash part - FirstSeenFromAccountID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Which account ID is the first one we saw using this tag? - Useable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // can our instance users use this tag? - Listable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // can our instance users look up this tag? - LastStatusAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was this tag last used? + ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database + CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + Name string `validate:"required" bun:",unique,nullzero,notnull"` // (lowercase) name of the tag without the hash prefix + Useable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // Tag is useable on this instance. + Listable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // Tagged statuses can be listed on this instance. } diff --git a/internal/processing/search/get.go b/internal/processing/search/get.go index aaade8908..8e1881ab1 100644 --- a/internal/processing/search/get.go +++ b/internal/processing/search/get.go @@ -33,6 +33,7 @@ "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -108,14 +109,16 @@ func (p *Processor) Get( // supply an offset greater than 0, return nothing as // though there were no additional results. if req.Offset > 0 { - return p.packageSearchResult(ctx, account, nil, nil) + return p.packageSearchResult(ctx, account, nil, nil, nil, req.APIv1) } var ( foundStatuses = make([]*gtsmodel.Status, 0, limit) foundAccounts = make([]*gtsmodel.Account, 0, limit) - appendStatus = func(foundStatus *gtsmodel.Status) { foundStatuses = append(foundStatuses, foundStatus) } - appendAccount = func(foundAccount *gtsmodel.Account) { foundAccounts = append(foundAccounts, foundAccount) } + foundTags = make([]*gtsmodel.Tag, 0, limit) + appendStatus = func(s *gtsmodel.Status) { foundStatuses = append(foundStatuses, s) } + appendAccount = func(a *gtsmodel.Account) { foundAccounts = append(foundAccounts, a) } + appendTag = func(t *gtsmodel.Tag) { foundTags = append(foundTags, t) } keepLooking bool err error ) @@ -162,6 +165,8 @@ func (p *Processor) Get( account, foundAccounts, foundStatuses, + foundTags, + req.APIv1, ) } } @@ -189,6 +194,48 @@ func (p *Processor) Get( account, foundAccounts, foundStatuses, + foundTags, + req.APIv1, + ) + } + + // If query looks like a hashtag (ie., starts + // with '#'), then search for tags. + // + // Since '#' is a very unique prefix and isn't + // shared among account or status searches, we + // can save a bit of time by searching for this + // now, and bailing quickly if we get no results, + // or we're not allowed to include hashtags in + // search results. + // + // We know that none of the subsequent searches + // would show any good results either, and those + // searches are *much* more expensive. + keepLooking, err = p.hashtag( + ctx, + maxID, + minID, + limit, + offset, + query, + queryType, + appendTag, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err = gtserror.Newf("error searching for hashtag: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + if !keepLooking { + // Return whatever we have. + return p.packageSearchResult( + ctx, + account, + foundAccounts, + foundStatuses, + foundTags, + req.APIv1, ) } @@ -218,6 +265,8 @@ func (p *Processor) Get( account, foundAccounts, foundStatuses, + foundTags, + req.APIv1, ) } @@ -559,6 +608,80 @@ func (p *Processor) statusByURI( return nil, gtserror.SetUnretrievable(err) } +func (p *Processor) hashtag( + ctx context.Context, + maxID string, + minID string, + limit int, + offset int, + query string, + queryType string, + appendTag func(*gtsmodel.Tag), +) (bool, error) { + if query[0] != '#' { + // Query doesn't look like a hashtag, + // but if we're being instructed to + // look explicitly *only* for hashtags, + // let's be generous and assume caller + // just left out the hash prefix. + + if queryType != queryTypeHashtags { + // Nope, search isn't explicitly + // for hashtags, keep looking. + return true, nil + } + + // Search is explicitly for + // tags, let this one through. + } else if !includeHashtags(queryType) { + // Query looks like a hashtag, + // but we're not meant to include + // hashtags in the results. + // + // Indicate to caller they should + // stop looking, since they're not + // going to get results for this by + // looking in any other way. + return false, nil + } + + // Query looks like a hashtag, and we're allowed + // to search for hashtags. + // + // Ensure this is a valid tag for our instance. + normalized, ok := text.NormalizeHashtag(query) + if !ok { + // Couldn't normalize/not a + // valid hashtag after all. + // Caller should stop looking. + return false, nil + } + + // Search for tags starting with the normalized string. + tags, err := p.state.DB.SearchForTags( + ctx, + normalized, + maxID, + minID, + limit, + offset, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf( + "error checking database for tags using text %s: %w", + normalized, err, + ) + return false, err + } + + // Return whatever we got. + for _, tag := range tags { + appendTag(tag) + } + + return false, nil +} + // byText searches in the database for accounts and/or // statuses containing the given query string, using // the provided parameters. diff --git a/internal/processing/search/util.go b/internal/processing/search/util.go index 4172e4e1a..171d0e570 100644 --- a/internal/processing/search/util.go +++ b/internal/processing/search/util.go @@ -36,6 +36,11 @@ func includeStatuses(queryType string) bool { return queryType == queryTypeAny || queryType == queryTypeStatuses } +// return true if given queryType should include hashtags. +func includeHashtags(queryType string) bool { + return queryType == queryTypeAny || queryType == queryTypeHashtags +} + // packageAccounts is a util function that just // converts the given accounts into an apimodel // account slice, or errors appropriately. @@ -111,14 +116,59 @@ func (p *Processor) packageStatuses( return apiStatuses, nil } +// packageHashtags is a util function that just +// converts the given hashtags into an apimodel +// hashtag slice, or errors appropriately. +func (p *Processor) packageHashtags( + ctx context.Context, + requestingAccount *gtsmodel.Account, + tags []*gtsmodel.Tag, + v1 bool, +) ([]any, gtserror.WithCode) { + apiTags := make([]any, 0, len(tags)) + + var rangeF func(*gtsmodel.Tag) + if v1 { + // If API version 1, just provide slice of tag names. + rangeF = func(tag *gtsmodel.Tag) { + apiTags = append(apiTags, tag.Name) + } + } else { + // If API not version 1, provide slice of full tags. + rangeF = func(tag *gtsmodel.Tag) { + apiTag, err := p.tc.TagToAPITag(ctx, tag, true) + if err != nil { + log.Debugf( + ctx, + "skipping tag %s because it couldn't be converted to its api representation: %s", + tag.Name, err, + ) + return + } + + apiTags = append(apiTags, &apiTag) + } + } + + for _, tag := range tags { + rangeF(tag) + } + + return apiTags, nil +} + // packageSearchResult wraps up the given accounts // and statuses into an apimodel SearchResult that // can be serialized to an API caller as JSON. +// +// Set v1 to 'true' if the search is using v1 of the API. func (p *Processor) packageSearchResult( ctx context.Context, requestingAccount *gtsmodel.Account, accounts []*gtsmodel.Account, statuses []*gtsmodel.Status, + tags []*gtsmodel.Tag, + v1 bool, ) (*apimodel.SearchResult, gtserror.WithCode) { apiAccounts, errWithCode := p.packageAccounts(ctx, requestingAccount, accounts) if errWithCode != nil { @@ -130,9 +180,14 @@ func (p *Processor) packageSearchResult( return nil, errWithCode } + apiTags, errWithCode := p.packageHashtags(ctx, requestingAccount, tags, v1) + if errWithCode != nil { + return nil, errWithCode + } + return &apimodel.SearchResult{ Accounts: apiAccounts, Statuses: apiStatuses, - Hashtags: make([]*apimodel.Tag, 0), + Hashtags: apiTags, }, nil } diff --git a/internal/processing/timeline/tag.go b/internal/processing/timeline/tag.go new file mode 100644 index 000000000..943aa1722 --- /dev/null +++ b/internal/processing/timeline/tag.go @@ -0,0 +1,141 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package timeline + +import ( + "context" + "errors" + "fmt" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/text" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// TagTimelineGet gets a pageable timeline for the given +// tagName and given paging parameters. It will ensure +// that each status in the timeline is actually visible +// to requestingAcct before returning it. +func (p *Processor) TagTimelineGet( + ctx context.Context, + requestingAcct *gtsmodel.Account, + tagName string, + maxID string, + sinceID string, + minID string, + limit int, +) (*apimodel.PageableResponse, gtserror.WithCode) { + tag, errWithCode := p.getTag(ctx, tagName) + if errWithCode != nil { + return nil, errWithCode + } + + if tag == nil || !*tag.Useable || !*tag.Listable { + // Obey mastodon API by returning 404 for this. + err := fmt.Errorf("tag was not found, or not useable/listable on this instance") + return nil, gtserror.NewErrorNotFound(err, err.Error()) + } + + statuses, err := p.state.DB.GetTagTimeline(ctx, tag.ID, maxID, sinceID, minID, limit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err = gtserror.Newf("db error getting statuses: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + return p.packageTagResponse( + ctx, + requestingAcct, + statuses, + limit, + // Use API URL for tag. + "/api/v1/timelines/tag/"+tagName, + ) +} + +func (p *Processor) getTag(ctx context.Context, tagName string) (*gtsmodel.Tag, gtserror.WithCode) { + // Normalize + validate tag name. + tagNameNormal, ok := text.NormalizeHashtag(tagName) + if !ok { + err := gtserror.Newf("string '%s' could not be normalized to a valid hashtag", tagName) + return nil, gtserror.NewErrorBadRequest(err, err.Error()) + } + + // Ensure we have tag with this name in the db. + tag, err := p.state.DB.GetTagByName(ctx, tagNameNormal) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real db error. + err = gtserror.Newf("db error getting tag by name: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + return tag, nil +} + +func (p *Processor) packageTagResponse( + ctx context.Context, + requestingAcct *gtsmodel.Account, + statuses []*gtsmodel.Status, + limit int, + requestPath string, +) (*apimodel.PageableResponse, gtserror.WithCode) { + count := len(statuses) + if count == 0 { + return util.EmptyPageableResponse(), nil + } + + var ( + items = make([]interface{}, 0, count) + + // Set next + prev values before filtering and API + // converting, so caller can still page properly. + nextMaxIDValue = statuses[count-1].ID + prevMinIDValue = statuses[0].ID + ) + + for _, s := range statuses { + timelineable, err := p.filter.StatusTagTimelineable(ctx, requestingAcct, s) + if err != nil { + log.Errorf(ctx, "error checking status visibility: %v", err) + continue + } + + if !timelineable { + continue + } + + apiStatus, err := p.tc.StatusToAPIStatus(ctx, s, requestingAcct) + if err != nil { + log.Errorf(ctx, "error converting to api status: %v", err) + continue + } + + items = append(items, apiStatus) + } + + return util.PackagePageableResponse(util.PageableResponseParams{ + Items: items, + Path: requestPath, + NextMaxIDValue: nextMaxIDValue, + PrevMinIDValue: prevMinIDValue, + Limit: limit, + }) +} diff --git a/internal/text/markdown_test.go b/internal/text/markdown_test.go index 86e663dad..2602506ca 100644 --- a/internal/text/markdown_test.go +++ b/internal/text/markdown_test.go @@ -49,13 +49,13 @@ withInlineCode2 = "`Nobody tells you about the SECRET CODE, do they?`" withInlineCode2Expected = "

Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?

" withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!" - withHashtagExpected = "

Title

here's a simple status that uses hashtag #Hashtag!

" + withHashtagExpected = "

Title

here's a simple status that uses hashtag #Hashtag!

" mdWithHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a link.\n\nHere's an image: \"The" mdWithHTMLExpected = "

Title

Here's a simple text in markdown.

Here's a link.

Here's an image: \"The

" mdWithCheekyHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a cheeky little script: " mdWithCheekyHTMLExpected = "

Title

Here's a simple text in markdown.

Here's a cheeky little script:

" mdWithHashtagInitial = "#welcome #Hashtag" - mdWithHashtagInitialExpected = "

#welcome #Hashtag

" + mdWithHashtagInitialExpected = "

#welcome #Hashtag

" mdCodeBlockWithNewlines = "some code coming up\n\n```\n\n\n\n```\nthat was some code" mdCodeBlockWithNewlinesExpected = "

some code coming up

\n\n\n

that was some code

" mdWithFootnote = "fox mulder,fbi.[^1]\n\n[^1]: federated bureau of investigation" @@ -63,7 +63,7 @@ mdWithBlockQuote = "get ready, there's a block quote coming:\n\n>line1\n>line2\n>\n>line3\n\n" mdWithBlockQuoteExpected = "

get ready, there's a block quote coming:

line1
line2

line3

" mdHashtagAndCodeBlock = "#Hashtag\n\n```\n#Hashtag\n```" - mdHashtagAndCodeBlockExpected = "

#Hashtag

#Hashtag\n
" + mdHashtagAndCodeBlockExpected = "

#Hashtag

#Hashtag\n
" mdMentionAndCodeBlock = "@the_mighty_zork\n\n```\n@the_mighty_zork\n```" mdMentionAndCodeBlockExpected = "

@the_mighty_zork

@the_mighty_zork\n
" mdWithSmartypants = "\"you have to quargle the bleepflorp\" they said with 1/2 of nominal speed and 1/3 of the usual glumping" @@ -77,9 +77,9 @@ mdObjectInCodeBlock = "@foss_satan@fossbros-anonymous.io this is how to mention a user\n```\n@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you've been writing lately! :rainbow:\n```\nhope that helps" mdObjectInCodeBlockExpected = "

@foss_satan this is how to mention a user

@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you've been writing lately! :rainbow:\n

hope that helps

" mdItalicHashtag = "_#hashtag_" - mdItalicHashtagExpected = "

#hashtag

" + mdItalicHashtagExpected = "

#hashtag

" mdItalicHashtags = "_#hashtag #hashtag #hashtag_" - mdItalicHashtagsExpected = "

#hashtag #hashtag #hashtag

" + mdItalicHashtagsExpected = "

#hashtag #hashtag #hashtag

" // BEWARE: sneaky unicode business going on. // the first ö is one rune, the second ö is an o with a combining diacritic. mdUnnormalizedHashtag = "#hellöthere #hellöthere" diff --git a/internal/text/normalize.go b/internal/text/normalize.go new file mode 100644 index 000000000..14caf6311 --- /dev/null +++ b/internal/text/normalize.go @@ -0,0 +1,60 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package text + +import ( + "strings" + + "github.com/superseriousbusiness/gotosocial/internal/util" + "golang.org/x/text/unicode/norm" +) + +const ( + maximumHashtagLength = 100 +) + +// NormalizeHashtag normalizes the given hashtag text by +// removing the initial '#' symbol, and then decomposing +// and canonically recomposing chars + combining diacritics +// in the text to single unicode characters, following +// Normalization Form C (https://unicode.org/reports/tr15/). +// +// Finally, it will do a check on the normalized string to +// ensure that it's below maximumHashtagLength chars, and +// contains only unicode letters and numbers. If this passes, +// returned bool will be true. +func NormalizeHashtag(text string) (string, bool) { + // This normalization is specifically to avoid cases + // where visually-identical hashtags are stored with + // different unicode representations (e.g. with combining + // diacritics). It allows a tasteful number of combining + // diacritics to be used, as long as they can be combined + // with parent characters to form regular letter symbols. + normalized := norm.NFC.String(strings.TrimPrefix(text, "#")) + + // Validate normalized. + ok := true + for i, r := range normalized { + if i >= maximumHashtagLength || !util.IsPermittedInHashtag(r) { + ok = false + break + } + } + + return normalized, ok +} diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go index 5a2918563..dfcf8b953 100644 --- a/internal/text/plain_test.go +++ b/internal/text/plain_test.go @@ -34,7 +34,7 @@ withHTML = "
blah this should just be html escaped blah
" withHTMLExpected = "

<div>blah this should just be html escaped blah</div>

" moreComplex = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\nText\n\n:rainbow:" - moreComplexExpected = "

Another test @foss_satan

#Hashtag

Text

:rainbow:

" + moreComplexExpected = "

Another test @foss_satan

#Hashtag

Text

:rainbow:

" ) type PlainTestSuite struct { @@ -103,7 +103,7 @@ func (suite *PlainTestSuite) TestDeriveHashtagsOK() { #111111 thisalsoshouldn'twork#### ## #alimentación, #saúde, #lävistää, #ö, #네 -#ThisOneIsThirtyOneCharactersLon... ...ng +#ThisOneIsOneHundredAndOneCharactersLongWhichIsReallyJustWayWayTooLongDefinitelyLongerThanYouWouldNeed... #ThisOneIsThirteyCharactersLong ` @@ -141,7 +141,7 @@ func (suite *PlainTestSuite) TestDeriveMultiple() { assert.Equal(suite.T(), "@foss_satan@fossbros-anonymous.io", f.Mentions[0].NameString) assert.Len(suite.T(), f.Tags, 1) - assert.Equal(suite.T(), "Hashtag", f.Tags[0].Name) + assert.Equal(suite.T(), "hashtag", f.Tags[0].Name) assert.Len(suite.T(), f.Emojis, 0) } diff --git a/internal/text/replace.go b/internal/text/replace.go index e8e02454e..db72aaf1d 100644 --- a/internal/text/replace.go +++ b/internal/text/replace.go @@ -23,19 +23,13 @@ "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/util" - "golang.org/x/text/unicode/norm" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) -const ( - maximumHashtagLength = 30 -) - -// given a mention or a hashtag string, the methods in this file will attempt to parse it, -// add it to the database, and render it as HTML. If any of these steps fails, the method -// will just return the original string and log an error. - // replaceMention takes a string in the form @username@domain.com or @localusername func (r *customRenderer) replaceMention(text string) string { mention, err := r.parseMention(r.ctx, text, r.accountID, r.statusID) @@ -90,55 +84,78 @@ func (r *customRenderer) replaceMention(text string) string { return b.String() } -// replaceMention takes a string in the form #HashedTag, and will normalize it before -// adding it to the db and turning it into HTML. +// replaceHashtag takes a string in the form #SomeHashtag, and will normalize +// it before adding it to the db (or just getting it from the db if it already +// exists) and turning it into HTML. func (r *customRenderer) replaceHashtag(text string) string { - // this normalization is specifically to avoid cases where visually-identical - // hashtags are stored with different unicode representations (e.g. with combining - // diacritics). It allows a tasteful number of combining diacritics to be used, - // as long as they can be combined with parent characters to form regular letter - // symbols. - normalized := norm.NFC.String(text[1:]) - - for i, r := range normalized { - if i >= maximumHashtagLength || !util.IsPermittedInHashtag(r) { - return text - } + normalized, ok := NormalizeHashtag(text) + if !ok { + // Not a valid hashtag. + return text } - tag, err := r.f.db.TagStringToTag(r.ctx, normalized, r.accountID) + tag, err := r.getOrCreateHashtag(normalized) if err != nil { log.Errorf(r.ctx, "error generating hashtags from status: %s", err) return text } - // only append if it's not been listed yet - listed := false - for _, t := range r.result.Tags { - if tag.ID == t.ID { - listed = true - break - } - } - if !listed { - err = r.f.db.Put(r.ctx, tag) - if err != nil { - if !errors.Is(err, db.ErrAlreadyExists) { - log.Errorf(r.ctx, "error putting tags in db: %s", err) - return text + // Append tag to result if not done already. + // + // This prevents multiple uses of a tag in + // the same status generating multiple + // entries for the same tag in result. + func() { + for _, t := range r.result.Tags { + if tag.ID == t.ID { + // Already appended. + return } } - r.result.Tags = append(r.result.Tags, tag) - } + // Not appended yet. + r.result.Tags = append(r.result.Tags, tag) + }() + + // Replace tag with the formatted tag content, eg. `#SomeHashtag` becomes: + // `` var b strings.Builder - // replace the #tag with the formatted tag content - // ` b.WriteString(``) return b.String() } + +func (r *customRenderer) getOrCreateHashtag(name string) (*gtsmodel.Tag, error) { + var ( + tag *gtsmodel.Tag + err error + ) + + // Check if we have a tag with this name already. + tag, err = r.f.db.GetTagByName(r.ctx, name) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, gtserror.Newf("db error getting tag %s: %w", name, err) + } + + if tag != nil { + // We had it! + return tag, nil + } + + // We didn't have a tag with + // this name, create one. + tag = >smodel.Tag{ + ID: id.NewULID(), + Name: name, + } + + if err = r.f.db.PutTag(r.ctx, tag); err != nil { + return nil, gtserror.Newf("db error putting new tag %s: %w", name, err) + } + + return tag, nil +} diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 9121564fb..cb69cba5d 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -71,7 +71,8 @@ type TypeConverter interface { // EmojiCategoryToAPIEmojiCategory converts a gts model emoji category into its api (frontend) representation. EmojiCategoryToAPIEmojiCategory(ctx context.Context, category *gtsmodel.EmojiCategory) (*apimodel.EmojiCategory, error) // TagToAPITag converts a gts model tag into its api (frontend) representation for serialization on the API. - TagToAPITag(ctx context.Context, t *gtsmodel.Tag) (apimodel.Tag, error) + // If stubHistory is set to 'true', then the 'history' field of the tag will be populated with a pointer to an empty slice, for API compatibility reasons. + TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistory bool) (apimodel.Tag, error) // StatusToAPIStatus converts a gts model status into its api (frontend) representation for serialization on the API. // // Requesting account can be nil. @@ -160,6 +161,8 @@ type TypeConverter interface { MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab.ActivityStreamsMention, error) // EmojiToAS converts a gts emoji into a mastodon ns Emoji, suitable for federation EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.TootEmoji, error) + // TagToAS converts a gts model tag into a toot Hashtag, suitable for federation. + TagToAS(ctx context.Context, t *gtsmodel.Tag) (vocab.TootHashtag, error) // AttachmentToAS converts a gts model media attachment into an activity streams Attachment, suitable for federation AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error) // FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation. diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index 3c1615cfb..60ab24383 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -24,6 +24,7 @@ "errors" "fmt" "net/url" + "strings" "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/activity/streams" @@ -33,6 +34,7 @@ "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) // Converts a gts model account into an Activity Streams person type. @@ -407,7 +409,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A if s.Account == nil { a, err := c.db.GetAccountByID(ctx, s.AccountID) if err != nil { - return nil, fmt.Errorf("StatusToAS: error retrieving author account from db: %s", err) + return nil, gtserror.Newf("error retrieving author account from db: %w", err) } s.Account = a } @@ -418,7 +420,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A // id statusURI, err := url.Parse(s.URI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.URI, err) + return nil, gtserror.Newf("error parsing url %s: %w", s.URI, err) } statusIDProp := streams.NewJSONLDIdProperty() statusIDProp.SetIRI(statusURI) @@ -436,7 +438,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A if s.InReplyToURI != "" { rURI, err := url.Parse(s.InReplyToURI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.InReplyToURI, err) + return nil, gtserror.Newf("error parsing url %s: %w", s.InReplyToURI, err) } inReplyToProp := streams.NewActivityStreamsInReplyToProperty() @@ -453,7 +455,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A if s.URL != "" { sURL, err := url.Parse(s.URL) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.URL, err) + return nil, gtserror.Newf("error parsing url %s: %w", s.URL, err) } urlProp := streams.NewActivityStreamsUrlProperty() @@ -464,7 +466,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A // attributedTo authorAccountURI, err := url.Parse(s.Account.URI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.Account.URI, err) + return nil, gtserror.Newf("error parsing url %s: %w", s.Account.URI, err) } attributedToProp := streams.NewActivityStreamsAttributedToProperty() attributedToProp.AppendIRI(authorAccountURI) @@ -478,13 +480,13 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A if len(s.MentionIDs) > len(mentions) { mentions, err = c.db.GetMentions(ctx, s.MentionIDs) if err != nil { - return nil, fmt.Errorf("StatusToAS: error getting mentions: %w", err) + return nil, gtserror.Newf("error getting mentions: %w", err) } } for _, m := range mentions { asMention, err := c.MentionToAS(ctx, m) if err != nil { - return nil, fmt.Errorf("StatusToAS: error converting mention to AS mention: %s", err) + return nil, gtserror.Newf("error converting mention to AS mention: %w", err) } tagProp.AppendActivityStreamsMention(asMention) } @@ -496,7 +498,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, emojiID := range s.EmojiIDs { emoji, err := c.db.GetEmojiByID(ctx, emojiID) if err != nil { - return nil, fmt.Errorf("StatusToAS: error getting emoji %s from database: %s", emojiID, err) + return nil, gtserror.Newf("error getting emoji %s from database: %w", emojiID, err) } emojis = append(emojis, emoji) } @@ -504,25 +506,38 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, emoji := range emojis { asEmoji, err := c.EmojiToAS(ctx, emoji) if err != nil { - return nil, fmt.Errorf("StatusToAS: error converting emoji to AS emoji: %s", err) + return nil, gtserror.Newf("error converting emoji to AS emoji: %w", err) } tagProp.AppendTootEmoji(asEmoji) } // tag -- hashtags - // TODO + hashtags := s.Tags + if len(s.TagIDs) > len(hashtags) { + hashtags, err = c.db.GetTags(ctx, s.TagIDs) + if err != nil { + return nil, gtserror.Newf("error getting tags: %w", err) + } + } + for _, ht := range hashtags { + asHashtag, err := c.TagToAS(ctx, ht) + if err != nil { + return nil, gtserror.Newf("error converting tag to AS tag: %w", err) + } + tagProp.AppendTootHashtag(asHashtag) + } status.SetActivityStreamsTag(tagProp) // parse out some URIs we need here authorFollowersURI, err := url.Parse(s.Account.FollowersURI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.Account.FollowersURI, err) + return nil, gtserror.Newf("error parsing url %s: %w", s.Account.FollowersURI, err) } publicURI, err := url.Parse(pub.PublicActivityPubIRI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", pub.PublicActivityPubIRI, err) + return nil, gtserror.Newf("error parsing url %s: %w", pub.PublicActivityPubIRI, err) } // to and cc @@ -534,7 +549,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, m := range mentions { iri, err := url.Parse(m.TargetAccount.URI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err) + return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err) } toProp.AppendIRI(iri) } @@ -546,7 +561,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, m := range mentions { iri, err := url.Parse(m.TargetAccount.URI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err) + return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err) } ccProp.AppendIRI(iri) } @@ -557,7 +572,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, m := range mentions { iri, err := url.Parse(m.TargetAccount.URI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err) + return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err) } ccProp.AppendIRI(iri) } @@ -568,7 +583,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, m := range mentions { iri, err := url.Parse(m.TargetAccount.URI) if err != nil { - return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err) + return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err) } ccProp.AppendIRI(iri) } @@ -592,7 +607,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, attachmentID := range s.AttachmentIDs { attachment, err := c.db.GetAttachmentByID(ctx, attachmentID) if err != nil { - return nil, fmt.Errorf("StatusToAS: error getting attachment %s from database: %s", attachmentID, err) + return nil, gtserror.Newf("error getting attachment %s from database: %w", attachmentID, err) } attachments = append(attachments, attachment) } @@ -600,7 +615,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A for _, a := range attachments { doc, err := c.AttachmentToAS(ctx, a) if err != nil { - return nil, fmt.Errorf("StatusToAS: error converting attachment: %s", err) + return nil, gtserror.Newf("error converting attachment: %w", err) } attachmentProp.AppendActivityStreamsDocument(doc) } @@ -609,7 +624,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A // replies repliesCollection, err := c.StatusToASRepliesCollection(ctx, s, false) if err != nil { - return nil, fmt.Errorf("error creating repliesCollection: %s", err) + return nil, fmt.Errorf("error creating repliesCollection: %w", err) } repliesProp := streams.NewActivityStreamsRepliesProperty() @@ -846,6 +861,32 @@ func (c *converter) MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab return mention, nil } +func (c *converter) TagToAS(ctx context.Context, t *gtsmodel.Tag) (vocab.TootHashtag, error) { + // This is probably already lowercase, + // but let's err on the safe side. + nameLower := strings.ToLower(t.Name) + tagURLString := uris.GenerateURIForTag(nameLower) + + // Create the tag. + tag := streams.NewTootHashtag() + + // `href` should be the URL of the tag. + hrefProp := streams.NewActivityStreamsHrefProperty() + tagURL, err := url.Parse(tagURLString) + if err != nil { + return nil, gtserror.Newf("error parsing url %s: %w", tagURLString, err) + } + hrefProp.SetIRI(tagURL) + tag.SetActivityStreamsHref(hrefProp) + + // `name` should be the name of the tag with the # prefix. + nameProp := streams.NewActivityStreamsNameProperty() + nameProp.AppendXMLSchemaString("#" + nameLower) + tag.SetActivityStreamsName(nameProp) + + return tag, nil +} + /* we're making something like this: { diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index 60c59326c..30e4f2135 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -403,17 +403,24 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() { }, "sensitive": false, "summary": "", - "tag": { - "icon": { - "mediaType": "image/png", - "type": "Image", - "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png" + "tag": [ + { + "icon": { + "mediaType": "image/png", + "type": "Image", + "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png" + }, + "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ", + "name": ":rainbow:", + "type": "Emoji", + "updated": "2021-09-20T10:40:37Z" }, - "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ", - "name": ":rainbow:", - "type": "Emoji", - "updated": "2021-09-20T10:40:37Z" - }, + { + "href": "http://localhost:8080/tags/welcome", + "name": "#welcome", + "type": "Hashtag" + } + ], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" @@ -463,17 +470,24 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { }, "sensitive": false, "summary": "", - "tag": { - "icon": { - "mediaType": "image/png", - "type": "Image", - "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png" + "tag": [ + { + "icon": { + "mediaType": "image/png", + "type": "Image", + "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png" + }, + "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ", + "name": ":rainbow:", + "type": "Emoji", + "updated": "2021-09-20T10:40:37Z" }, - "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ", - "name": ":rainbow:", - "type": "Emoji", - "updated": "2021-09-20T10:40:37Z" - }, + { + "href": "http://localhost:8080/tags/welcome", + "name": "#welcome", + "type": "Hashtag" + } + ], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 975214da7..8ad1681d0 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -32,6 +32,7 @@ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/uris" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -568,10 +569,18 @@ func (c *converter) EmojiCategoryToAPIEmojiCategory(ctx context.Context, categor }, nil } -func (c *converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag) (apimodel.Tag, error) { +func (c *converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistory bool) (apimodel.Tag, error) { return apimodel.Tag{ - Name: t.Name, - URL: t.URL, + Name: strings.ToLower(t.Name), + URL: uris.GenerateURIForTag(t.Name), + History: func() *[]any { + if !stubHistory { + return nil + } + + h := make([]any, 0) + return &h + }(), }, nil } @@ -1297,19 +1306,11 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T var errs gtserror.MultiError if len(tags) == 0 { - // GTS model tags were not populated + var err error - // Preallocate expected GTS slice - tags = make([]*gtsmodel.Tag, 0, len(tagIDs)) - - // Fetch GTS models for tag IDs - for _, id := range tagIDs { - tag := new(gtsmodel.Tag) - if err := c.db.GetByID(ctx, id, tag); err != nil { - errs.Appendf("error fetching tag %s from database: %v", id, err) - continue - } - tags = append(tags, tag) + tags, err = c.db.GetTags(ctx, tagIDs) + if err != nil { + errs.Appendf("error fetching tags from database: %v", err) } } @@ -1318,7 +1319,7 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T // Convert GTS models to frontend models for _, tag := range tags { - apiTag, err := c.TagToAPITag(ctx, tag) + apiTag, err := c.TagToAPITag(ctx, tag, false) if err != nil { errs.Appendf("error converting tag %s to api tag: %v", tag.ID, err) continue diff --git a/internal/uris/uri.go b/internal/uris/uri.go index 8a8968f38..1e631bcbc 100644 --- a/internal/uris/uri.go +++ b/internal/uris/uri.go @@ -20,6 +20,7 @@ import ( "fmt" "net/url" + "strings" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/regexes" @@ -43,6 +44,7 @@ ConfirmEmailPath = "confirm_email" // ConfirmEmailPath is used to generate the URI for an email confirmation link FileserverPath = "fileserver" // FileserverPath is a path component for serving attachments + media EmojiPath = "emoji" // EmojiPath represents the activitypub emoji location + TagsPath = "tags" // TagsPath represents the activitypub tags location ) // UserURIs contains a bunch of UserURIs and URLs for a user, host, account, etc. @@ -178,6 +180,13 @@ func GenerateURIForEmoji(emojiID string) string { return fmt.Sprintf("%s://%s/%s/%s", protocol, host, EmojiPath, emojiID) } +// GenerateURIForTag generates an activitypub uri for a tag. +func GenerateURIForTag(name string) string { + protocol := config.GetProtocol() + host := config.GetHost() + return fmt.Sprintf("%s://%s/%s/%s", protocol, host, TagsPath, strings.ToLower(name)) +} + // IsUserPath returns true if the given URL path corresponds to eg /users/example_username func IsUserPath(id *url.URL) bool { return regexes.UserPath.MatchString(id.Path) diff --git a/internal/validate/tag_test.go b/internal/validate/tag_test.go deleted file mode 100644 index 43726fd5f..000000000 --- a/internal/validate/tag_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package validate_test - -import ( - "testing" - "time" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/validate" - "github.com/superseriousbusiness/gotosocial/testrig" -) - -func happyTag() *gtsmodel.Tag { - return >smodel.Tag{ - ID: "01FE91RJR88PSEEE30EV35QR8N", - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - URL: "https://example.org/tags/some_tag", - Name: "some_tag", - FirstSeenFromAccountID: "01FE91SR5P2GW06K3AJ98P72MT", - Useable: testrig.TrueBool(), - Listable: testrig.TrueBool(), - LastStatusAt: time.Now(), - } -} - -type TagValidateTestSuite struct { - suite.Suite -} - -func (suite *TagValidateTestSuite) TestValidateTagHappyPath() { - // no problem here - t := happyTag() - err := validate.Struct(t) - suite.NoError(err) -} - -func (suite *TagValidateTestSuite) TestValidateTagNoName() { - t := happyTag() - t.Name = "" - - err := validate.Struct(t) - suite.EqualError(err, "Key: 'Tag.Name' Error:Field validation for 'Name' failed on the 'required' tag") -} - -func (suite *TagValidateTestSuite) TestValidateTagBadURL() { - t := happyTag() - - t.URL = "" - err := validate.Struct(t) - suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'required' tag") - - t.URL = "no-schema.com" - err = validate.Struct(t) - suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'url' tag") - - t.URL = "justastring" - err = validate.Struct(t) - suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'url' tag") - - t.URL = "https://aaa\n\n\naaaaaaaa" - err = validate.Struct(t) - suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'url' tag") -} - -func (suite *TagValidateTestSuite) TestValidateTagNoFirstSeenFromAccountID() { - t := happyTag() - t.FirstSeenFromAccountID = "" - - err := validate.Struct(t) - suite.NoError(err) -} - -func TestTagValidateTestSuite(t *testing.T) { - suite.Run(t, new(TagValidateTestSuite)) -} diff --git a/internal/visibility/tag_timeline.go b/internal/visibility/tag_timeline.go new file mode 100644 index 000000000..b2c9dbf29 --- /dev/null +++ b/internal/visibility/tag_timeline.go @@ -0,0 +1,60 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package visibility + +import ( + "context" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" +) + +// StatusHomeTimelineable checks if given status should be included +// on requester's tag timeline, primarily relying on status visibility +// to requester and the AP visibility setting. +func (f *Filter) StatusTagTimelineable( + ctx context.Context, + requester *gtsmodel.Account, + status *gtsmodel.Status, +) (bool, error) { + if status.CreatedAt.After(time.Now().Add(24 * time.Hour)) { + // Statuses made over 1 day in the future we don't show... + log.Warnf(ctx, "status >24hrs in the future: %+v", status) + return false, nil + } + + // Don't show boosts on tag timeline. + if status.BoostOfID != "" { + return false, nil + } + + // Check whether status is visible to requesting account. + visible, err := f.StatusVisible(ctx, requester, status) + if err != nil { + return false, err + } + + if !visible { + log.Trace(ctx, "status not visible to timeline requester") + return false, nil + } + + // Looks good! + return true, nil +} diff --git a/internal/web/tag.go b/internal/web/tag.go new file mode 100644 index 000000000..d52de81d8 --- /dev/null +++ b/internal/web/tag.go @@ -0,0 +1,71 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package web + +import ( + "context" + "net/http" + + "github.com/gin-gonic/gin" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" +) + +func (m *Module) tagGETHandler(c *gin.Context) { + ctx := c.Request.Context() + + // We'll need the instance later, and we can also use it + // before then to make it easier to return a web error. + instance, errWithCode := m.processor.InstanceGetV1(ctx) + if errWithCode != nil { + apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + // Return instance we already got from the db, + // don't try to fetch it again when erroring. + instanceGet := func(ctx context.Context) (*apimodel.InstanceV1, gtserror.WithCode) { + return instance, nil + } + + // We only serve text/html at this endpoint. + if _, err := apiutil.NegotiateAccept(c, []apiutil.MIME{apiutil.TextHTML}...); err != nil { + apiutil.WebErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), instanceGet) + return + } + + tagName, errWithCode := apiutil.ParseTagName(c.Param(apiutil.TagNameKey)) + if errWithCode != nil { + apiutil.WebErrorHandler(c, errWithCode, instanceGet) + return + } + + stylesheets := []string{ + assetsPathPrefix + "/Fork-Awesome/css/fork-awesome.min.css", + distPathPrefix + "/status.css", + distPathPrefix + "/tag.css", + } + + c.HTML(http.StatusOK, "tag.tmpl", gin.H{ + "instance": instance, + "ogMeta": ogBase(instance), + "tagName": tagName, + "stylesheets": stylesheets, + }) +} diff --git a/internal/web/web.go b/internal/web/web.go index 5c1c4750d..6d785667b 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -25,6 +25,7 @@ "codeberg.org/gruf/go-cache/v3" "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -37,7 +38,8 @@ const ( confirmEmailPath = "/" + uris.ConfirmEmailPath profileGroupPath = "/@:" + usernameKey - statusPath = "/statuses/:" + statusIDKey // leave out the '/@:username' prefix as this will be served within the profile group + statusPath = "/statuses/:" + apiutil.WebStatusIDKey // leave out the '/@:username' prefix as this will be served within the profile group + tagsPath = "/tags/:" + apiutil.TagNameKey customCSSPath = profileGroupPath + "/custom.css" rssFeedPath = profileGroupPath + "/feed.rss" assetsPathPrefix = "/assets" @@ -49,7 +51,6 @@ tokenParam = "token" usernameKey = "username" - statusIDKey = "status" cacheControlHeader = "Cache-Control" // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control cacheControlNoCache = "no-cache" // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#response_directives @@ -107,6 +108,7 @@ func (m *Module) Route(r router.Router, mi ...gin.HandlerFunc) { r.AttachHandler(http.MethodGet, robotsPath, m.robotsGETHandler) r.AttachHandler(http.MethodGet, aboutPath, m.aboutGETHandler) r.AttachHandler(http.MethodGet, domainBlockListPath, m.domainBlockListGETHandler) + r.AttachHandler(http.MethodGet, tagsPath, m.tagGETHandler) // Attach redirects from old endpoints to current ones for backwards compatibility r.AttachHandler(http.MethodGet, "/auth/edit", func(c *gin.Context) { c.Redirect(http.StatusMovedPermanently, userPanelPath) }) diff --git a/test/envparsing.sh b/test/envparsing.sh index b9017d0be..6506d0f27 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -82,6 +82,9 @@ EXPECT=$(cat <<"EOF" "status-max-size": 2000, "status-sweep-freq": 60000000000, "status-ttl": 1800000000000, + "tag-max-size": 2000, + "tag-sweep-freq": 60000000000, + "tag-ttl": 1800000000000, "tombstone-max-size": 500, "tombstone-sweep-freq": 60000000000, "tombstone-ttl": 1800000000000, diff --git a/testrig/testmodels.go b/testrig/testmodels.go index c7a293455..1a5264db3 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -1760,26 +1760,20 @@ func NewTestStatuses() map[string]*gtsmodel.Status { func NewTestTags() map[string]*gtsmodel.Tag { return map[string]*gtsmodel.Tag{ "welcome": { - ID: "01F8MHA1A2NF9MJ3WCCQ3K8BSZ", - URL: "http://localhost:8080/tags/welcome", - Name: "welcome", - FirstSeenFromAccountID: "", - CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - Useable: TrueBool(), - Listable: TrueBool(), - LastStatusAt: TimeMustParse("2022-05-14T13:21:09+02:00"), + ID: "01F8MHA1A2NF9MJ3WCCQ3K8BSZ", + Name: "welcome", + CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), + UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), + Useable: TrueBool(), + Listable: TrueBool(), }, "Hashtag": { - ID: "01FCT9SGYA71487N8D0S1M638G", - URL: "http://localhost:8080/tags/Hashtag", - Name: "Hashtag", - FirstSeenFromAccountID: "", - CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - Useable: TrueBool(), - Listable: TrueBool(), - LastStatusAt: TimeMustParse("2022-05-14T13:21:09+02:00"), + ID: "01FCT9SGYA71487N8D0S1M638G", + Name: "hashtag", + CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), + UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), + Useable: TrueBool(), + Listable: TrueBool(), }, } } @@ -2092,6 +2086,7 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit nil, true, []vocab.ActivityStreamsMention{}, + []vocab.TootHashtag{}, nil, ) createDmForZork := WrapAPNoteInCreate( @@ -2115,6 +2110,7 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit URLMustParse("http://localhost:8080/users/1happyturtle"), "@1happyturtle@localhost:8080", )}, + []vocab.TootHashtag{}, nil, ) createReplyToTurtle := WrapAPNoteInCreate( @@ -2136,6 +2132,7 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit nil, false, []vocab.ActivityStreamsMention{}, + []vocab.TootHashtag{}, []vocab.ActivityStreamsImage{ newAPImage( URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpg"), @@ -2483,6 +2480,7 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { nil, false, []vocab.ActivityStreamsMention{}, + []vocab.TootHashtag{}, []vocab.ActivityStreamsImage{ newAPImage( URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpg"), @@ -2504,6 +2502,7 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { []*url.URL{}, false, nil, + []vocab.TootHashtag{}, nil, ), "https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV": NewAPNote( @@ -2524,6 +2523,28 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { "@the_mighty_zork@localhost:8080", ), }, + []vocab.TootHashtag{}, + nil, + ), + "https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7": NewAPNote( + URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7"), + URLMustParse("https://unknown-instance.com/users/@brand_new_person/01H641QSRS3TCXSVC10X4GPKW7"), + TimeMustParse("2023-04-12T12:13:12+02:00"), + "

Babe are you okay, you've hardly touched your #piss

", + "", + URLMustParse("https://unknown-instance.com/users/brand_new_person"), + []*url.URL{ + URLMustParse(pub.PublicActivityPubIRI), + }, + []*url.URL{}, + false, + []vocab.ActivityStreamsMention{}, + []vocab.TootHashtag{ + newAPHashtag( + URLMustParse("https://unknown-instance.com/tags/piss"), + "#piss", + ), + }, nil, ), "https://turnip.farm/users/turniplover6969/statuses/70c53e54-3146-42d5-a630-83c8b6c7c042": NewAPNote( @@ -2539,6 +2560,7 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { []*url.URL{}, false, nil, + []vocab.TootHashtag{}, []vocab.ActivityStreamsImage{ newAPImage( URLMustParse("https://turnip.farm/attachments/f17843c7-015e-4251-9b5a-91389c49ee57.jpg"), @@ -2566,6 +2588,7 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { "@the_mighty_zork@localhost:8080", ), }, + []vocab.TootHashtag{}, nil, ), } @@ -3349,6 +3372,20 @@ func newAPMention(uri *url.URL, namestring string) vocab.ActivityStreamsMention return mention } +func newAPHashtag(href *url.URL, name string) vocab.TootHashtag { + hashtag := streams.NewTootHashtag() + + hrefProp := streams.NewActivityStreamsHrefProperty() + hrefProp.SetIRI(href) + hashtag.SetActivityStreamsHref(hrefProp) + + nameProp := streams.NewActivityStreamsNameProperty() + nameProp.AppendXMLSchemaString(name) + hashtag.SetActivityStreamsName(nameProp) + + return hashtag +} + func newAPImage(url *url.URL, mediaType string, imageDescription string, blurhash string) vocab.ActivityStreamsImage { image := streams.NewActivityStreamsImage() @@ -3413,6 +3450,7 @@ func NewAPNote( noteCC []*url.URL, noteSensitive bool, noteMentions []vocab.ActivityStreamsMention, + noteTags []vocab.TootHashtag, noteAttachments []vocab.ActivityStreamsImage, ) vocab.ActivityStreamsNote { // create the note itself @@ -3475,13 +3513,20 @@ func NewAPNote( note.SetActivityStreamsCc(cc) } - // mentions + // Tag entries tag := streams.NewActivityStreamsTagProperty() + + // mentions for _, m := range noteMentions { tag.AppendActivityStreamsMention(m) } note.SetActivityStreamsTag(tag) + // hashtags + for _, t := range noteTags { + tag.AppendTootHashtag(t) + } + // append any attachments as ActivityStreamsImage if noteAttachments != nil { attachmentProperty := streams.NewActivityStreamsAttachmentProperty() diff --git a/web/source/css/tag.css b/web/source/css/tag.css new file mode 100644 index 000000000..98a129874 --- /dev/null +++ b/web/source/css/tag.css @@ -0,0 +1,24 @@ +/* + GoToSocial + Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +.thread { + #tag-name { + /* Ensure ridiculous length tags get wrapped */ + word-wrap: anywhere; + } +} diff --git a/web/template/tag.tmpl b/web/template/tag.tmpl new file mode 100644 index 000000000..c84d7a1a4 --- /dev/null +++ b/web/template/tag.tmpl @@ -0,0 +1,27 @@ +{{- /* +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +*/ -}} + +{{ template "header.tmpl" .}} + +
+

#{{.tagName}}

+

There's nothing here yet!

+
+ +{{ template "footer.tmpl" .}} \ No newline at end of file