[bugfix] Fix incorrect json-ld @context serialization (#3243)

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.9.0
-	github.com/superseriousbusiness/activity v1.8.0-gts
+	github.com/superseriousbusiness/activity v1.9.0-gts
 	github.com/superseriousbusiness/httpsig v1.2.0-SSB
 	github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB.0.20230227143000-f4900831d6c8
 	github.com/tdewolff/minify/v2 v2.20.37

go.sum

@@ -537,8 +537,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/superseriousbusiness/activity v1.8.0-gts h1:CMSN1eZUwNfIX1DFo4YxRCzSeT4jmGoIdakt/ZuDkQM=
+github.com/superseriousbusiness/activity v1.9.0-gts h1:qWMDeiGdnVi+XG7CfuM7ET87qe9adousU6utWItBX/o=
-github.com/superseriousbusiness/activity v1.8.0-gts/go.mod h1:AZw0Xb4Oju8rmaJCZ21gc5CPg47MmNgyac+Hx5jo8VM=
+github.com/superseriousbusiness/activity v1.9.0-gts/go.mod h1:9l74ZCv8zw07vipNMzahq8oQZt2xPaJZ+L+gLicQntQ=
 github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe h1:ksl2oCx/Qo8sNDc3Grb8WGKBM9nkvhCm25uvlT86azE=
 github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe/go.mod h1:gH4P6gN1V+wmIw5o97KGaa1RgXB/tVpC2UNzijhg3E4=
 github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB h1:8psprYSK1KdOSH7yQ4PbJq0YYaGQY+gzdW/B0ExDb/8=

internal/typeutils/internaltoas_test.go

@@ -44,14 +44,23 @@ func (suite *InternalToASTestSuite) TestAccountToAS() {
 	ser, err := ap.Serialize(asPerson)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://w3id.org/security/v1",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "discoverable": "toot:discoverable",
+      "featured": {
+        "@id": "toot:featured",
+        "@type": "@id"
+      },
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
   "discoverable": true,
   "featured": "http://localhost:8080/users/the_mighty_zork/collections/featured",
   "followers": "http://localhost:8080/users/the_mighty_zork/followers",
@@ -94,14 +103,26 @@ func (suite *InternalToASTestSuite) TestAccountToASWithFields() {
 	ser, err := ap.Serialize(asPerson)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://w3id.org/security/v1",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "PropertyValue": "schema:PropertyValue",
+      "discoverable": "toot:discoverable",
+      "featured": {
+        "@id": "toot:featured",
+        "@type": "@id"
+      },
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "schema": "http://schema.org#",
+      "toot": "http://joinmastodon.org/ns#",
+      "value": "schema:value"
+    }
+  ],
   "attachment": [
     {
       "name": "should you follow me?",
@@ -159,14 +180,28 @@ func (suite *InternalToASTestSuite) TestAccountToASAliasedAndMoved() {
 	ser, err := ap.Serialize(asPerson)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://w3id.org/security/v1",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "alsoKnownAs": "as:alsoKnownAs",
+      "discoverable": "toot:discoverable",
+      "featured": {
+        "@id": "toot:featured",
+        "@type": "@id"
+      },
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "movedTo": {
+        "@id": "as:movedTo",
+        "@type": "@id"
+      },
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
   "alsoKnownAs": [
     "http://localhost:8080/users/1happyturtle"
   ],
@@ -214,15 +249,27 @@ func (suite *InternalToASTestSuite) TestAccountToASWithOneField() {
 	ser, err := ap.Serialize(asPerson)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	// Despite only one field being set, attachments should still be a slice/array.
 	suite.Equal(`{
+  "@context": [
+    "https://w3id.org/security/v1",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "PropertyValue": "schema:PropertyValue",
+      "discoverable": "toot:discoverable",
+      "featured": {
+        "@id": "toot:featured",
+        "@type": "@id"
+      },
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "schema": "http://schema.org#",
+      "toot": "http://joinmastodon.org/ns#",
+      "value": "schema:value"
+    }
+  ],
   "attachment": [
     {
       "name": "should you follow me?",
@@ -263,14 +310,24 @@ func (suite *InternalToASTestSuite) TestAccountToASWithEmoji() {
 	ser, err := ap.Serialize(asPerson)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://w3id.org/security/v1",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "Emoji": "toot:Emoji",
+      "discoverable": "toot:discoverable",
+      "featured": {
+        "@id": "toot:featured",
+        "@type": "@id"
+      },
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
   "discoverable": true,
   "featured": "http://localhost:8080/users/the_mighty_zork/collections/featured",
   "followers": "http://localhost:8080/users/the_mighty_zork/followers",
@@ -325,14 +382,23 @@ func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() {
 	ser, err := ap.Serialize(asPerson)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://w3id.org/security/v1",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "discoverable": "toot:discoverable",
+      "featured": {
+        "@id": "toot:featured",
+        "@type": "@id"
+      },
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
   "discoverable": true,
   "endpoints": {
     "sharedInbox": "http://localhost:8080/sharedInbox"
@@ -378,14 +444,17 @@ func (suite *InternalToASTestSuite) TestStatusToAS() {
 	ser, err := ap.Serialize(asStatus)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://gotosocial.org/ns",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "sensitive": "as:sensitive"
+    }
+  ],
   "attachment": [],
   "attributedTo": "http://localhost:8080/users/the_mighty_zork",
   "cc": "http://localhost:8080/users/the_mighty_zork/followers",
@@ -445,14 +514,21 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
 	ser, err := ap.Serialize(asStatus)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://gotosocial.org/ns",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "Emoji": "toot:Emoji",
+      "Hashtag": "as:Hashtag",
+      "blurhash": "toot:blurhash",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
   "attachment": [
     {
       "blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
@@ -538,14 +614,21 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
 	ser, err := ap.Serialize(asStatus)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://gotosocial.org/ns",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "Emoji": "toot:Emoji",
+      "Hashtag": "as:Hashtag",
+      "blurhash": "toot:blurhash",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
   "attachment": [
     {
       "blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
@@ -632,14 +715,17 @@ func (suite *InternalToASTestSuite) TestStatusToASWithMentions() {
 	ser, err := ap.Serialize(asStatus)
 	suite.NoError(err)
 
-	// Drop "@context" property as
-	// the ordering is non-determinate.
-	delete(ser, "@context")
-
 	bytes, err := json.MarshalIndent(ser, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://gotosocial.org/ns",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "sensitive": "as:sensitive"
+    }
+  ],
   "attachment": [],
   "attributedTo": "http://localhost:8080/users/admin",
   "cc": [

internal/typeutils/wrap_test.go

@@ -72,14 +72,17 @@ func (suite *WrapTestSuite) TestWrapNoteInCreate() {
 	createI, err := ap.Serialize(create)
 	suite.NoError(err)
 
-	// Chop off @context since
-	// ordering is non-determinate.
-	delete(createI, "@context")
-
 	bytes, err := json.MarshalIndent(createI, "", "  ")
 	suite.NoError(err)
 
 	suite.Equal(`{
+  "@context": [
+    "https://gotosocial.org/ns",
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "sensitive": "as:sensitive"
+    }
+  ],
   "actor": "http://localhost:8080/users/the_mighty_zork",
   "cc": "http://localhost:8080/users/the_mighty_zork/followers",
   "id": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/activity#Create",

vendor/github.com/superseriousbusiness/activity/streams/util.go

@@ -1,6 +1,9 @@
 package streams
 
 import (
+	"maps"
+	"slices"
+
 	"github.com/superseriousbusiness/activity/streams/vocab"
 )
 
@@ -10,47 +13,200 @@
 	// rest of the payload. Important for linked-data representations, but
 	// only applicable to go-fed at code-generation time.
 	jsonLDContext = "@context"
+
+	asNS     = "https://www.w3.org/ns/activitystreams"
+	tootNS   = "http://joinmastodon.org/ns"
+	schemaNS = "http://schema.org"
 )
 
+// Map of inlines @context entries that may need to be added
+// when vocabs include "https://www.w3.org/ns/activitystreams".
+var asInlines = map[string]any{
+	"Hashtag":                   "as:Hashtag",
+	"alsoKnownAs":               "as:alsoKnownAs",
+	"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+	"sensitive":                 "as:sensitive",
+
+	"movedTo": map[string]string{
+		"@id":   "as:movedTo",
+		"@type": "@id",
+	},
+}
+
+// Map of inlines @context entries that may need to be
+// added when vocabs include "http://joinmastodon.org/ns".
+var tootInlines = map[string]any{
+	"Emoji":        "toot:Emoji",
+	"blurhash":     "toot:blurhash",
+	"discoverable": "toot:discoverable",
+	"indexable":    "toot:indexable",
+	"memorial":     "toot:memorial",
+	"suspended":    "toot:suspended",
+	"votersCount":  "toot:votersCount",
+
+	"featured": map[string]string{
+		"@id":   "toot:featured",
+		"@type": "@id",
+	},
+
+	"featuredTags": map[string]string{
+		"@id":   "toot:featuredTags",
+		"@type": "@id",
+	},
+
+	"focalPoint": map[string]string{
+		"@container": "@list",
+		"@id":        "toot:focalPoint",
+	},
+}
+
+// Map of inlines @context entries that may need to
+// be added when vocabs include "http://schema.org".
+var schemaInlines = map[string]any{
+	"PropertyValue": "schema:PropertyValue",
+	"value":         "schema:value",
+}
+
+// getLookup returns a lookup map of all interesting field names
+// + type names on the given "in" map that may need to be inlined.
+func getLookup(in map[string]any) map[string]struct{} {
+	out := make(map[string]struct{})
+
+	for k, v := range in {
+		// Pull out keys from any nested maps.
+		if nested, ok := v.(map[string]any); ok {
+			maps.Copy(out, getLookup(nested))
+			continue
+		}
+
+		// Pull out keys from any
+		// arrays of nested maps.
+		if nestedIs, ok := v.([]any); ok {
+			for _, nestedI := range nestedIs {
+				if nested, ok := nestedI.(map[string]any); ok {
+					maps.Copy(out, getLookup(nested))
+					continue
+				}
+			}
+		}
+
+		// For types, we actually care about
+		// the *value*, ie., the name of the
+		// type, not the type key itself.
+		if k == "type" {
+			out[v.(string)] = struct{}{}
+			continue
+		}
+
+		out[k] = struct{}{}
+	}
+
+	return out
+}
+
+func copyInlines(
+	src map[string]any,
+	dst map[string]any,
+	lookup map[string]struct{},
+) {
+	for k, v := range src {
+		_, ok := lookup[k]
+		if ok {
+			dst[k] = v
+		}
+	}
+}
+
 // Serialize adds the context vocabularies contained within the type
 // into the JSON-LD @context field, and aliases them appropriately.
-func Serialize(a vocab.Type) (m map[string]interface{}, e error) {
+func Serialize(a vocab.Type) (m map[string]any, e error) {
 	m, e = a.Serialize()
 	if e != nil {
 		return
 	}
-	v := a.JSONLDContext()
+
-	// Transform the map of vocabulary-to-aliases into a context payload,
+	var (
-	// but do so in a way that at least keeps it readable for other humans.
+		// Slice of vocab URIs
-	var contextValue interface{}
+		// used in this vocab.Type.
-	if len(v) == 1 {
+		vocabs = a.JSONLDContext()
-		for vocab, alias := range v {
+
-			if len(alias) == 0 {
+		// Slice of vocab URIs to add
-				contextValue = vocab
+		// to the base @context slice.
+		includeVocabs []string
+
+		// Object to inline as an extra
+		// entry in the @context slice.
+		inlinedContext = make(map[string]any)
+	)
+
+	// Get a lookup of all field and
+	// type names we need to care about.
+	lookup := getLookup(m)
+
+	// Go through each used vocab and see
+	// if we need to special case it.
+	for vocab := range vocabs {
+
+		switch vocab {
+
+		case asNS:
+			// ActivityStreams vocab.
+			//
+			// The namespace URI already points to
+			// a proper @context document but we
+			// need to add some extra inlines.
+			includeVocabs = append(includeVocabs, asNS)
+			copyInlines(asInlines, inlinedContext, lookup)
+
+		case schemaNS:
+			// Schema vocab.
+			//
+			// The URI doesn't point to a @context
+			// document so we need to inline everything.
+			inlinedContext["schema"] = schemaNS + "#"
+			copyInlines(schemaInlines, inlinedContext, lookup)
+
+		case tootNS:
+			// Toot/Mastodon vocab.
+			//
+			// The URI doesn't point to a @context
+			// document so we need to inline everything.
+			inlinedContext["toot"] = tootNS + "#"
+			copyInlines(tootInlines, inlinedContext, lookup)
+
+		default:
+			// No special case.
+			includeVocabs = append(includeVocabs, vocab)
+		}
+	}
+
+	// Sort used vocab entries alphabetically
+	// to make their ordering predictable.
+	slices.Sort(includeVocabs)
+
+	// Create final slice of @context
+	// entries we'll need to include.
+	contextEntries := make([]any, 0, len(includeVocabs)+1)
+
+	// Append each included vocab to the slice.
+	for _, vocab := range includeVocabs {
+		contextEntries = append(contextEntries, vocab)
+	}
+
+	// Append any inlinedContext to the slice.
+	if len(inlinedContext) != 0 {
+		contextEntries = append(contextEntries, inlinedContext)
+	}
+
+	// Include @context on the final output,
+	// using an array if there's more than
+	// one entry, just a property otherwise.
+	if len(contextEntries) != 1 {
+		m[jsonLDContext] = contextEntries
 	} else {
-				contextValue = map[string]string{
+		m[jsonLDContext] = contextEntries[0]
-					alias: vocab,
 	}
-			}
+
-		}
-	} else {
-		var arr []interface{}
-		aliases := make(map[string]string)
-		for vocab, alias := range v {
-			if len(alias) == 0 {
-				arr = append(arr, vocab)
-			} else {
-				aliases[alias] = vocab
-			}
-		}
-		if len(aliases) > 0 {
-			arr = append(arr, aliases)
-		}
-		contextValue = arr
-	}
-	// TODO: Update the context instead if it already exists
-	m[jsonLDContext] = contextValue
-	// TODO: Sort the context based on arbitrary order.
 	// Delete any existing `@context` in child maps.
 	var cleanFnRecur func(map[string]interface{})
 	cleanFnRecur = func(r map[string]interface{}) {

vendor/modules.txt

@@ -647,8 +647,8 @@ github.com/stretchr/testify/suite
 # github.com/subosito/gotenv v1.6.0
 ## explicit; go 1.18
 github.com/subosito/gotenv
-# github.com/superseriousbusiness/activity v1.8.0-gts
+# github.com/superseriousbusiness/activity v1.9.0-gts
-## explicit; go 1.18
+## explicit; go 1.21
 github.com/superseriousbusiness/activity/pub
 github.com/superseriousbusiness/activity/streams
 github.com/superseriousbusiness/activity/streams/impl/activitystreams/property_accuracy