Blocklist import (#77)

* first steps on importing blocklists

* unblock domains properly

PROGRESS.md

@@ -41,9 +41,9 @@
   * [ ] Blocks
     * [ ] /api/v1/blocks GET                                (See list of blocked accounts)
   * [ ] Domain Blocks
-    * [ ] /api/v1/domain_blocks GET                         (See list of domain blocks)
+    * [x] /api/v1/domain_blocks GET                         (See list of domain blocks)
-    * [ ] /api/v1/domain_blocks POST                        (Create a domain block)
+    * [x] /api/v1/domain_blocks POST                        (Create a domain block)
-    * [ ] /api/v1/domain_blocks DELETE                      (Remove a domain block)
+    * [x] /api/v1/domain_blocks DELETE                      (Remove a domain block)
   * [ ] Filters
     * [ ] /api/v1/filters GET                               (Get list of filters)
     * [ ] /api/v1/filters/:id GET                           (View a filter)
@@ -134,7 +134,7 @@
     * [x] /api/v2/search GET                                (Get search query results)
   * [ ] Instance
     * [x] /api/v1/instance GET                              (Get instance information)
-    * [ ] /api/v1/instance PATCH                            (Update instance information)
+    * [x] /api/v1/instance PATCH                            (Update instance information)
     * [ ] /api/v1/instance/peers GET                        (Get list of federated servers)
     * [ ] /api/v1/instance/activity GET                     (Instance activity over the last 3 months, binned weekly.)
   * [ ] Trends
@@ -198,10 +198,10 @@
   * [ ] App creation guide
 * [ ] Tooling
   * [ ] Database migration tool
-  * [ ] Admin CLI tool
+  * [x] Admin CLI tool
 * [ ] Build
-  * [ ] Docker containerization
+  * [x] Docker containerization
-    * [ ] Dockerfile
+    * [x] Dockerfile
     * [ ] docker-compose.yml
 * [ ] Tests
   * [ ] Unit/integration

README.md

@@ -26,9 +26,15 @@ A grab-bag of things that are already included or will be included in the projec
 
 ## Implementation Status
 
-Things are moving on the project! As of June 2021 you can now:
+Things are moving on the project! As of July 2021 you can now:
+
+### Admin
 
 * Build and deploy GoToSocial as a binary, with automatic LetsEncrypt certificate support built-in.
+* Create, confirm, and promote users using self-documented CLI tool.
+
+### User
+
 * Connect to the running instance via Tusky or Pinafore, using email address and password (stored encrypted).
 * Post/delete posts.
 * Reply/delete replies.
@@ -44,7 +50,12 @@ Things are moving on the project! As of June 2021 you can now:
 * View local timeline.
 * View and scroll home timeline (with ~10ms latency hell yeah).
 * Stream new posts, notifications and deletes through a websockets connection via Pinafore.
+
+### Federation
+
 * Federation support and interoperability with Mastodon and others.
+* Domain blocking: create, update, delete, and export domain blocks.
+* Domain blocking: import lists of domain blocks -- no more blocking domains one-by-one.
 
 In other words, a deployed GoToSocial instance is already pretty useable!
 

internal/api/client/admin/admin.go

@@ -35,11 +35,13 @@
 	EmojiPath = BasePath + "/custom_emojis"
 	// DomainBlocksPath is used for posting domain blocks.
 	DomainBlocksPath = BasePath + "/domain_blocks"
-	// DomainBlockPath is used for interacting with a single domain block.
+	// DomainBlocksPathWithID is used for interacting with a single domain block.
-	DomainBlockPath = DomainBlocksPath + "/:" + IDKey
+	DomainBlocksPathWithID = DomainBlocksPath + "/:" + IDKey
 
 	// ExportQueryKey is for requesting a public export of some data.
 	ExportQueryKey = "export"
+	// ImportQueryKey is for submitting an import of some data.
+	ImportQueryKey = "import"
 	// IDKey specifies the ID of a single item being interacted with.
 	IDKey = "id"
 )
@@ -65,7 +67,7 @@ func (m *Module) Route(r router.Router) error {
 	r.AttachHandler(http.MethodPost, EmojiPath, m.emojiCreatePOSTHandler)
 	r.AttachHandler(http.MethodPost, DomainBlocksPath, m.DomainBlocksPOSTHandler)
 	r.AttachHandler(http.MethodGet, DomainBlocksPath, m.DomainBlocksGETHandler)
-	r.AttachHandler(http.MethodGet, DomainBlockPath, m.DomainBlockGETHandler)
+	r.AttachHandler(http.MethodGet, DomainBlocksPathWithID, m.DomainBlockGETHandler)
-	r.AttachHandler(http.MethodDelete, DomainBlockPath, m.DomainBlockDELETEHandler)
+	r.AttachHandler(http.MethodDelete, DomainBlocksPathWithID, m.DomainBlockDELETEHandler)
 	return nil
 }

internal/api/client/admin/domainblockcreate.go

@@ -4,6 +4,7 @@
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
@@ -33,6 +34,18 @@ func (m *Module) DomainBlocksPOSTHandler(c *gin.Context) {
 		return
 	}
 
+	imp := false
+	importString := c.Query(ImportQueryKey)
+	if importString != "" {
+		i, err := strconv.ParseBool(importString)
+		if err != nil {
+			l.Debugf("error parsing import string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse import query param"})
+			return
+		}
+		imp = i
+	}
+
 	// extract the media create form from the request context
 	l.Tracef("parsing request form: %+v", c.Request.Form)
 	form := &model.DomainBlockCreateRequest{}
@@ -44,26 +57,43 @@ func (m *Module) DomainBlocksPOSTHandler(c *gin.Context) {
 
 	// Give the fields on the request form a first pass to make sure the request is superficially valid.
 	l.Tracef("validating form %+v", form)
-	if err := validateCreateDomainBlock(form); err != nil {
+	if err := validateCreateDomainBlock(form, imp); err != nil {
 		l.Debugf("error validating form: %s", err)
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
 
-	domainBlock, err := m.processor.AdminDomainBlockCreate(authed, form)
+	if imp {
-	if err != nil {
+		// we're importing multiple blocks
-		l.Debugf("error creating domain block: %s", err)
+		domainBlocks, err := m.processor.AdminDomainBlocksImport(authed, form)
-		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		if err != nil {
-		return
+			l.Debugf("error importing domain blocks: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+		c.JSON(http.StatusOK, domainBlocks)
+	} else {
+		// we're just creating one block
+		domainBlock, err := m.processor.AdminDomainBlockCreate(authed, form)
+		if err != nil {
+			l.Debugf("error creating domain block: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+		c.JSON(http.StatusOK, domainBlock)
 	}
-
-	c.JSON(http.StatusOK, domainBlock)
 }
 
-func validateCreateDomainBlock(form *model.DomainBlockCreateRequest) error {
+func validateCreateDomainBlock(form *model.DomainBlockCreateRequest, imp bool) error {
-	// add some more validation here later if necessary
+	if imp {
-	if form.Domain == "" {
+		if form.Domains.Size == 0 {
-		return errors.New("empty domain provided")
+			return errors.New("import was specified but list of domains is empty")
+		}
+	} else {
+		// add some more validation here later if necessary
+		if form.Domain == "" {
+			return errors.New("empty domain provided")
+		}
 	}
 
 	return nil

internal/api/model/domainblock.go

@@ -18,13 +18,15 @@
 
 package model
 
+import "mime/multipart"
+
 // DomainBlock represents a block on one domain
 type DomainBlock struct {
 	ID             string `json:"id,omitempty"`
-	Domain         string `json:"domain"`
+	Domain         string `form:"domain" json:"domain" validation:"required"`
 	Obfuscate      bool   `json:"obfuscate,omitempty"`
 	PrivateComment string `json:"private_comment,omitempty"`
-	PublicComment  string `json:"public_comment,omitempty"`
+	PublicComment  string `form:"public_comment" json:"public_comment,omitempty"`
 	SubscriptionID string `json:"subscription_id,omitempty"`
 	CreatedBy      string `json:"created_by,omitempty"`
 	CreatedAt      string `json:"created_at,omitempty"`
@@ -32,8 +34,10 @@ type DomainBlock struct {
 
 // DomainBlockCreateRequest is the form submitted as a POST to /api/v1/admin/domain_blocks to create a new block.
 type DomainBlockCreateRequest struct {
+	// A list of domains to block. Only used if import=true is specified.
+	Domains *multipart.FileHeader `form:"domains" json:"domains" xml:"domains"`
 	// hostname/domain to block
-	Domain string `form:"domain" json:"domain" xml:"domain" validation:"required"`
+	Domain string `form:"domain" json:"domain" xml:"domain"`
 	// whether the domain should be obfuscated when being displayed publicly
 	Obfuscate bool `form:"obfuscate" json:"obfuscate" xml:"obfuscate"`
 	// private comment for other admins on why the domain was blocked

internal/db/db.go

@@ -86,6 +86,9 @@ type DB interface {
 	// UpdateOneByID updates interface i with database the given database id. It will update one field of key key and value value.
 	UpdateOneByID(id string, key string, value interface{}, i interface{}) error
 
+	// UpdateWhere updates column key of interface i with the given value, where the given parameters apply.
+	UpdateWhere(where []Where, key string, value interface{}, i interface{}) error
+
 	// DeleteByID removes i with id id.
 	// If i didn't exist anyway, then no error should be returned.
 	DeleteByID(id string, i interface{}) error

internal/db/pg/delete.go

@@ -0,0 +1,57 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package pg
+
+import (
+	"errors"
+
+	"github.com/go-pg/pg/v10"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+func (ps *postgresService) DeleteByID(id string, i interface{}) error {
+	if _, err := ps.conn.Model(i).Where("id = ?", id).Delete(); err != nil {
+		// if there are no rows *anyway* then that's fine
+		// just return err if there's an actual error
+		if err != pg.ErrNoRows {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ps *postgresService) DeleteWhere(where []db.Where, i interface{}) error {
+	if len(where) == 0 {
+		return errors.New("no queries provided")
+	}
+
+	q := ps.conn.Model(i)
+	for _, w := range where {
+		q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
+	}
+
+	if _, err := q.Delete(); err != nil {
+		// if there are no rows *anyway* then that's fine
+		// just return err if there's an actual error
+		if err != pg.ErrNoRows {
+			return err
+		}
+	}
+	return nil
+}

internal/db/pg/get.go

@@ -0,0 +1,75 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package pg
+
+import (
+	"errors"
+
+	"github.com/go-pg/pg/v10"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+func (ps *postgresService) GetByID(id string, i interface{}) error {
+	if err := ps.conn.Model(i).Where("id = ?", id).Select(); err != nil {
+		if err == pg.ErrNoRows {
+			return db.ErrNoEntries{}
+		}
+		return err
+
+	}
+	return nil
+}
+
+func (ps *postgresService) GetWhere(where []db.Where, i interface{}) error {
+	if len(where) == 0 {
+		return errors.New("no queries provided")
+	}
+
+	q := ps.conn.Model(i)
+	for _, w := range where {
+
+		if w.Value == nil {
+			q = q.Where("? IS NULL", pg.Ident(w.Key))
+		} else {
+			if w.CaseInsensitive {
+				q = q.Where("LOWER(?) = LOWER(?)", pg.Safe(w.Key), w.Value)
+			} else {
+				q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
+			}
+		}
+	}
+
+	if err := q.Select(); err != nil {
+		if err == pg.ErrNoRows {
+			return db.ErrNoEntries{}
+		}
+		return err
+	}
+	return nil
+}
+
+func (ps *postgresService) GetAll(i interface{}) error {
+	if err := ps.conn.Model(i).Select(); err != nil {
+		if err == pg.ErrNoRows {
+			return db.ErrNoEntries{}
+		}
+		return err
+	}
+	return nil
+}

internal/db/pg/instance.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package pg
 
 import (
@@ -42,8 +60,8 @@ func (ps *postgresService) GetDomainCountForInstance(domain string) (int, error)
 
 	if domain == ps.config.Host {
 		// if the domain is *this* domain, just count other instances it knows about
-		// TODO: exclude domains that are blocked or silenced
+		// exclude domains that are blocked
-		q = q.Where("domain != ?", domain)
+		q = q.Where("domain != ?", domain).Where("? IS NULL", pg.Ident("suspended_at"))
 	} else {
 		// TODO: implement federated domain counting properly for remote domains
 		return 0, nil

internal/db/pg/pg.go

@@ -197,119 +197,6 @@ func (ps *postgresService) CreateSchema(ctx context.Context) error {
 	return nil
 }
 
-func (ps *postgresService) GetByID(id string, i interface{}) error {
-	if err := ps.conn.Model(i).Where("id = ?", id).Select(); err != nil {
-		if err == pg.ErrNoRows {
-			return db.ErrNoEntries{}
-		}
-		return err
-
-	}
-	return nil
-}
-
-func (ps *postgresService) GetWhere(where []db.Where, i interface{}) error {
-	if len(where) == 0 {
-		return errors.New("no queries provided")
-	}
-
-	q := ps.conn.Model(i)
-	for _, w := range where {
-
-		if w.Value == nil {
-			q = q.Where("? IS NULL", pg.Ident(w.Key))
-		} else {
-			if w.CaseInsensitive {
-				q = q.Where("LOWER(?) = LOWER(?)", pg.Safe(w.Key), w.Value)
-			} else {
-				q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
-			}
-		}
-	}
-
-	if err := q.Select(); err != nil {
-		if err == pg.ErrNoRows {
-			return db.ErrNoEntries{}
-		}
-		return err
-	}
-	return nil
-}
-
-func (ps *postgresService) GetAll(i interface{}) error {
-	if err := ps.conn.Model(i).Select(); err != nil {
-		if err == pg.ErrNoRows {
-			return db.ErrNoEntries{}
-		}
-		return err
-	}
-	return nil
-}
-
-func (ps *postgresService) Put(i interface{}) error {
-	_, err := ps.conn.Model(i).Insert(i)
-	if err != nil && strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
-		return db.ErrAlreadyExists{}
-	}
-	return err
-}
-
-func (ps *postgresService) Upsert(i interface{}, conflictColumn string) error {
-	if _, err := ps.conn.Model(i).OnConflict(fmt.Sprintf("(%s) DO UPDATE", conflictColumn)).Insert(); err != nil {
-		if err == pg.ErrNoRows {
-			return db.ErrNoEntries{}
-		}
-		return err
-	}
-	return nil
-}
-
-func (ps *postgresService) UpdateByID(id string, i interface{}) error {
-	if _, err := ps.conn.Model(i).Where("id = ?", id).OnConflict("(id) DO UPDATE").Insert(); err != nil {
-		if err == pg.ErrNoRows {
-			return db.ErrNoEntries{}
-		}
-		return err
-	}
-	return nil
-}
-
-func (ps *postgresService) UpdateOneByID(id string, key string, value interface{}, i interface{}) error {
-	_, err := ps.conn.Model(i).Set("? = ?", pg.Safe(key), value).Where("id = ?", id).Update()
-	return err
-}
-
-func (ps *postgresService) DeleteByID(id string, i interface{}) error {
-	if _, err := ps.conn.Model(i).Where("id = ?", id).Delete(); err != nil {
-		// if there are no rows *anyway* then that's fine
-		// just return err if there's an actual error
-		if err != pg.ErrNoRows {
-			return err
-		}
-	}
-	return nil
-}
-
-func (ps *postgresService) DeleteWhere(where []db.Where, i interface{}) error {
-	if len(where) == 0 {
-		return errors.New("no queries provided")
-	}
-
-	q := ps.conn.Model(i)
-	for _, w := range where {
-		q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
-	}
-
-	if _, err := q.Delete(); err != nil {
-		// if there are no rows *anyway* then that's fine
-		// just return err if there's an actual error
-		if err != pg.ErrNoRows {
-			return err
-		}
-	}
-	return nil
-}
-
 /*
 	HANDY SHORTCUTS
 */

internal/db/pg/put.go

@@ -0,0 +1,33 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package pg
+
+import (
+	"strings"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+func (ps *postgresService) Put(i interface{}) error {
+	_, err := ps.conn.Model(i).Insert(i)
+	if err != nil && strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
+		return db.ErrAlreadyExists{}
+	}
+	return err
+}

internal/db/pg/statuscontext.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package pg
 
 import (

internal/db/pg/update.go

@@ -0,0 +1,73 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package pg
+
+import (
+	"fmt"
+
+	"github.com/go-pg/pg/v10"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+func (ps *postgresService) Upsert(i interface{}, conflictColumn string) error {
+	if _, err := ps.conn.Model(i).OnConflict(fmt.Sprintf("(%s) DO UPDATE", conflictColumn)).Insert(); err != nil {
+		if err == pg.ErrNoRows {
+			return db.ErrNoEntries{}
+		}
+		return err
+	}
+	return nil
+}
+
+func (ps *postgresService) UpdateByID(id string, i interface{}) error {
+	if _, err := ps.conn.Model(i).Where("id = ?", id).OnConflict("(id) DO UPDATE").Insert(); err != nil {
+		if err == pg.ErrNoRows {
+			return db.ErrNoEntries{}
+		}
+		return err
+	}
+	return nil
+}
+
+func (ps *postgresService) UpdateOneByID(id string, key string, value interface{}, i interface{}) error {
+	_, err := ps.conn.Model(i).Set("? = ?", pg.Safe(key), value).Where("id = ?", id).Update()
+	return err
+}
+
+func (ps *postgresService) UpdateWhere(where []db.Where, key string, value interface{}, i interface{}) error {
+	q := ps.conn.Model(i)
+
+	for _, w := range where {
+		if w.Value == nil {
+			q = q.Where("? IS NULL", pg.Ident(w.Key))
+		} else {
+			if w.CaseInsensitive {
+				q = q.Where("LOWER(?) = LOWER(?)", pg.Safe(w.Key), w.Value)
+			} else {
+				q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
+			}
+		}
+	}
+
+	q = q.Set("? = ?", pg.Safe(key), value)
+
+	_, err := q.Update()
+
+	return err
+}

internal/gtsmodel/account.go

@@ -115,7 +115,7 @@ type Account struct {
 		CRYPTO FIELDS
 	*/
 
-	// Privatekey for validating activitypub requests, will obviously only be defined for local accounts
+	// Privatekey for validating activitypub requests, will only be defined for local accounts
 	PrivateKey *rsa.PrivateKey
 	// Publickey for encoding activitypub requests, will be defined for both local and remote accounts
 	PublicKey *rsa.PublicKey
@@ -134,8 +134,8 @@ type Account struct {
 	SuspendedAt time.Time `pg:"type:timestamp"`
 	// Should we hide this account's collections?
 	HideCollections bool
-	// id of the user that suspended this account through an admin action
+	// id of the database entry that caused this account to become suspended -- can be an account ID or a domain block ID
-	SuspensionOrigin string
+	SuspensionOrigin string `pg:"type:CHAR(26)"`
 }
 
 // Field represents a key value field on an account, for things like pronouns, website, etc.

internal/processing/account/account.go

@@ -40,7 +40,8 @@ type Processor interface {
 	// Create processes the given form for creating a new account, returning an oauth token for that account if successful.
 	Create(applicationToken oauth2.TokenInfo, application *gtsmodel.Application, form *apimodel.AccountCreateRequest) (*apimodel.Token, error)
 	// Delete deletes an account, and all of that account's statuses, media, follows, notifications, etc etc etc.
-	Delete(account *gtsmodel.Account, deletedBy string) error
+	// The origin passed here should be either the ID of the account doing the delete (can be itself), or the ID of a domain block.
+	Delete(account *gtsmodel.Account, origin string) error
 	// Get processes the given request for account information.
 	Get(requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Account, error)
 	// Update processes the update of an account with the given form

internal/processing/account/delete.go

@@ -48,7 +48,7 @@
 // 16. Delete account's user
 // 17. Delete account's timeline
 // 18. Delete account itself
-func (p *processor) Delete(account *gtsmodel.Account, deletedBy string) error {
+func (p *processor) Delete(account *gtsmodel.Account, origin string) error {
 	l := p.log.WithFields(logrus.Fields{
 		"func":     "Delete",
 		"username": account.Username,
@@ -100,6 +100,7 @@ func (p *processor) Delete(account *gtsmodel.Account, deletedBy string) error {
 	// nothing to do here
 
 	// 4. Delete account's follow requests
+	// TODO: federate these if necessary
 	l.Debug("deleting account follow requests")
 	// first delete any follow requests that this account created
 	if err := p.db.DeleteWhere([]db.Where{{Key: "account_id", Value: account.ID}}, &[]*gtsmodel.FollowRequest{}); err != nil {
@@ -112,6 +113,7 @@ func (p *processor) Delete(account *gtsmodel.Account, deletedBy string) error {
 	}
 
 	// 5. Delete account's follows
+	// TODO: federate these if necessary
 	l.Debug("deleting account follows")
 	// first delete any follows that this account created
 	if err := p.db.DeleteWhere([]db.Where{{Key: "account_id", Value: account.ID}}, &[]*gtsmodel.Follow{}); err != nil {
@@ -217,6 +219,7 @@ func (p *processor) Delete(account *gtsmodel.Account, deletedBy string) error {
 	}
 
 	// 12. Delete account's faves
+	// TODO: federate these if necessary
 	l.Debug("deleting account faves")
 	if err := p.db.DeleteWhere([]db.Where{{Key: "account_id", Value: account.ID}}, &[]*gtsmodel.StatusFave{}); err != nil {
 		l.Errorf("error deleting faves created by account: %s", err)
@@ -229,6 +232,7 @@ func (p *processor) Delete(account *gtsmodel.Account, deletedBy string) error {
 	}
 
 	// 14. Delete account's streams
+	// TODO
 
 	// 15. Delete account's tags
 	// TODO
@@ -240,6 +244,7 @@ func (p *processor) Delete(account *gtsmodel.Account, deletedBy string) error {
 	}
 
 	// 17. Delete account's timeline
+	// TODO
 
 	// 18. Delete account itself
 	// to prevent the account being created again, set all these fields and update it in the db
@@ -259,7 +264,7 @@ func (p *processor) Delete(account *gtsmodel.Account, deletedBy string) error {
 	account.UpdatedAt = time.Now()
 
 	account.SuspendedAt = time.Now()
-	account.SuspensionOrigin = deletedBy
+	account.SuspensionOrigin = origin
 
 	if err := p.db.UpdateByID(account.ID, account); err != nil {
 		return err

internal/processing/admin.go

@@ -29,7 +29,11 @@ func (p *processor) AdminEmojiCreate(authed *oauth.Auth, form *apimodel.EmojiCre
 }
 
 func (p *processor) AdminDomainBlockCreate(authed *oauth.Auth, form *apimodel.DomainBlockCreateRequest) (*apimodel.DomainBlock, gtserror.WithCode) {
-	return p.adminProcessor.DomainBlockCreate(authed.Account, form)
+	return p.adminProcessor.DomainBlockCreate(authed.Account, form.Domain, form.Obfuscate, form.PublicComment, form.PrivateComment, "")
+}
+
+func (p *processor) AdminDomainBlocksImport(authed *oauth.Auth, form *apimodel.DomainBlockCreateRequest) ([]*apimodel.DomainBlock, gtserror.WithCode) {
+	return p.adminProcessor.DomainBlocksImport(authed.Account, form.Domains)
 }
 
 func (p *processor) AdminDomainBlocksGet(authed *oauth.Auth, export bool) ([]*apimodel.DomainBlock, gtserror.WithCode) {

internal/processing/admin/admin.go

@@ -19,6 +19,8 @@
 package admin
 
 import (
+	"mime/multipart"
+
 	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
@@ -31,7 +33,8 @@
 
 // Processor wraps a bunch of functions for processing admin actions.
 type Processor interface {
-	DomainBlockCreate(account *gtsmodel.Account, form *apimodel.DomainBlockCreateRequest) (*apimodel.DomainBlock, gtserror.WithCode)
+	DomainBlockCreate(account *gtsmodel.Account, domain string, obfuscate bool, publicComment string, privateComment string, subscriptionID string) (*apimodel.DomainBlock, gtserror.WithCode)
+	DomainBlocksImport(account *gtsmodel.Account, domains *multipart.FileHeader) ([]*apimodel.DomainBlock, gtserror.WithCode)
 	DomainBlocksGet(account *gtsmodel.Account, export bool) ([]*apimodel.DomainBlock, gtserror.WithCode)
 	DomainBlockGet(account *gtsmodel.Account, id string, export bool) (*apimodel.DomainBlock, gtserror.WithCode)
 	DomainBlockDelete(account *gtsmodel.Account, id string) (*apimodel.DomainBlock, gtserror.WithCode)

internal/processing/admin/createdomainblock.go

@@ -30,37 +30,38 @@
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 )
 
-func (p *processor) DomainBlockCreate(account *gtsmodel.Account, form *apimodel.DomainBlockCreateRequest) (*apimodel.DomainBlock, gtserror.WithCode) {
+func (p *processor) DomainBlockCreate(account *gtsmodel.Account, domain string, obfuscate bool, publicComment string, privateComment string, subscriptionID string) (*apimodel.DomainBlock, gtserror.WithCode) {
 	// first check if we already have a block -- if err == nil we already had a block so we can skip a whole lot of work
 	domainBlock := &gtsmodel.DomainBlock{}
-	err := p.db.GetWhere([]db.Where{{Key: "domain", Value: form.Domain, CaseInsensitive: true}}, domainBlock)
+	err := p.db.GetWhere([]db.Where{{Key: "domain", Value: domain, CaseInsensitive: true}}, domainBlock)
 	if err != nil {
 		if _, ok := err.(db.ErrNoEntries); !ok {
 			// something went wrong in the DB
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: db error checking for existence of domain block %s: %s", form.Domain, err))
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: db error checking for existence of domain block %s: %s", domain, err))
 		}
 
 		// there's no block for this domain yet so create one
 		// note: we take a new ulid from timestamp here in case we need to sort blocks
 		blockID, err := id.NewULID()
 		if err != nil {
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: error creating id for new domain block %s: %s", form.Domain, err))
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: error creating id for new domain block %s: %s", domain, err))
 		}
 
 		domainBlock = &gtsmodel.DomainBlock{
 			ID:                 blockID,
-			Domain:             form.Domain,
+			Domain:             domain,
 			CreatedByAccountID: account.ID,
-			PrivateComment:     form.PrivateComment,
+			PrivateComment:     privateComment,
-			PublicComment:      form.PublicComment,
+			PublicComment:      publicComment,
-			Obfuscate:          form.Obfuscate,
+			Obfuscate:          obfuscate,
+			SubscriptionID:     subscriptionID,
 		}
 
 		// put the new block in the database
 		if err := p.db.Put(domainBlock); err != nil {
 			if _, ok := err.(db.ErrAlreadyExists); !ok {
 				// there's a real error creating the block
-				return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: db error putting new domain block %s: %s", form.Domain, err))
+				return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: db error putting new domain block %s: %s", domain, err))
 			}
 		}
 
@@ -70,7 +71,7 @@ func (p *processor) DomainBlockCreate(account *gtsmodel.Account, form *apimodel.
 
 	mastoDomainBlock, err := p.tc.DomainBlockToMasto(domainBlock, false)
 	if err != nil {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: error converting domain block to frontend/masto representation %s: %s", form.Domain, err))
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("DomainBlockCreate: error converting domain block to frontend/masto representation %s: %s", domain, err))
 	}
 
 	return mastoDomainBlock, nil
@@ -140,7 +141,7 @@ func (p *processor) initiateDomainBlockSideEffects(account *gtsmodel.Account, bl
 			p.fromClientAPI <- gtsmodel.FromClientAPI{
 				APObjectType:   gtsmodel.ActivityStreamsPerson,
 				APActivityType: gtsmodel.ActivityStreamsDelete,
-				GTSModel:       a,
+				GTSModel:       block,
 				OriginAccount:  account,
 				TargetAccount:  a,
 			}

internal/processing/admin/deletedomainblock.go

@@ -1,7 +1,26 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package admin
 
 import (
 	"fmt"
+	"time"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -32,5 +51,33 @@ func (p *processor) DomainBlockDelete(account *gtsmodel.Account, id string) (*ap
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
+	// remove the domain block reference from the instance, if we have an entry for it
+	i := &gtsmodel.Instance{}
+	if err := p.db.GetWhere([]db.Where{
+		{Key: "domain", Value: domainBlock.Domain, CaseInsensitive: true},
+		{Key: "domain_block_id", Value: id},
+	}, i); err == nil {
+		i.SuspendedAt = time.Time{}
+		i.DomainBlockID = ""
+		if err := p.db.UpdateByID(i.ID, i); err != nil {
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("couldn't update database entry for instance %s: %s", domainBlock.Domain, err))
+		}
+	}
+
+	// unsuspend all accounts whose suspension origin was this domain block
+	// 1. remove the 'suspended_at' entry from their accounts
+	if err := p.db.UpdateWhere([]db.Where{
+		{Key: "suspension_origin", Value: domainBlock.ID},
+	}, "suspended_at", nil, &[]*gtsmodel.Account{}); err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("database error removing suspended_at from accounts: %s", err))
+	}
+
+	// 2. remove the 'suspension_origin' entry from their accounts
+	if err := p.db.UpdateWhere([]db.Where{
+		{Key: "suspension_origin", Value: domainBlock.ID},
+	}, "suspension_origin", nil, &[]*gtsmodel.Account{}); err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("database error removing suspension_origin from accounts: %s", err))
+	}
+
 	return mastoDomainBlock, nil
 }

internal/processing/admin/getdomainblock.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package admin
 
 import (

internal/processing/admin/getdomainblocks.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package admin
 
 import (

internal/processing/admin/importdomainblocks.go

@@ -0,0 +1,67 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// DomainBlocksImport handles the import of a bunch of domain blocks at once, by calling the DomainBlockCreate function for each domain in the provided file.
+func (p *processor) DomainBlocksImport(account *gtsmodel.Account, domains *multipart.FileHeader) ([]*apimodel.DomainBlock, gtserror.WithCode) {
+
+	f, err := domains.Open()
+	if err != nil {
+		return nil, gtserror.NewErrorBadRequest(fmt.Errorf("DomainBlocksImport: error opening attachment: %s", err))
+	}
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		return nil, gtserror.NewErrorBadRequest(fmt.Errorf("DomainBlocksImport: error reading attachment: %s", err))
+	}
+	if size == 0 {
+		return nil, gtserror.NewErrorBadRequest(errors.New("DomainBlocksImport: could not read provided attachment: size 0 bytes"))
+	}
+
+	d := []apimodel.DomainBlock{}
+	if err := json.Unmarshal(buf.Bytes(), &d); err != nil {
+		return nil, gtserror.NewErrorBadRequest(fmt.Errorf("DomainBlocksImport: could not read provided attachment: %s", err))
+	}
+
+	blocks := []*apimodel.DomainBlock{}
+	for _, d := range d {
+		block, err := p.DomainBlockCreate(account, d.Domain, false, d.PublicComment, "", "")
+
+		if err != nil {
+			return nil, err
+		}
+
+		blocks = append(blocks, block)
+	}
+
+	return blocks, nil
+}

internal/processing/fromclientapi.go

@@ -193,17 +193,17 @@ func (p *processor) processFromClientAPI(clientMsg gtsmodel.FromClientAPI) error
 			return p.federateStatusDelete(statusToDelete)
 		case gtsmodel.ActivityStreamsProfile, gtsmodel.ActivityStreamsPerson:
 			// DELETE ACCOUNT/PROFILE
-			accountToDelete, ok := clientMsg.GTSModel.(*gtsmodel.Account)
-			if !ok {
-				return errors.New("account was not parseable as *gtsmodel.Account")
-			}
 
-			var deletedBy string
+			// the origin of the delete could be either a domain block, or an action by another (or this) account
-			if clientMsg.OriginAccount != nil {
+			var origin string
-				deletedBy = clientMsg.OriginAccount.ID
+			if domainBlock, ok := clientMsg.GTSModel.(*gtsmodel.DomainBlock); ok {
+				// origin is a domain block
+				origin = domainBlock.ID
+			} else {
+				// origin is whichever account caused this message
+				origin = clientMsg.OriginAccount.ID
 			}
-
+			return p.accountProcessor.Delete(clientMsg.TargetAccount, origin)
-			return p.accountProcessor.Delete(accountToDelete, deletedBy)
 		}
 	}
 	return nil

internal/processing/processor.go

@@ -87,6 +87,8 @@ type Processor interface {
 	AdminEmojiCreate(authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error)
 	// AdminDomainBlockCreate handles the creation of a new domain block by an admin, using the given form.
 	AdminDomainBlockCreate(authed *oauth.Auth, form *apimodel.DomainBlockCreateRequest) (*apimodel.DomainBlock, gtserror.WithCode)
+	// AdminDomainBlocksImport handles the import of multiple domain blocks by an admin, using the given form.
+	AdminDomainBlocksImport(authed *oauth.Auth, form *apimodel.DomainBlockCreateRequest) ([]*apimodel.DomainBlock, gtserror.WithCode)
 	// AdminDomainBlocksGet returns a list of currently blocked domains.
 	AdminDomainBlocksGet(authed *oauth.Auth, export bool) ([]*apimodel.DomainBlock, gtserror.WithCode)
 	// AdminDomainBlockGet returns one domain block, specified by ID.