gotosocial/web/source/settings/lib/query/admin/import-export.js
f0x52 a59dc855d9
[feature/frogend] (Mastodon) domain block CSV import (#1390)
* checkbox-list styling with taller <p> element

* CSV import/export, UI/UX improvements to import-export interface

* minor styling tweaks

* csv export, clean up export type branching

* abstract domain block entry validation

* foundation for PSL check + suggestions

* Squashed commit of the following:

commit e3655ba4fb
Author: f0x <f0x@cthu.lu>
Date:   Tue Jan 31 15:19:10 2023 +0100

    let debug depend on env (prod/debug) again

commit 79c792b832
Author: f0x <f0x@cthu.lu>
Date:   Tue Jan 31 00:34:01 2023 +0100

    update checklist components

commit 4367960fe4
Author: f0x <f0x@cthu.lu>
Date:   Mon Jan 30 23:46:20 2023 +0100

    checklist performance improvements

commit 204a4c02d1
Author: f0x <f0x@cthu.lu>
Date:   Mon Jan 30 20:05:34 2023 +0100

    checklist field: use reducer for state

* remove debug logging

* show and use domain block suggestion

* restructure import/export buttons

* updating suggestions

* suggestion overview

* restructure check-list behavior, domain import/export
2023-02-03 12:07:40 +01:00

265 lines
6.6 KiB
JavaScript

/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const Promise = require("bluebird");
const fileDownload = require("js-file-download");
const csv = require("papaparse");
const { nanoid } = require("nanoid");
const { isValidDomainBlock, hasBetterScope } = require("../../domain-block");
const {
replaceCacheOnMutation,
domainListToObject,
unwrapRes
} = require("../lib");
function parseDomainList(list) {
if (list[0] == "[") {
return JSON.parse(list);
} else if (list.startsWith("#domain")) { // Mastodon CSV
const { data, errors } = csv.parse(list, {
header: true,
transformHeader: (header) => header.slice(1), // removes starting '#'
skipEmptyLines: true,
dynamicTyping: true
});
if (errors.length > 0) {
let error = "";
errors.forEach((err) => {
error += `${err.message} (line ${err.row})`;
});
throw error;
}
return data;
} else {
return list.split("\n").map((line) => {
let domain = line.trim();
let valid = true;
if (domain.startsWith("http")) {
try {
domain = new URL(domain).hostname;
} catch (e) {
valid = false;
}
}
return domain.length > 0
? { domain, valid }
: null;
}).filter((a) => a); // not `null`
}
}
function validateDomainList(list) {
list.forEach((entry) => {
if (entry.domain.startsWith("*.")) {
// domain block always includes all subdomains, wildcard is meaningless here
entry.domain = entry.domain.slice(2);
}
entry.valid = (entry.valid !== false) && isValidDomainBlock(entry.domain);
if (entry.valid) {
entry.suggest = hasBetterScope(entry.domain);
}
entry.checked = entry.valid;
});
return list;
}
function deduplicateDomainList(list) {
let domains = new Set();
return list.filter((entry) => {
if (domains.has(entry.domain)) {
return false;
} else {
domains.add(entry.domain);
return true;
}
});
}
module.exports = (build) => ({
processDomainList: build.mutation({
queryFn: (formData) => {
return Promise.try(() => {
if (formData.domains == undefined || formData.domains.length == 0) {
throw "No domains entered";
}
return parseDomainList(formData.domains);
}).then((parsed) => {
return deduplicateDomainList(parsed);
}).then((deduped) => {
return validateDomainList(deduped);
}).then((data) => {
data.forEach((entry) => {
entry.key = nanoid(); // unique id that stays stable even if domain gets modified by user
});
return { data };
}).catch((e) => {
return { error: e.toString() };
});
}
}),
exportDomainList: build.mutation({
queryFn: (formData, api, _extraOpts, baseQuery) => {
let process;
if (formData.exportType == "json") {
process = {
transformEntry: (entry) => ({
domain: entry.domain,
public_comment: entry.public_comment,
obfuscate: entry.obfuscate
}),
stringify: (list) => JSON.stringify(list),
extension: ".json",
mime: "application/json"
};
} else if (formData.exportType == "csv") {
process = {
transformEntry: (entry) => [
entry.domain,
"suspend", // severity
false, // reject_media
false, // reject_reports
entry.public_comment,
entry.obfuscate ?? false
],
stringify: (list) => csv.unparse({
fields: "#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate".split(","),
data: list
}),
extension: ".csv",
mime: "text/csv"
};
} else {
process = {
transformEntry: (entry) => entry.domain,
stringify: (list) => list.join("\n"),
extension: ".txt",
mime: "text/plain"
};
}
return Promise.try(() => {
return baseQuery({
url: `/api/v1/admin/domain_blocks`
});
}).then(unwrapRes).then((blockedInstances) => {
return blockedInstances.map(process.transformEntry);
}).then((exportList) => {
return process.stringify(exportList);
}).then((exportAsString) => {
if (formData.action == "export") {
return {
data: exportAsString
};
} else if (formData.action == "export-file") {
let domain = new URL(api.getState().oauth.instance).host;
let date = new Date();
let filename = [
domain,
"blocklist",
date.getFullYear(),
(date.getMonth() + 1).toString().padStart(2, "0"),
date.getDate().toString().padStart(2, "0"),
].join("-");
fileDownload(
exportAsString,
filename + process.extension,
process.mime
);
}
return { data: null };
}).catch((e) => {
return { error: e };
});
}
}),
importDomainList: build.mutation({
query: (formData) => {
const { domains } = formData;
// add/replace comments, obfuscation data
let process = entryProcessor(formData);
domains.forEach((entry) => {
process(entry);
});
return {
method: "POST",
url: `/api/v1/admin/domain_blocks?import=true`,
asForm: true,
discardEmpty: true,
body: {
domains: new Blob([JSON.stringify(domains)], { type: "application/json" })
}
};
},
transformResponse: domainListToObject,
...replaceCacheOnMutation("instanceBlocks")
})
});
const internalKeys = new Set("key,suggest,valid,checked".split(","));
function entryProcessor(formData) {
let funcs = [];
["private_comment", "public_comment"].forEach((type) => {
let text = formData[type].trim();
if (text.length > 0) {
let behavior = formData[`${type}_behavior`];
if (behavior == "append") {
funcs.push(function appendComment(entry) {
if (entry[type] == undefined) {
entry[type] = text;
} else {
entry[type] = [entry[type], text].join("\n");
}
});
} else if (behavior == "replace") {
funcs.push(function replaceComment(entry) {
entry[type] = text;
});
}
}
});
return function process(entry) {
funcs.forEach((func) => {
func(entry);
});
entry.obfuscate = formData.obfuscate;
Object.entries(entry).forEach(([key, val]) => {
if (internalKeys.has(key) || val == undefined) {
delete entry[key];
}
});
};
}