From 3ddb8ad28f31c3da20b71400548b39f5f3dbcc52 Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Thu, 21 Feb 2019 19:17:59 -0500 Subject: [PATCH] Store incoming Notes --- lib/activitypub/activity.ts | 7 +- lib/activitypub/articles.ts | 5 +- lib/activitypub/conversation.ts | 28 ++++ lib/activitypub/inbox.ts | 27 +++- lib/activitypub/index.ts | 2 + lib/index.ts | 5 +- package-lock.json | 222 ++++++++++++++++++++++++++++++++ package.json | 2 + 8 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 lib/activitypub/conversation.ts diff --git a/lib/activitypub/activity.ts b/lib/activitypub/activity.ts index 365fded..a118ef5 100644 --- a/lib/activitypub/activity.ts +++ b/lib/activitypub/activity.ts @@ -28,10 +28,11 @@ export interface Undo extends Activity { } export interface Note extends Activity { - inReplyTo: string; - published: string; - content: string; attributedTo: string; + content: string; + published: string; + inReplyTo: string; + conversation: string; } export interface Actor { diff --git a/lib/activitypub/articles.ts b/lib/activitypub/articles.ts index 8608db1..8d90d87 100644 --- a/lib/activitypub/articles.ts +++ b/lib/activitypub/articles.ts @@ -2,6 +2,7 @@ import express, { Router, Request, Response } from "express"; import { Page, PostMetadata } from "../metadata"; import { Article } from "./activity"; import { Database } from "sqlite3"; +import uuidv4 from "uuid/v4"; const domain = process.env.DOMAIN; @@ -16,6 +17,7 @@ export async function setup(posts: Page[], db: Database) { "id": `https://${domain}${post.metadata.permalink}`, "published": (postMeta.date).toISOString(), "inReplyTo": null, + "conversation": `https://${domain}/ap/conversation/${uuidv4()}`, "url": `https://${domain}${postMeta.permalink}`, "attributedTo": `https://${domain}/ap/actor`, "to": [ @@ -27,9 +29,10 @@ export async function setup(posts: Page[], db: Database) { "name": postMeta.title, "content": post.text }; - db.run("INSERT OR IGNORE INTO articles(id, article_doc, has_federated) VALUES($id, $article_doc, $has_federated)", { + db.run("INSERT OR IGNORE INTO articles(id, article_doc, conversation, has_federated) VALUES($id, $article_doc, $conversation, $has_federated)", { $id: postMeta.permalink, $article_doc: JSON.stringify(articleObject), + $conversation: articleObject.conversation, $has_federated: 0 }, (err) => { if (err) console.log(`Encountered error inserting article ${postMeta.permalink}`, err); diff --git a/lib/activitypub/conversation.ts b/lib/activitypub/conversation.ts new file mode 100644 index 0000000..e14cdc7 --- /dev/null +++ b/lib/activitypub/conversation.ts @@ -0,0 +1,28 @@ +import { Router } from "express"; +import { Database } from "sqlite3"; + +const domain = process.env.DOMAIN; + +export default function conversation(router: Router) { + router.get("/ap/conversation/:id", (req, res) => { + const db = req.app.get("db") as Database; + db.all("SELECT * FROM notes WHERE conversation = $conversation", { + $conversation: `https://${domain}/ap/conversation/${req.params.id}` + }, (err, rows) => { + if (err) { + res.status(500).end(err); + } else { + const notes = rows.map(row => { + return { + attributedTo: row.attributed_to, + content: row.content, + published: row.published, + inReplyTo: row.in_reply_to, + conversation: row.conversation + }; + }); + res.json(notes).end(); + } + }); + }); +} \ No newline at end of file diff --git a/lib/activitypub/inbox.ts b/lib/activitypub/inbox.ts index e960b37..d1ab0fb 100644 --- a/lib/activitypub/inbox.ts +++ b/lib/activitypub/inbox.ts @@ -4,6 +4,7 @@ import { fetchActor, signAndSend } from "./federate"; import { Activity, Follow, Accept, Undo, Create, Note } from "./activity"; import { Database } from "sqlite3"; import { URL } from "url"; +import sanitizeHtml from "sanitize-html"; const domain = process.env.DOMAIN; @@ -68,9 +69,33 @@ async function handleCreate(activity: Activity, req: Request, res: Response) { } } +function sanitizeStatus(content: string): string { + return sanitizeHtml(content, { + allowedTags: ["a", "span", "p", "br", "b", "strong", "i", "em", "s", "del", "u", "code", "pre", "ul", "ol", "li", "blockquote", "img"], + allowedAttributes: { + "a": ["href", "data-user"], + "img": ["src"] + }, + transformTags: { + "a": sanitizeHtml.simpleTransform("a", { rel: "noopener", target: "_blank" }) + } + }); +} + async function handleCreateNote(create: Create, req: Request, res: Response) { const note = create.object as Note; - console.log(note); + const db = req.app.get("db") as Database; + const sanitizedContent = sanitizeStatus(note.content); + db.run("INSERT OR IGNORE INTO notes(id, content, attributed_to, in_reply_to, conversation, published) VALUES($id, $content, $attributed_to, $in_reply_to, $conversation, $published)", { + $id: note.id, + $content: sanitizedContent, + $attributed_to: note.attributedTo, + $in_reply_to: note.inReplyTo, + $conversation: note.conversation, + $published: note.published + }, (err) => { + if (err) console.error(`Encountered error storing reply ${note.id}`, err); + }); } async function handleUndo(activity: Activity, req: Request, res: Response) { diff --git a/lib/activitypub/index.ts b/lib/activitypub/index.ts index a9309e4..0cd48a1 100644 --- a/lib/activitypub/index.ts +++ b/lib/activitypub/index.ts @@ -1,5 +1,6 @@ import actor from "./actor"; import * as articles from "./articles"; +import conversation from "./conversation"; import federate from "./federate"; import followers from "./followers"; import inbox from "./inbox"; @@ -8,6 +9,7 @@ import webfinger from "./webfinger"; export = { actor, articles, + conversation, federate, followers, inbox, diff --git a/lib/index.ts b/lib/index.ts index 9be5390..04cd2e2 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -12,8 +12,8 @@ import sqlite3 from "sqlite3"; const db = new (sqlite3.verbose().Database)(process.env.DB_PATH!); db.run("CREATE TABLE IF NOT EXISTS followers (id TEXT PRIMARY KEY, inbox TEXT)"); -db.run("CREATE TABLE IF NOT EXISTS articles (id TEXT PRIMARY KEY, article_doc TEXT, has_federated INT)"); -db.run("CREATE TABLE IF NOT EXISTS notes (id TEXT PRIMARY KEY, content TEXT, attributed_to TEXT, in_reply_to TEXT, published TEXT)"); +db.run("CREATE TABLE IF NOT EXISTS articles (id TEXT PRIMARY KEY, article_doc TEXT, conversation TEXT, has_federated INT)"); +db.run("CREATE TABLE IF NOT EXISTS notes (id TEXT PRIMARY KEY, content TEXT, attributed_to TEXT, in_reply_to TEXT, conversation TEXT, published TEXT)"); async function generate(): Promise { generators.copy(); @@ -47,6 +47,7 @@ app.use(bodyParser.json({ type: "application/activity+json" })); const apRouter = Router(); apRouter.use(validateHttpSig); await activitypub.actor(apRouter); + activitypub.conversation(apRouter); activitypub.followers(apRouter); activitypub.inbox(apRouter); activitypub.webfinger(apRouter); diff --git a/package-lock.json b/package-lock.json index 8e20203..7bc61b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,6 +74,14 @@ "integrity": "sha512-pGF/zvYOACZ/gLGWdQH8zSwteQS1epp68yRcVLJMgUck/MjEn/FBYmPub9pXT8C1e4a8YZfHo1CKyV8q1vKUnQ==", "dev": true }, + "@types/htmlparser2": { + "version": "3.7.31", + "resolved": "https://registry.npmjs.org/@types/htmlparser2/-/htmlparser2-3.7.31.tgz", + "integrity": "sha512-6Kjy02k+KfJJE2uUiCytS31SXCYnTjKA+G0ydb83DTlMFzorBlezrV2XiKazRO5HSOEvVW3cpzDFPoP9n/9rSA==", + "requires": { + "@types/node": "*" + } + }, "@types/linkify-it": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-2.0.4.tgz", @@ -132,6 +140,14 @@ "@types/tough-cookie": "*" } }, + "@types/sanitize-html": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/@types/sanitize-html/-/sanitize-html-1.18.2.tgz", + "integrity": "sha512-WSE/HsqOHfHd1c0vPOOWOWNippsscBU72r5tpWT/+pFL3zBiCPJCp0NO7sQT8V0gU0xjSKpMAve3iMEJrRhUWQ==", + "requires": { + "@types/htmlparser2": "*" + } + }, "@types/serve-static": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.2.tgz", @@ -240,6 +256,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=" + }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -405,6 +426,19 @@ "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, "colors": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", @@ -534,6 +568,37 @@ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -853,6 +918,11 @@ "ansi-regex": "^2.0.0" } }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + }, "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", @@ -874,6 +944,31 @@ "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==" }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.1.1.tgz", + "integrity": "sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "http-errors": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", @@ -1118,6 +1213,21 @@ "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", "integrity": "sha1-3bG7s+8HRYwBd7oH3hRCLLAz/5s=" }, + "lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=" + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, + "lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" + }, "lodash.mergewith": { "version": "4.6.1", "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.1.tgz", @@ -1639,6 +1749,59 @@ } } }, + "postcss": { + "version": "7.0.14", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.14.tgz", + "integrity": "sha512-NsbD6XUUMZvBxtQAJuWDJeeC4QFsmWsfozWxCJPWf3M55K9iu2iMDaKqyoOdTJ1R4usBXuxlVFAIo8rZPQD4Bg==", + "requires": { + "chalk": "^2.4.2", + "source-map": "^0.6.1", + "supports-color": "^6.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "dependencies": { + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, + "supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, "process-nextick-args": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", @@ -1811,6 +1974,51 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sanitize-html": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-1.20.0.tgz", + "integrity": "sha512-BpxXkBoAG+uKCHjoXFmox6kCSYpnulABoGcZ/R3QyY9ndXbIM5S94eOr1IqnzTG8TnbmXaxWoDDzKC5eJv7fEQ==", + "requires": { + "chalk": "^2.4.1", + "htmlparser2": "^3.10.0", + "lodash.clonedeep": "^4.5.0", + "lodash.escaperegexp": "^4.1.2", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.mergewith": "^4.6.1", + "postcss": "^7.0.5", + "srcset": "^1.0.0", + "xtend": "^4.0.1" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, "sass-graph": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/sass-graph/-/sass-graph-2.2.4.tgz", @@ -1960,6 +2168,15 @@ } } }, + "srcset": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/srcset/-/srcset-1.0.0.tgz", + "integrity": "sha1-pWad4StC87HV6D7QPHEEb8SPQe8=", + "requires": { + "array-uniq": "^1.0.2", + "number-is-nan": "^1.0.0" + } + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -2228,6 +2445,11 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, "y18n": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", diff --git a/package.json b/package.json index f7a27a3..1a9994c 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@types/express": "^4.16.1", "@types/morgan": "^1.7.35", "@types/request": "^2.48.1", + "@types/sanitize-html": "^1.18.2", "@types/sqlite3": "^3.1.4", "@types/uuid": "^3.4.4", "body-parser": "^1.18.3", @@ -25,6 +26,7 @@ "morgan": "^1.9.1", "node-sass": "^4.11.0", "request": "^2.88.0", + "sanitize-html": "^1.20.0", "sqlite3": "^4.0.6", "typescript": "^3.2.2", "uuid": "^3.3.2"