Store incoming Notes

2019-02-21 19:17:59 -05:00 · 2019-02-21 19:17:59 -05:00 · 3ddb8ad28f
parent 9191a9d987
commit 3ddb8ad28f
8 changed files with 291 additions and 7 deletions
--- a/lib/activitypub/activity.ts
+++ b/lib/activitypub/activity.ts
@ -28,10 +28,11 @@ export interface Undo extends Activity {
 }

 export interface Note extends Activity {
-	inReplyTo: string;
-	published: string;
-	content: string;
 	attributedTo: string;
+	content: string;
+	published: string;
+	inReplyTo: string;
+	conversation: string;
 }

 export interface Actor {
--- a/lib/activitypub/articles.ts
+++ b/lib/activitypub/articles.ts
@ -2,6 +2,7 @@ import express, { Router, Request, Response } from "express";
 import { Page, PostMetadata } from "../metadata";
 import { Article } from "./activity";
 import { Database } from "sqlite3";
+import uuidv4 from "uuid/v4";

 const domain = process.env.DOMAIN;

@ -16,6 +17,7 @@ export async function setup(posts: Page[], db: Database) {
 			"id": `https://${domain}${post.metadata.permalink}`,
 			"published": (<Date>postMeta.date).toISOString(),
 			"inReplyTo": null,
+			"conversation": `https://${domain}/ap/conversation/${uuidv4()}`,
 			"url": `https://${domain}${postMeta.permalink}`,
 			"attributedTo": `https://${domain}/ap/actor`,
 			"to": [
@ -27,9 +29,10 @@ export async function setup(posts: Page[], db: Database) {
 			"name": postMeta.title,
 			"content": post.text
 		};
-		db.run("INSERT OR IGNORE INTO articles(id, article_doc, has_federated) VALUES($id, $article_doc, $has_federated)", {
+		db.run("INSERT OR IGNORE INTO articles(id, article_doc, conversation, has_federated) VALUES($id, $article_doc, $conversation, $has_federated)", {
 			$id: postMeta.permalink,
 			$article_doc: JSON.stringify(articleObject),
+			$conversation: articleObject.conversation,
 			$has_federated: 0
 		}, (err) => {
 			if (err) console.log(`Encountered error inserting article ${postMeta.permalink}`, err);
--- a/lib/activitypub/conversation.ts
+++ b/lib/activitypub/conversation.ts
@ -0,0 +1,28 @@
+import { Router } from "express";
+import { Database } from "sqlite3";
+
+const domain = process.env.DOMAIN;
+
+export default function conversation(router: Router) {
+	router.get("/ap/conversation/:id", (req, res) => {
+		const db = req.app.get("db") as Database;
+		db.all("SELECT * FROM notes WHERE conversation = $conversation", {
+			$conversation: `https://${domain}/ap/conversation/${req.params.id}`
+		}, (err, rows) => {
+			if (err) {
+				res.status(500).end(err);
+			} else {
+				const notes = rows.map(row => {
+					return {
+						attributedTo: row.attributed_to,
+						content: row.content,
+						published: row.published,
+						inReplyTo: row.in_reply_to,
+						conversation: row.conversation
+					};
+				});
+				res.json(notes).end();
+			}
+		});
+	});
+}
--- a/lib/activitypub/inbox.ts
+++ b/lib/activitypub/inbox.ts
@ -4,6 +4,7 @@ import { fetchActor, signAndSend } from "./federate";
 import { Activity, Follow, Accept, Undo, Create, Note } from "./activity";
 import { Database } from "sqlite3";
 import { URL } from "url";
+import sanitizeHtml from "sanitize-html";

 const domain = process.env.DOMAIN;

@ -68,9 +69,33 @@ async function handleCreate(activity: Activity, req: Request, res: Response) {
 	}
 }

+function sanitizeStatus(content: string): string {
+	return sanitizeHtml(content, {
+		allowedTags: ["a", "span", "p", "br", "b", "strong", "i", "em", "s", "del", "u", "code", "pre", "ul", "ol", "li", "blockquote", "img"],
+		allowedAttributes: {
+			"a": ["href", "data-user"],
+			"img": ["src"]
+		},
+		transformTags: {
+			"a": sanitizeHtml.simpleTransform("a", { rel: "noopener", target: "_blank" })
+		}
+	});
+}
+
 async function handleCreateNote(create: Create, req: Request, res: Response) {
 	const note = create.object as Note;
-	console.log(note);
+	const db = req.app.get("db") as Database;
+	const sanitizedContent = sanitizeStatus(note.content);
+	db.run("INSERT OR IGNORE INTO notes(id, content, attributed_to, in_reply_to, conversation, published) VALUES($id, $content, $attributed_to, $in_reply_to, $conversation, $published)", {
+		$id: note.id,
+		$content: sanitizedContent,
+		$attributed_to: note.attributedTo,
+		$in_reply_to: note.inReplyTo,
+		$conversation: note.conversation,
+		$published: note.published
+	}, (err) => {
+		if (err) console.error(`Encountered error storing reply ${note.id}`, err);
+	});
 }

 async function handleUndo(activity: Activity, req: Request, res: Response) {
--- a/lib/activitypub/index.ts
+++ b/lib/activitypub/index.ts
@ -1,5 +1,6 @@
 import actor from "./actor";
 import * as articles from "./articles";
+import conversation from "./conversation";
 import federate from "./federate";
 import followers from "./followers";
 import inbox from "./inbox";
@ -8,6 +9,7 @@ import webfinger from "./webfinger";
 export = {
 	actor,
 	articles,
+	conversation,
 	federate,
 	followers,
 	inbox,
--- a/lib/index.ts
+++ b/lib/index.ts
@ -12,8 +12,8 @@ import sqlite3 from "sqlite3";
 const db = new (sqlite3.verbose().Database)(process.env.DB_PATH!);

 db.run("CREATE TABLE IF NOT EXISTS followers (id TEXT PRIMARY KEY, inbox TEXT)");
-db.run("CREATE TABLE IF NOT EXISTS articles (id TEXT PRIMARY KEY, article_doc TEXT, has_federated INT)");
-db.run("CREATE TABLE IF NOT EXISTS notes (id TEXT PRIMARY KEY, content TEXT, attributed_to TEXT, in_reply_to TEXT, published TEXT)");
+db.run("CREATE TABLE IF NOT EXISTS articles (id TEXT PRIMARY KEY, article_doc TEXT, conversation TEXT, has_federated INT)");
+db.run("CREATE TABLE IF NOT EXISTS notes (id TEXT PRIMARY KEY, content TEXT, attributed_to TEXT, in_reply_to TEXT, conversation TEXT, published TEXT)");

 async function generate(): Promise<Page[]> {
 	generators.copy();
@ -47,6 +47,7 @@ app.use(bodyParser.json({ type: "application/activity+json" }));
 	const apRouter = Router();
 	apRouter.use(validateHttpSig);
 	await activitypub.actor(apRouter);
+	activitypub.conversation(apRouter);
 	activitypub.followers(apRouter);
 	activitypub.inbox(apRouter);
 	activitypub.webfinger(apRouter);
--- a/package-lock.json
+++ b/package-lock.json
@ -74,6 +74,14 @@
      "integrity": "sha512-pGF/zvYOACZ/gLGWdQH8zSwteQS1epp68yRcVLJMgUck/MjEn/FBYmPub9pXT8C1e4a8YZfHo1CKyV8q1vKUnQ==",
      "dev": true
    },
+    "@types/htmlparser2": {
+      "version": "3.7.31",
+      "resolved": "https://registry.npmjs.org/@types/htmlparser2/-/htmlparser2-3.7.31.tgz",
+      "integrity": "sha512-6Kjy02k+KfJJE2uUiCytS31SXCYnTjKA+G0ydb83DTlMFzorBlezrV2XiKazRO5HSOEvVW3cpzDFPoP9n/9rSA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/linkify-it": {
      "version": "2.0.4",
      "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-2.0.4.tgz",
@ -132,6 +140,14 @@
        "@types/tough-cookie": "*"
      }
    },
+    "@types/sanitize-html": {
+      "version": "1.18.2",
+      "resolved": "https://registry.npmjs.org/@types/sanitize-html/-/sanitize-html-1.18.2.tgz",
+      "integrity": "sha512-WSE/HsqOHfHd1c0vPOOWOWNippsscBU72r5tpWT/+pFL3zBiCPJCp0NO7sQT8V0gU0xjSKpMAve3iMEJrRhUWQ==",
+      "requires": {
+        "@types/htmlparser2": "*"
+      }
+    },
    "@types/serve-static": {
      "version": "1.13.2",
      "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.2.tgz",
@ -240,6 +256,11 @@
      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
      "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
    },
+    "array-uniq": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz",
+      "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY="
+    },
    "asn1": {
      "version": "0.2.4",
      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@ -405,6 +426,19 @@
      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
    },
+    "color-convert": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+      "requires": {
+        "color-name": "1.1.3"
+      }
+    },
+    "color-name": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
+    },
    "colors": {
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
@ -534,6 +568,37 @@
      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
      "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
    },
+    "dom-serializer": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
+      "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
+      "requires": {
+        "domelementtype": "^1.3.0",
+        "entities": "^1.1.1"
+      }
+    },
+    "domelementtype": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
+    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "requires": {
+        "domelementtype": "1"
+      }
+    },
+    "domutils": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz",
+      "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==",
+      "requires": {
+        "dom-serializer": "0",
+        "domelementtype": "1"
+      }
+    },
    "ecc-jsbn": {
      "version": "0.1.2",
      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@ -853,6 +918,11 @@
        "ansi-regex": "^2.0.0"
      }
    },
+    "has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
+    },
    "has-unicode": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
@ -874,6 +944,31 @@
      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz",
      "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w=="
    },
+    "htmlparser2": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
+      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
+      "requires": {
+        "domelementtype": "^1.3.1",
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^3.1.1"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.1.1.tgz",
+          "integrity": "sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
+      }
+    },
    "http-errors": {
      "version": "1.6.3",
      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
@ -1118,6 +1213,21 @@
      "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz",
      "integrity": "sha1-3bG7s+8HRYwBd7oH3hRCLLAz/5s="
    },
+    "lodash.escaperegexp": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz",
+      "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c="
+    },
+    "lodash.isplainobject": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
+      "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
+    },
+    "lodash.isstring": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
+      "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE="
+    },
    "lodash.mergewith": {
      "version": "4.6.1",
      "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.1.tgz",
@ -1639,6 +1749,59 @@
        }
      }
    },
+    "postcss": {
+      "version": "7.0.14",
+      "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.14.tgz",
+      "integrity": "sha512-NsbD6XUUMZvBxtQAJuWDJeeC4QFsmWsfozWxCJPWf3M55K9iu2iMDaKqyoOdTJ1R4usBXuxlVFAIo8rZPQD4Bg==",
+      "requires": {
+        "chalk": "^2.4.2",
+        "source-map": "^0.6.1",
+        "supports-color": "^6.1.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.2",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+          "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          },
+          "dependencies": {
+            "supports-color": {
+              "version": "5.5.0",
+              "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+              "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+              "requires": {
+                "has-flag": "^3.0.0"
+              }
+            }
+          }
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
+        },
+        "supports-color": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz",
+          "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==",
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
    "process-nextick-args": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
@ -1811,6 +1974,51 @@
      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
    },
+    "sanitize-html": {
+      "version": "1.20.0",
+      "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-1.20.0.tgz",
+      "integrity": "sha512-BpxXkBoAG+uKCHjoXFmox6kCSYpnulABoGcZ/R3QyY9ndXbIM5S94eOr1IqnzTG8TnbmXaxWoDDzKC5eJv7fEQ==",
+      "requires": {
+        "chalk": "^2.4.1",
+        "htmlparser2": "^3.10.0",
+        "lodash.clonedeep": "^4.5.0",
+        "lodash.escaperegexp": "^4.1.2",
+        "lodash.isplainobject": "^4.0.6",
+        "lodash.isstring": "^4.0.1",
+        "lodash.mergewith": "^4.6.1",
+        "postcss": "^7.0.5",
+        "srcset": "^1.0.0",
+        "xtend": "^4.0.1"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.2",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+          "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "supports-color": {
+          "version": "5.5.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+          "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
    "sass-graph": {
      "version": "2.2.4",
      "resolved": "https://registry.npmjs.org/sass-graph/-/sass-graph-2.2.4.tgz",
@ -1960,6 +2168,15 @@
        }
      }
    },
+    "srcset": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/srcset/-/srcset-1.0.0.tgz",
+      "integrity": "sha1-pWad4StC87HV6D7QPHEEb8SPQe8=",
+      "requires": {
+        "array-uniq": "^1.0.2",
+        "number-is-nan": "^1.0.0"
+      }
+    },
    "sshpk": {
      "version": "1.16.1",
      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
@ -2228,6 +2445,11 @@
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
    },
+    "xtend": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
+      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
+    },
    "y18n": {
      "version": "3.2.1",
      "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz",
--- a/package.json
+++ b/package.json
@ -14,6 +14,7 @@
 		"@types/express": "^4.16.1",
 		"@types/morgan": "^1.7.35",
 		"@types/request": "^2.48.1",
+		"@types/sanitize-html": "^1.18.2",
 		"@types/sqlite3": "^3.1.4",
 		"@types/uuid": "^3.4.4",
 		"body-parser": "^1.18.3",
@ -25,6 +26,7 @@
 		"morgan": "^1.9.1",
 		"node-sass": "^4.11.0",
 		"request": "^2.88.0",
+		"sanitize-html": "^1.20.0",
 		"sqlite3": "^4.0.6",
 		"typescript": "^3.2.2",
 		"uuid": "^3.3.2"