Implement research papers

10 files changed, 496 insertions, 111 deletions

diff --git a/backend/index.ts b/backend/index.ts
index 9efec62..9849e50 100644
--- a/backend/index.ts
+++ b/backend/index.ts
@@ -7,12 +7,21 @@ import {
     DatabaseTransactionConnectionType,
     sql,
 } from "slonik";
+import { parse } from "node-html-parser";
+import got from "got";
+import { decode } from "he";
+
 import * as config from "./config";
-import { ValueRecord } from "./shared-types";
+import { Paper, ValueRecord } from "./shared-types";
 import { Server as SocketIOServer } from "socket.io";
 
 const app = express();
 
+app.use((req, res, next) => {
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    next();
+});
+
 const httpServer = http.createServer(app);
 const io = new SocketIOServer(httpServer, {
     cors: {
@@ -53,6 +62,29 @@ io.on("connection", async (socket) => {
     });
 });
 
+const extractDataFromHtml = (html: string): Paper[] => {
+    const dom = parse(html);
+
+    const tds = dom.querySelectorAll(".gsc_a_tr .gsc_a_t a");
+    const data = tds.map(td => ({
+        url: `https://scholar.google.com${td.getAttribute("data-href") || ""}`,
+        title: decode(td.textContent),
+    }));
+
+    return data;
+};
+
+app.get("/api/papers", async (req, res) => {
+    const resp = await got("https://scholar.google.com/citations?user=HMp8VyMAAAAJ", {
+        responseType: "text",
+        encoding: "latin1",
+    });
+    const html = resp.body;
+    const papers = extractDataFromHtml(html);
+    res.contentType("json");
+    res.end(JSON.stringify(papers));
+});
+
 httpServer.listen(config.port, () => {
     console.log(`Listening on http://localhost:${config.port}`);
 });
diff --git a/backend/package-lock.json b/backend/package-lock.json
index e3a68f9..9213df1 100644
--- a/backend/package-lock.json
+++ b/backend/package-lock.json
@@ -113,10 +113,9 @@
       }
     },
     "@sindresorhus/is": {
-      "version": "0.14.0",
-      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",
-      "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==",
-      "dev": true
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.1.tgz",
+      "integrity": "sha512-Qm9hBEBu18wt1PO2flE7LPb30BHMQt1eQgbV76YntdNk73XZGpn3izvGTYxbGgzXKgbCjiia0uxTd3aTNQrY/g=="
     },
     "@slonik/migrator": {
       "version": "0.8.3",
@@ -128,12 +127,11 @@
       }
     },
     "@szmarczak/http-timer": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz",
-      "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==",
-      "dev": true,
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz",
+      "integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==",
       "requires": {
-        "defer-to-connect": "^1.0.1"
+        "defer-to-connect": "^2.0.0"
       }
     },
     "@tsconfig/node10": {
@@ -176,6 +174,17 @@
         "@types/node": "*"
       }
     },
+    "@types/cacheable-request": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz",
+      "integrity": "sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==",
+      "requires": {
+        "@types/http-cache-semantics": "*",
+        "@types/keyv": "*",
+        "@types/node": "*",
+        "@types/responselike": "*"
+      }
+    },
     "@types/component-emitter": {
       "version": "1.2.10",
       "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.10.tgz",
@@ -223,12 +232,31 @@
         "@types/range-parser": "*"
       }
     },
+    "@types/he": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@types/he/-/he-1.1.1.tgz",
+      "integrity": "sha512-jpzrsR1ns0n3kyWt92QfOUQhIuJGQ9+QGa7M62rO6toe98woQjnsnzjdMtsQXCdvjjmqjS2ZBCC7xKw0cdzU+Q==",
+      "dev": true
+    },
+    "@types/http-cache-semantics": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz",
+      "integrity": "sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A=="
+    },
     "@types/json-schema": {
       "version": "7.0.7",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz",
       "integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==",
       "dev": true
     },
+    "@types/keyv": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz",
+      "integrity": "sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/mime": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@@ -252,6 +280,14 @@
       "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==",
       "dev": true
     },
+    "@types/responselike": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
+      "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/serve-static": {
       "version": "1.13.9",
       "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.9.tgz",
@@ -594,6 +630,11 @@
         }
       }
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
     "boolean": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.1.2.tgz",
@@ -720,36 +761,23 @@
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
       "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
     },
+    "cacheable-lookup": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
+      "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="
+    },
     "cacheable-request": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz",
-      "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==",
-      "dev": true,
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz",
+      "integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==",
       "requires": {
         "clone-response": "^1.0.2",
         "get-stream": "^5.1.0",
         "http-cache-semantics": "^4.0.0",
-        "keyv": "^3.0.0",
+        "keyv": "^4.0.0",
         "lowercase-keys": "^2.0.0",
-        "normalize-url": "^4.1.0",
-        "responselike": "^1.0.2"
-      },
-      "dependencies": {
-        "get-stream": {
-          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
-          "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
-          "dev": true,
-          "requires": {
-            "pump": "^3.0.0"
-          }
-        },
-        "lowercase-keys": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
-          "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==",
-          "dev": true
-        }
+        "normalize-url": "^6.0.1",
+        "responselike": "^2.0.0"
       }
     },
     "callsites": {
@@ -847,7 +875,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",
       "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=",
-      "dev": true,
       "requires": {
         "mimic-response": "^1.0.0"
       }
@@ -977,6 +1004,23 @@
       "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==",
       "dev": true
     },
+    "css-select": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz",
+      "integrity": "sha512-gT3wBNd9Nj49rAbmtFHj1cljIAOLYSX1nZ8CB7TBO3INYckygm5B7LISU/szY//YmdiSLbJvDLOx9VnMVpMBxA==",
+      "requires": {
+        "boolbase": "^1.0.0",
+        "css-what": "^5.0.0",
+        "domhandler": "^4.2.0",
+        "domutils": "^2.6.0",
+        "nth-check": "^2.0.0"
+      }
+    },
+    "css-what": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-5.0.1.tgz",
+      "integrity": "sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg=="
+    },
     "debug": {
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
@@ -986,12 +1030,18 @@
       }
     },
     "decompress-response": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
-      "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
-      "dev": true,
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
       "requires": {
-        "mimic-response": "^1.0.0"
+        "mimic-response": "^3.1.0"
+      },
+      "dependencies": {
+        "mimic-response": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+          "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
+        }
       }
     },
     "deep-extend": {
@@ -1012,10 +1062,9 @@
       "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg=="
     },
     "defer-to-connect": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz",
-      "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==",
-      "dev": true
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
+      "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg=="
     },
     "define-properties": {
       "version": "1.1.3",
@@ -1069,6 +1118,39 @@
         "esutils": "^2.0.2"
       }
     },
+    "dom-serializer": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.3.2.tgz",
+      "integrity": "sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig==",
+      "requires": {
+        "domelementtype": "^2.0.1",
+        "domhandler": "^4.2.0",
+        "entities": "^2.0.0"
+      }
+    },
+    "domelementtype": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.2.0.tgz",
+      "integrity": "sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A=="
+    },
+    "domhandler": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.2.0.tgz",
+      "integrity": "sha512-zk7sgt970kzPks2Bf+dwT/PLzghLnsivb9CcxkvR8Mzr66Olr0Ofd8neSbglHJHaHa2MadfoSdNlKYAaafmWfA==",
+      "requires": {
+        "domelementtype": "^2.2.0"
+      }
+    },
+    "domutils": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.7.0.tgz",
+      "integrity": "sha512-8eaHa17IwJUPAiB+SoTYBo5mCdeMgdcAoXJ59m6DT1vw+5iLS3gNoqYaRowaBKtGVrOF1Jz4yDTgYKLK2kvfJg==",
+      "requires": {
+        "dom-serializer": "^1.0.1",
+        "domelementtype": "^2.2.0",
+        "domhandler": "^4.2.0"
+      }
+    },
     "dot-prop": {
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz",
@@ -1116,7 +1198,6 @@
       "version": "1.4.4",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
       "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-      "dev": true,
       "requires": {
         "once": "^1.4.0"
       }
@@ -1159,6 +1240,11 @@
         "ansi-colors": "^4.1.1"
       }
     },
+    "entities": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
+      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
+    },
     "es6-error": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
@@ -1597,10 +1683,9 @@
       }
     },
     "get-stream": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
-      "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
-      "dev": true,
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
       "requires": {
         "pump": "^3.0.0"
       }
@@ -1669,22 +1754,21 @@
       }
     },
     "got": {
-      "version": "9.6.0",
-      "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz",
-      "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==",
-      "dev": true,
+      "version": "11.8.2",
+      "resolved": "https://registry.npmjs.org/got/-/got-11.8.2.tgz",
+      "integrity": "sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==",
       "requires": {
-        "@sindresorhus/is": "^0.14.0",
-        "@szmarczak/http-timer": "^1.1.2",
-        "cacheable-request": "^6.0.0",
-        "decompress-response": "^3.3.0",
-        "duplexer3": "^0.1.4",
-        "get-stream": "^4.1.0",
-        "lowercase-keys": "^1.0.1",
-        "mimic-response": "^1.0.1",
-        "p-cancelable": "^1.0.0",
-        "to-readable-stream": "^1.0.0",
-        "url-parse-lax": "^3.0.0"
+        "@sindresorhus/is": "^4.0.0",
+        "@szmarczak/http-timer": "^4.0.5",
+        "@types/cacheable-request": "^6.0.1",
+        "@types/responselike": "^1.0.0",
+        "cacheable-lookup": "^5.0.3",
+        "cacheable-request": "^7.0.1",
+        "decompress-response": "^6.0.0",
+        "http2-wrapper": "^1.0.0-beta.5.2",
+        "lowercase-keys": "^2.0.0",
+        "p-cancelable": "^2.0.0",
+        "responselike": "^2.0.0"
       }
     },
     "graceful-fs": {
@@ -1705,11 +1789,15 @@
       "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==",
       "dev": true
     },
+    "he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="
+    },
     "http-cache-semantics": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
-      "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==",
-      "dev": true
+      "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
     },
     "http-errors": {
       "version": "1.7.2",
@@ -1730,6 +1818,15 @@
         }
       }
     },
+    "http2-wrapper": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz",
+      "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==",
+      "requires": {
+        "quick-lru": "^5.1.1",
+        "resolve-alpn": "^1.0.0"
+      }
+    },
     "hyperid": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-2.1.0.tgz",
@@ -1935,10 +2032,9 @@
       }
     },
     "json-buffer": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz",
-      "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=",
-      "dev": true
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+      "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
     },
     "json-schema-traverse": {
       "version": "0.4.1",
@@ -1957,12 +2053,11 @@
       "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
     },
     "keyv": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz",
-      "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==",
-      "dev": true,
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz",
+      "integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==",
       "requires": {
-        "json-buffer": "3.0.0"
+        "json-buffer": "3.0.1"
       }
     },
     "latest-version": {
@@ -2008,10 +2103,9 @@
       "dev": true
     },
     "lowercase-keys": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
-      "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==",
-      "dev": true
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
+      "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
     },
     "lru-cache": {
       "version": "6.0.0",
@@ -2097,8 +2191,7 @@
     "mimic-response": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
-      "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
-      "dev": true
+      "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ=="
     },
     "minimatch": {
       "version": "3.0.4",
@@ -2136,6 +2229,15 @@
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
       "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
     },
+    "node-html-parser": {
+      "version": "3.3.6",
+      "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-3.3.6.tgz",
+      "integrity": "sha512-VkWDHvNgFGB3mbQGMyzqRE1i/BG7TKX9wRXC8e/v8kL0kZR/Oy6RjYxXH91K6/+m3g8iQ8dTqRy75lTYoA2Cjg==",
+      "requires": {
+        "css-select": "^4.1.3",
+        "he": "1.2.0"
+      }
+    },
     "nodemon": {
       "version": "2.0.8",
       "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.8.tgz",
@@ -2187,10 +2289,17 @@
       "dev": true
     },
     "normalize-url": {
-      "version": "4.5.1",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz",
-      "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==",
-      "dev": true
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz",
+      "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A=="
+    },
+    "nth-check": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.0.0.tgz",
+      "integrity": "sha512-i4sc/Kj8htBrAiH1viZ0TgU8Y5XqCaV/FziYK6TBczxmeKm3AEFWqqF3195yKudrarqy7Zu80Ra5dobFjn9X/Q==",
+      "requires": {
+        "boolbase": "^1.0.0"
+      }
     },
     "object-assign": {
       "version": "4.1.1",
@@ -2219,7 +2328,6 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
-      "dev": true,
       "requires": {
         "wrappy": "1"
       }
@@ -2239,10 +2347,9 @@
       }
     },
     "p-cancelable": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz",
-      "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==",
-      "dev": true
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz",
+      "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg=="
     },
     "package-json": {
       "version": "6.5.0",
@@ -2256,6 +2363,138 @@
         "semver": "^6.2.0"
       },
       "dependencies": {
+        "@sindresorhus/is": {
+          "version": "0.14.0",
+          "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",
+          "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==",
+          "dev": true
+        },
+        "@szmarczak/http-timer": {
+          "version": "1.1.2",
+          "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz",
+          "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==",
+          "dev": true,
+          "requires": {
+            "defer-to-connect": "^1.0.1"
+          }
+        },
+        "cacheable-request": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz",
+          "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==",
+          "dev": true,
+          "requires": {
+            "clone-response": "^1.0.2",
+            "get-stream": "^5.1.0",
+            "http-cache-semantics": "^4.0.0",
+            "keyv": "^3.0.0",
+            "lowercase-keys": "^2.0.0",
+            "normalize-url": "^4.1.0",
+            "responselike": "^1.0.2"
+          },
+          "dependencies": {
+            "get-stream": {
+              "version": "5.2.0",
+              "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+              "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
+              "dev": true,
+              "requires": {
+                "pump": "^3.0.0"
+              }
+            },
+            "lowercase-keys": {
+              "version": "2.0.0",
+              "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
+              "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==",
+              "dev": true
+            }
+          }
+        },
+        "decompress-response": {
+          "version": "3.3.0",
+          "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
+          "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
+          "dev": true,
+          "requires": {
+            "mimic-response": "^1.0.0"
+          }
+        },
+        "defer-to-connect": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz",
+          "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==",
+          "dev": true
+        },
+        "get-stream": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
+          "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+          "dev": true,
+          "requires": {
+            "pump": "^3.0.0"
+          }
+        },
+        "got": {
+          "version": "9.6.0",
+          "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz",
+          "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==",
+          "dev": true,
+          "requires": {
+            "@sindresorhus/is": "^0.14.0",
+            "@szmarczak/http-timer": "^1.1.2",
+            "cacheable-request": "^6.0.0",
+            "decompress-response": "^3.3.0",
+            "duplexer3": "^0.1.4",
+            "get-stream": "^4.1.0",
+            "lowercase-keys": "^1.0.1",
+            "mimic-response": "^1.0.1",
+            "p-cancelable": "^1.0.0",
+            "to-readable-stream": "^1.0.0",
+            "url-parse-lax": "^3.0.0"
+          }
+        },
+        "json-buffer": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz",
+          "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=",
+          "dev": true
+        },
+        "keyv": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz",
+          "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==",
+          "dev": true,
+          "requires": {
+            "json-buffer": "3.0.0"
+          }
+        },
+        "lowercase-keys": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
+          "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==",
+          "dev": true
+        },
+        "normalize-url": {
+          "version": "4.5.1",
+          "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz",
+          "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==",
+          "dev": true
+        },
+        "p-cancelable": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz",
+          "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==",
+          "dev": true
+        },
+        "responselike": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz",
+          "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=",
+          "dev": true,
+          "requires": {
+            "lowercase-keys": "^1.0.0"
+          }
+        },
         "semver": {
           "version": "6.3.0",
           "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@@ -2498,7 +2737,6 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
       "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-      "dev": true,
       "requires": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -2529,6 +2767,11 @@
       "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
       "dev": true
     },
+    "quick-lru": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
+      "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="
+    },
     "range-parser": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
@@ -2614,6 +2857,11 @@
       "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
       "dev": true
     },
+    "resolve-alpn": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.1.2.tgz",
+      "integrity": "sha512-8OyfzhAtA32LVUsJSke3auIyINcwdh5l3cvYKdKO0nvsYSKuiLfTM5i78PJswFPT8y6cPW+L1v6/hE95chcpDA=="
+    },
     "resolve-from": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
@@ -2621,12 +2869,11 @@
       "dev": true
     },
     "responselike": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz",
-      "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=",
-      "dev": true,
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
+      "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
       "requires": {
-        "lowercase-keys": "^1.0.0"
+        "lowercase-keys": "^2.0.0"
       }
     },
     "reusify": {
@@ -3394,8 +3641,7 @@
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
-      "dev": true
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
     },
     "write-file-atomic": {
       "version": "3.0.3",
diff --git a/backend/package.json b/backend/package.json
index 5311ef1..74e275f 100644
--- a/backend/package.json
+++ b/backend/package.json
@@ -5,12 +5,16 @@
   "main": "index.ts",
   "dependencies": {
     "express": "4.17.1",
+    "got": "11.8.2",
+    "he": "1.2.0",
+    "node-html-parser": "3.3.6",
     "slonik": "23.8.3",
     "socket.io": "4.1.2"
   },
   "devDependencies": {
     "@slonik/migrator": "0.8.3",
     "@types/express": "4.17.12",
+    "@types/he": "1.1.1",
     "@types/node": "15.12.5",
     "@types/socket.io": "3.0.2",
     "@typescript-eslint/eslint-plugin": "4.28.1",
diff --git a/backend/shared-types.ts b/backend/shared-types.ts
index 51f3281..f68b896 100644
--- a/backend/shared-types.ts
+++ b/backend/shared-types.ts
@@ -1,3 +1,8 @@
 export interface ValueRecord {
   value: number;
 }
+
+export interface Paper {
+  title: string;
+  url: string;
+}
diff --git a/frontend/components/Marquees/Marquee.module.css b/frontend/components/Marquees/Marquee.module.css
new file mode 100644
index 0000000..0fb8ece
--- /dev/null
+++ b/frontend/components/Marquees/Marquee.module.css
@@ -0,0 +1,38 @@
+.marqueeContainer {
+  height: 100vh;
+  overflow: hidden;
+  position: relative;
+  background: #fefefe;
+  color: #333;
+  z-index: 10;
+}
+
+.marquee {
+  position: absolute;
+  width: 100%;
+  height: 50px;
+  margin: 0;
+  line-height: 50px;
+  text-align: center;
+  transform: translateX(100%);
+  animation: scroll-left 10s linear infinite;
+  text-decoration: underline;
+}
+
+@keyframes scroll-left {
+  0% {
+      transform: translateX(100%);
+  }
+  100% {
+      transform: translateX(-100%);
+  }
+}
+
+@keyframes scroll-right {
+  0% {
+      transform: translateX(-100%);
+  }
+  100% {
+      transform: translateX(100%);
+  }
+}
diff --git a/frontend/components/Marquees/Marquee.tsx b/frontend/components/Marquees/Marquee.tsx
new file mode 100644
index 0000000..4cc41a2
--- /dev/null
+++ b/frontend/components/Marquees/Marquee.tsx
@@ -0,0 +1,20 @@
+import React, { ReactElement } from "react";
+import styles from "./Marquee.module.css";
+
+interface Props {
+  delayShorthand: string;
+  topShorthand: string;
+  url: string;
+  text: string;
+  direction: "left" | "right";
+}
+
+const Marquee = ({ delayShorthand, url, text, topShorthand, direction }: Props): ReactElement => (
+    <a className={styles.marquee} href={url} style={{
+        animationDelay: delayShorthand,
+        top: topShorthand,
+        animationDirection: direction === "left" ? "normal" : "reverse",
+    }}>{text}</a>
+);
+
+export default Marquee;
diff --git a/frontend/components/Marquees/Marquees.tsx b/frontend/components/Marquees/Marquees.tsx
new file mode 100644
index 0000000..04747fb
--- /dev/null
+++ b/frontend/components/Marquees/Marquees.tsx
@@ -0,0 +1,35 @@
+import React, { ReactElement } from "react";
+import styles from "./Marquee.module.css";
+import Marquee from "./Marquee";
+
+export interface MarqueeLink {
+  title: string;
+  url: string;
+}
+
+interface Props {
+    links: MarqueeLink[]
+}
+
+const getRandomDelayBetween = (start: number, end: number) => `${Math.random() * (end - start) + start}s`;
+const getTopFromIndex = (index: number, n: number) => `calc((100% - 50px) * (${index} / ${n - 1}))`;
+const getRandomAnimationName = () => Math.random() > 0.5 ? "right" : "left";
+
+const Marquees = ({ links }: Props): ReactElement => {
+    return (
+        <div className={styles.marqueeContainer}>
+            { links.map((link, idx) => (
+                <Marquee
+                    key={idx}
+                    url={link.url}
+                    text={link.title}
+                    delayShorthand={getRandomDelayBetween(0, 9)}
+                    topShorthand={getTopFromIndex(idx, links.length)}
+                    direction={getRandomAnimationName()}
+                />
+            ))}
+        </div>
+    );
+};
+
+export default Marquees;
diff --git a/frontend/pages/index.tsx b/frontend/pages/index.tsx
index 58b3b56..6193488 100644
--- a/frontend/pages/index.tsx
+++ b/frontend/pages/index.tsx
@@ -7,19 +7,9 @@ import styles from "../styles/Home.module.css";
 
 const socket = io(config.backendUrl);
 
-const mockData: MarqueeLink[] = [
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-    { title: "Foobar of Foos and Bars", url: "https://example.com" },
-];
-
 const Home = (): ReactElement => {
     const [value, setValue] = useState<number | undefined>(undefined);
+    const [papers, setPapers] = useState<MarqueeLink[]>([]);
 
     useEffect(() => {
         socket.on("connect", () => {
@@ -29,11 +19,20 @@ const Home = (): ReactElement => {
             console.log("disconnected, reason:", reason);
         });
         socket.on("update", (arg: number) => {
-            console.log("update:", arg);
             setValue(arg);
         });
     });
 
+    useEffect(() => {
+        async function fetchPapers() {
+            const resp = await fetch(`${config.backendUrl}/api/papers`);
+            const data = await resp.json();
+
+            setPapers(data);
+        }
+        fetchPapers();
+    }, []);
+
     const increment = () => {
         socket.emit("increment");
     };
@@ -57,7 +56,7 @@ const Home = (): ReactElement => {
                     <img src="/tomi_sq.png" alt="" />
                 </div>
             </button>
-            <Marquees links={mockData} />
+            <Marquees links={papers} />
         </div>
     );
 };
diff --git a/frontend/shared-types.ts b/frontend/shared-types.ts
index 51f3281..f68b896 100644
--- a/frontend/shared-types.ts
+++ b/frontend/shared-types.ts
@@ -1,3 +1,8 @@
 export interface ValueRecord {
   value: number;
 }
+
+export interface Paper {
+  title: string;
+  url: string;
+}
diff --git a/frontend/styles/Home.module.css b/frontend/styles/Home.module.css
index 6f43b54..814ebba 100644
--- a/frontend/styles/Home.module.css
+++ b/frontend/styles/Home.module.css
@@ -19,6 +19,7 @@
   top: 50%;
   margin-left: -40vw;
   margin-top: -40vw;
+  z-index: 20;
 }
 
 .btnInner {