[SAT-1957] Convert haste-server to Typescript

.eslintrc.js

@@ -0,0 +1,56 @@
+module.exports = {
+  env: {
+    node: true,
+  },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'airbnb-base',
+    'airbnb-typescript/base',
+    'plugin:import/errors',
+    'plugin:import/warnings',
+    'plugin:import/typescript',
+    'prettier',
+  ],
+  plugins: [
+    'import',
+    '@typescript-eslint'
+  ],
+  settings: {
+    'import/parsers': {
+      '@typescript-eslint/parser': ['.ts'],
+    },
+    'import/resolver': {
+      node: {
+        extensions: ['.js', '.ts'],
+        moduleDirectory: ['node_modules', 'src/'],
+      },
+      typescript: {
+        alwaysTryTypes: true,
+        project: '.',
+      },
+    },
+  },
+  overrides: [
+    {
+      env: {
+        jest: true,
+      },
+      files: ['**/__tests__/**/*.[jt]s', '**/?(*.)+(spec|test).[jt]s'],
+      extends: ['plugin:jest/recommended'],
+      rules: {
+        'import/no-extraneous-dependencies': [
+          'off',
+          { devDependencies: ['**/?(*.)+(spec|test).[jt]s'] },
+        ],
+        camelcase: ['off'],
+      },
+    },
+  ],
+  ignorePatterns: ['**/*.js', 'node_modules', 'dist'],
+  parserOptions: {
+    root: true,
+    tsconfigRootDir: __dirname,
+    project: ['./tsconfig.json'],
+  },
+}

.eslintrc.json

@@ -1,25 +0,0 @@
-{
-    "env": {
-        "es6": true,
-        "node": true
-    },
-    "extends": "eslint:recommended",
-    "rules": {
-        "indent": [
-            "error",
-            2
-        ],
-        "linebreak-style": [
-            "error",
-            "unix"
-        ],
-        "quotes": [
-            "error",
-            "single"
-        ],
-        "semi": [
-            "error",
-            "always"
-        ]
-    }
-}

.gitignore

@@ -5,3 +5,4 @@ node_modules
 data
 *.DS_Store
 docker-compose.override.yml
+dist

.prettierrc.json

@@ -0,0 +1,8 @@
+{
+  "tabWidth": 2,
+  "semi": false,
+  "trailingComma": "all",
+  "printWidth": 80,
+  "arrowParens": "avoid",
+  "singleQuote": true
+}

config.json

@@ -1,16 +1,10 @@
 {
-
   "host": "0.0.0.0",
   "port": 7777,
-
   "keyLength": 10,
-
   "maxLength": 400000,
-
   "staticMaxAge": 86400,
-
   "recompressStaticAssets": true,
-
   "logging": [
     {
       "level": "verbose",
@@ -18,11 +12,9 @@
       "colorize": true
     }
   ],
-
   "keyGenerator": {
     "type": "phonetic"
   },
-
   "rateLimits": {
     "categories": {
       "normal": {
@@ -31,13 +23,10 @@
       }
     }
   },
-
   "storage": {
     "type": "file"
   },
-
   "documents": {
     "about": "./about.md"
   }
-
 }

package.json

@@ -12,36 +12,66 @@
     "email": "john.crepezzi@gmail.com",
     "url": "http://seejohncode.com/"
   },
-  "main": "haste",
   "dependencies": {
+    "@google-cloud/datastore": "^6.6.2",
+    "aws-sdk": "^2.1142.0",
     "busboy": "0.2.4",
     "connect": "^3.7.0",
-    "connect-ratelimit": "0.0.7",
+    "connect-ratelimit": "^0.0.7",
     "connect-route": "0.1.5",
-    "pg": "^8.0.0",
+    "dotenv": "^16.0.1",
+    "express": "^4.18.1",
+    "memcached": "^2.2.2",
+    "mongodb": "^4.6.0",
+    "pg": "^8.7.3",
     "redis": "0.8.1",
     "redis-url": "0.1.0",
-    "st": "^2.0.0",
+    "rethinkdbdash": "^2.3.31",
+    "st": "^3.0.0",
     "uglify-js": "3.1.6",
     "winston": "^2.0.0"
   },
   "devDependencies": {
-    "mocha": "^8.1.3"
+    "@types/busboy": "^1.5.0",
+    "@types/express": "^4.17.13",
+    "@types/memcached": "^2.2.7",
+    "@types/node": "^17.0.35",
+    "@types/pg": "^8.6.5",
+    "@types/uglify-js": "^3.13.2",
+    "@typescript-eslint/eslint-plugin": "^5.26.0",
+    "@typescript-eslint/parser": "^5.26.0",
+    "concurrently": "^7.2.1",
+    "eslint": "^8.10.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-airbnb-typescript": "^17.0.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-import-resolver-typescript": "^2.7.1",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-jest": "^26.2.2",
+    "mocha": "^8.1.3",
+    "module-resolver": "^1.0.0",
+    "nodemon": "^2.0.16",
+    "npm-run-all": "^4.1.5",
+    "prettier": "^2.5.1",
+    "rimraf": "^3.0.2",
+    "ts-node": "^9.1.1",
+    "typescript": "^4.6.4"
   },
   "bundledDependencies": [],
+  "main": "haste",
   "bin": {
-    "haste-server": "./server.js"
+    "haste-server": ".dist/src/server.js"
   },
   "files": [
-    "server.js",
+    "src",
-    "lib",
     "static"
   ],
-  "directories": {
-    "lib": "./lib"
-  },
   "scripts": {
-    "start": "node server.js",
+    "test": "mocha --recursive",
-    "test": "mocha --recursive"
+    "build": "rimraf dist && tsc --project ./",
+    "start:dev": "nodemon src/server.ts",
+    "start:prod": "node dist/src/server.js",
+    "lint": "eslint src --fix",
+    "types:check": "tsc --noEmit --pretty"
   }
 }

src/app.ts

@@ -0,0 +1,175 @@
+import express, { Router, Express, Request } from 'express'
+import * as fs from 'fs'
+import * as winston from 'winston'
+import uglify from 'uglify-js'
+import connectSt from 'st'
+import connectRateLimit from 'connect-ratelimit'
+import getConfig from './lib/helpers/config'
+import addLogging from './lib/helpers/log'
+import build from './lib/document-handler/builder'
+import DocumentHandler from './lib/document-handler'
+import { Config } from './types/config'
+import {
+  getStaticDirectory,
+  getStaticItemDirectory,
+} from './lib/helpers/directory'
+
+class App {
+  public server: Express
+
+  public config: Config
+
+  documentHandler?: DocumentHandler
+
+  constructor() {
+    this.config = getConfig()
+    this.server = express()
+    this.setLogging()
+    this.setDocumentHandler()
+    this.compressStaticAssets()
+    this.sendDocumentsToStore()
+    this.middlewares()
+    this.setRateLimits()
+    this.apiCalls()
+    this.staticPages()
+  }
+
+  middlewares() {
+    this.server.use(express.json())
+  }
+
+  setLogging() {
+    if (this.config.logging) {
+      addLogging(this.config)
+    }
+  }
+
+  setDocumentHandler = async () => {
+    this.documentHandler = await build(this.config)
+  }
+
+  apiCalls() {
+    const router = Router()
+
+    // get raw documents - support getting with extension
+    router.get('/raw/:id', async (request, response) =>
+      this.documentHandler?.handleRawGet(request, response),
+    )
+
+    router.head('/raw/:id', (request, response) =>
+      this.documentHandler?.handleRawGet(request, response),
+    )
+
+    // // add documents
+    router.post('/documents', (request, response) =>
+      this.documentHandler?.handlePost(request, response),
+    )
+
+    // get documents
+    router.get('/documents/:id', (request, response) =>
+      this.documentHandler?.handleGet(request, response),
+    )
+
+    router.head('/documents/:id', (request, response) =>
+      this.documentHandler?.handleGet(request, response),
+    )
+
+    this.server.use(router)
+  }
+
+  setRateLimits() {
+    if (this.config.rateLimits) {
+      this.config.rateLimits.end = true
+      this.server.use(connectRateLimit(this.config.rateLimits))
+    }
+  }
+
+  compressStaticAssets() {
+    // Compress the static javascript assets
+    if (this.config.recompressStaticAssets) {
+      const list = fs.readdirSync(getStaticDirectory(__dirname))
+      for (let j = 0; j < list.length; j += 1) {
+        const item = list[j]
+        if (
+          item.indexOf('.js') === item.length - 3 &&
+          item.indexOf('.min.js') === -1
+        ) {
+          const dest = `${item.substring(
+            0,
+            item.length - 3,
+          )}.min${item.substring(item.length - 3)}`
+          const origCode = fs.readFileSync(
+            getStaticItemDirectory(__dirname, item),
+            'utf8',
+          )
+
+          fs.writeFileSync(
+            getStaticItemDirectory(__dirname, dest),
+            uglify.minify(origCode).code,
+            'utf8',
+          )
+          winston.info(`compressed ${item} into ${dest}`)
+        }
+      }
+    }
+  }
+
+  sendDocumentsToStore() {
+    // Send the static documents into the preferred store, skipping expirations
+    let documentPath
+    let data
+
+    Object.keys(this.config.documents).forEach(name => {
+      documentPath = this.config.documents[name]
+      data = fs.readFileSync(documentPath, 'utf8')
+      winston.info('loading static document', { name, path: documentPath })
+
+      if (data) {
+        this.documentHandler?.store.set(
+          name,
+          data,
+          cb => {
+            winston.debug('loaded static document', { success: cb })
+          },
+          true,
+        )
+      } else {
+        winston.warn('failed to load static document', {
+          name,
+          path: documentPath,
+        })
+      }
+    })
+  }
+
+  staticPages() {
+
+    // Otherwise, try to match static files
+    this.server.use(
+      connectSt({
+        path: getStaticDirectory(__dirname),
+        content: { maxAge: this.config.staticMaxAge },
+        passthrough: true,
+        index: false,
+      }),
+    )
+
+    // Then we can loop back - and everything else should be a token,
+    // so route it back to /
+    this.server.get('/:id', (request: Request, response, next) => {
+      request.sturl = '/'
+      next()
+    })
+
+    // And match index
+    this.server.use(
+      connectSt({
+        path: getStaticDirectory(__dirname),
+        content: { maxAge: this.config.staticMaxAge },
+        index: 'index.html',
+      }),
+    )
+  }
+}
+
+export default App

src/global.d.ts

@@ -0,0 +1,74 @@
+declare module 'rethinkdbdash' {
+  type Result = {
+    data: string
+  }
+
+  type Callback = (error: unknown, result?: Result) => void
+
+  interface RethinkRun {
+    run(callback: Callback)
+  }
+
+  type RethinkInsertObject = {
+    id: string
+    data: string
+  }
+
+  interface RethinkFunctions {
+    insert(object: RethinkInsertObject): RethinkRun
+    get(x: string): RethinkRun
+  }
+
+  export interface RethinkClient {
+    table(s: string): RethinkFunctions
+  }
+
+  // function rethink<T>(obj: T[]): RethinkArray<T>
+  function rethink<T>(obj: T): RethinkClient<T>
+
+  export = rethink
+}
+
+// export {}
+
+// declare module 'connect-ratelimit' {
+//   export = connectRateLimit
+// }
+
+// declare namespace Express {
+//   export interface Request {
+//     sturl?: string
+//   }
+// }
+
+declare module 'connect-ratelimit' {
+  function connectRateLimit(
+    as: RateLimits,
+  ): (
+    req: express.Request,
+    res: express.Response,
+    next: express.NextFunction,
+  ) => void
+
+  export = connectRateLimit
+}
+
+declare namespace Express {
+  export interface Request {
+    sturl: string
+  }
+}
+
+declare module 'st' {
+
+  type ConnectSt = {
+    path: string
+    content: { maxAge : number }
+    passthrough? : boolean
+    index: boolean | string
+  }
+
+  function connectSt(st: ConnectSt): Middleware
+
+  export = connectSt
+}

src/lib/document-handler/builder.ts

@@ -0,0 +1,21 @@
+import type { Config } from '../../types/config'
+import buildGenerator from '../key-generators/builder'
+import buildStore from '../document-stores/builder'
+import DocumentHandler from "./index"
+
+const build = async (config: Config) => {
+  const storage = await buildStore(config)
+  const keyGenerator = await buildGenerator(config)
+
+  const documentHandler = new DocumentHandler({
+    store: storage,
+    config,
+    maxLength: config.maxLength,
+    keyLength: config.keyLength,
+    keyGenerator,
+  })
+
+  return documentHandler
+}
+
+export default build

src/lib/document-handler/index.ts

@@ -0,0 +1,177 @@
+import { Request, Response } from 'express'
+import * as winston from 'winston'
+import Busboy from 'busboy'
+import type { Config } from '../../types/config'
+import type { Store } from '../../types/store'
+import type { KeyGenerator } from '../../types/key-generator'
+import type { Document } from '../../types/document'
+
+const defaultKeyLength = 10
+
+class DocumentHandler {
+  keyLength: number
+
+  maxLength: number
+
+  public store: Store
+
+  keyGenerator: KeyGenerator
+
+  config: Config
+
+  constructor(options: Document) {
+    this.keyLength = options.keyLength || defaultKeyLength
+    this.maxLength = options.maxLength // none by default
+    this.store = options.store
+    this.config = options.config
+    this.keyGenerator = options.keyGenerator
+  }
+
+  public handleGet(request: Request, response: Response) {
+    const key = request.params.id.split('.')[0]
+    const skipExpire = !!this.config.documents[key]
+
+    this.store.get(
+      key,
+      ret => {
+        if (ret) {
+          winston.verbose('retrieved document', { key })
+          response.writeHead(200, { 'content-type': 'application/json' })
+          if (request.method === 'HEAD') {
+            response.end()
+          } else {
+            response.end(JSON.stringify({ data: ret, key }))
+          }
+        } else {
+          winston.warn('document not found', { key })
+          response.writeHead(404, { 'content-type': 'application/json' })
+          if (request.method === 'HEAD') {
+            response.end()
+          } else {
+            response.end(JSON.stringify({ message: 'Document not found.' }))
+          }
+        }
+      },
+      skipExpire,
+    )
+  }
+
+  public handlePost(request: Request, response: Response) {
+    // const this = this
+    let buffer = ''
+    let cancelled = false
+
+    // What to do when done
+    const onSuccess = () => {
+      // Check length
+      if (this.maxLength && buffer.length > this.maxLength) {
+        cancelled = true
+        winston.warn('document >maxLength', { maxLength: this.maxLength })
+        response.writeHead(400, { 'content-type': 'application/json' })
+        response.end(
+          JSON.stringify({ message: 'Document exceeds maximum length.' }),
+        )
+        return
+      }
+      // And then save if we should
+      this.chooseKey(key => {
+        this.store.set(key, buffer, res => {
+          if (res) {
+            winston.verbose('added document', { key })
+            response.writeHead(200, { 'content-type': 'application/json' })
+            response.end(JSON.stringify({ key }))
+          } else {
+            winston.verbose('error adding document')
+            response.writeHead(500, { 'content-type': 'application/json' })
+            response.end(JSON.stringify({ message: 'Error adding document.' }))
+          }
+        })
+      })
+    }
+
+    // If we should, parse a form to grab the data
+    const ct = request.headers['content-type']
+    if (ct && ct.split(';')[0] === 'multipart/form-data') {
+      const busboy = Busboy({ headers: request.headers })
+      busboy.on('field', (fieldname, val) => {
+        if (fieldname === 'data') {
+          buffer = val
+        }
+      })
+      busboy.on('finish', () => {
+        onSuccess()
+      })
+      request.pipe(busboy)
+      // Otherwise, use our own and just grab flat data from POST body
+    } else {
+      request.on('data', data => {
+        buffer += data.toString()
+      })
+      request.on('end', () => {
+        if (cancelled) {
+          return
+        }
+        onSuccess()
+      })
+      request.on('error', error => {
+        winston.error(`connection error: ${error.message}`)
+        response.writeHead(500, { 'content-type': 'application/json' })
+        response.end(JSON.stringify({ message: 'Connection error.' }))
+        cancelled = true
+      })
+    }
+  }
+
+  public handleRawGet(request: Request, response: Response) {
+    const key = request.params.id.split('.')[0]
+    const skipExpire = !!this.config.documents[key]
+
+    this.store.get(
+      key,
+      ret => {
+        if (ret) {
+          winston.verbose('retrieved raw document', { key })
+          response.writeHead(200, {
+            'content-type': 'text/plain; charset=UTF-8',
+          })
+          if (request.method === 'HEAD') {
+            response.end()
+          } else {
+            response.end(ret)
+          }
+        } else {
+          winston.warn('raw document not found', { key })
+          response.writeHead(404, { 'content-type': 'application/json' })
+          if (request.method === 'HEAD') {
+            response.end()
+          } else {
+            response.end(JSON.stringify({ message: 'Document not found.' }))
+          }
+        }
+      },
+      skipExpire,
+    )
+  }
+
+  chooseKey = (callback: { (key: string): void }) => {
+    const key = this.acceptableKey()
+
+    if (!key) return
+
+    this.store.get(
+      key,
+      (ret: string | boolean) => {
+        if (ret) {
+          this.chooseKey(callback)
+        } else {
+          callback(key)
+        }
+      },
+      true,
+    ) // Don't bump expirations when key searching
+  }
+
+  acceptableKey = () => this.keyGenerator.createKey?.(this.keyLength)
+}
+
+export default DocumentHandler

src/lib/document-stores/amazon-s3.ts

@@ -0,0 +1,80 @@
+import * as winston from 'winston'
+import AWS from 'aws-sdk'
+import type { Callback, Store } from '../../types/store'
+import type { AmazonStoreConfig } from '../../types/config'
+
+class AmazonS3DocumentStore implements Store {
+  bucket: string | undefined
+
+  client: AWS.S3
+
+  type: string
+
+  expire?: number | undefined
+
+  constructor(options: AmazonStoreConfig) {
+    this.expire = options.expire
+    this.bucket = options.bucket
+    this.type = options.type
+    this.client = new AWS.S3({ region: options.region })
+  }
+
+  get = (
+    key: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    if (!this.bucket) {
+      callback(false)
+      return
+    }
+
+    const req = {
+      Bucket: this.bucket,
+      Key: key,
+    }
+
+    this.client.getObject(req, (err, data) => {
+      if (err || !data.Body) {
+        callback(false)
+      } else {
+        callback(data.Body.toString('utf-8'))
+        if (this.expire && !skipExpire) {
+          winston.warn('amazon s3 store cannot set expirations on keys')
+        }
+      }
+    })
+  }
+
+  set = (
+    key: string,
+    data: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    if (!this.bucket) {
+      callback(false)
+      return
+    }
+
+    const req = {
+      Bucket: this.bucket,
+      Key: key,
+      Body: data as AWS.S3.PutObjectOutput,
+      ContentType: 'text/plain',
+    }
+
+    this.client.putObject(req, err => {
+      if (err) {
+        callback(false)
+      } else {
+        callback(true)
+        if (this.expire && !skipExpire) {
+          winston.warn('amazon s3 store cannot set expirations on keys')
+        }
+      }
+    })
+  }
+}
+
+export default AmazonS3DocumentStore

src/lib/document-stores/builder.ts

@@ -0,0 +1,22 @@
+import type { Config } from '../../types/config'
+import type { Store } from '../../types/store'
+
+const build = async (config: Config): Promise<Store> => {
+
+  if (process.env.REDISTOGO_URL && config.storage.type === 'redis') {
+    // const redisClient = require("redis-url").connect(process.env.REDISTOGO_URL);
+    // Store = require("./lib/document-stores/redis");
+    // preferredStore = new Store(config.storage, redisClient);
+
+    const DocumentStore = (await import(`../document-stores/${config.storage.type}`)).default
+
+    return new DocumentStore(config.storage)
+  } 
+    const DocumentStore = (await import(`../document-stores/${config.storage.type}`)).default
+
+    return new DocumentStore(config.storage)
+  
+}
+
+export default build
+

src/lib/document-stores/file.ts

@@ -0,0 +1,79 @@
+import * as winston from 'winston'
+import * as fs from 'fs'
+import * as crypto from 'crypto'
+
+import type { Callback, Store } from '../../types/store'
+import type { FileStoreConfig } from '../../types/config'
+
+// Generate md5 of a string
+const md5 = (str: string) => {
+  const md5sum = crypto.createHash('md5')
+  md5sum.update(str)
+  return md5sum.digest('hex')
+}
+
+// For storing in files
+// options[type] = file
+// options[path] - Where to store
+
+class FileDocumentStore implements Store {
+  type: string
+
+  expire?: number | undefined
+
+  basePath: string
+
+  constructor(options: FileStoreConfig) {
+    this.basePath = options.path || './data'
+    this.expire = options.expire
+    this.type = options.type
+  }
+
+  // Get data from a file from key
+  get = (
+    key: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    const fn = `${this.basePath}/${md5(key)}`
+    fs.readFile(fn, 'utf8', (err, data) => {
+      if (err) {
+        callback(false)
+      } else {
+        callback(data)
+        if (this.expire && !skipExpire) {
+          winston.warn('file store cannot set expirations on keys')
+        }
+      }
+    })
+  }
+
+  // Save data in a file, key as md5 - since we don't know what we could
+  // be passed here
+  set = (
+    key: string,
+    data: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    try {
+      fs.mkdir(this.basePath, '700', () => {
+        const fn = `${this.basePath}/${md5(key)}`
+        fs.writeFile(fn, data, 'utf8', err => {
+          if (err) {
+            callback(false)
+          } else {
+            callback(true)
+            if (this.expire && !skipExpire) {
+              winston.warn('file store cannot set expirations on keys')
+            }
+          }
+        })
+      })
+    } catch (err) {
+      callback(false)
+    }
+  }
+}
+
+export default FileDocumentStore

src/lib/document-stores/google-datastore.ts

@@ -0,0 +1,114 @@
+import { Datastore, PathType } from '@google-cloud/datastore'
+import * as winston from 'winston'
+
+import type { Callback, Store } from '../../types/store'
+import type { GoogleStoreConfig } from '../../types/config'
+
+class GoogleDatastoreDocumentStore implements Store {
+  kind: string
+
+  expire?: number
+
+  datastore: Datastore
+
+  type: string
+
+  // Create a new store with options
+  constructor(options: GoogleStoreConfig) {
+    this.kind = 'Haste'
+    this.expire = options.expire
+    this.type = options.type
+    this.datastore = new Datastore()
+  }
+
+  // Save file in a key
+  set = (
+    key: PathType,
+    data: string,
+    callback: Callback,
+    skipExpire?: boolean,
+  ) => {
+    const expireTime =
+      skipExpire || this.expire === undefined
+        ? null
+        : new Date(Date.now() + this.expire * 1000)
+
+    const taskKey = this.datastore.key([this.kind, key])
+    const task = {
+      key: taskKey,
+      data: [
+        {
+          name: 'value',
+          value: data,
+          excludeFromIndexes: true,
+        },
+        {
+          name: 'expiration',
+          value: expireTime,
+        },
+      ],
+    }
+
+    this.datastore
+      .insert(task)
+      .then(() => {
+        callback(true)
+      })
+      .catch(() => {
+        callback(false)
+      })
+  }
+
+  // Get a file from a key
+  get = (key: PathType, callback: Callback, skipExpire?: boolean): void => {
+    const taskKey = this.datastore.key([this.kind, key])
+
+    this.datastore
+      .get(taskKey)
+      .then(entity => {
+        if (skipExpire || entity[0].expiration == null) {
+          callback(entity[0].value)
+        } else if (entity[0].expiration < new Date()) {
+          winston.info('document expired', {
+            key,
+            expiration: entity[0].expiration,
+            check: new Date(),
+          })
+          callback(false)
+        } else {
+          // update expiry
+          const task = {
+            key: taskKey,
+            data: [
+              {
+                name: 'value',
+                value: entity[0].value,
+                excludeFromIndexes: true,
+              },
+              {
+                name: 'expiration',
+                value: new Date(
+                  Date.now() + (this.expire ? this.expire * 1000 : 0),
+                ),
+              },
+            ],
+          }
+          this.datastore
+            .update(task)
+            .then(() => {})
+            .catch(err => {
+              winston.error('failed to update expiration', { error: err })
+            })
+          callback(entity[0].value)
+        }
+      })
+      .catch(err => {
+        winston.error('Error retrieving value from Google Datastore', {
+          error: err,
+        })
+        callback(false)
+      })
+  }
+}
+
+export default GoogleDatastoreDocumentStore

src/lib/document-stores/memcached.ts

@@ -0,0 +1,75 @@
+import * as winston from 'winston'
+import Memcached = require('memcached')
+
+import type { Callback, Store } from '../../types/store'
+import type { MemcachedStoreConfig } from '../../types/config'
+
+class MemcachedDocumentStore implements Store {
+  expire: number | undefined
+
+  client?: Memcached
+
+  type: string
+
+  // Create a new store with options
+  constructor(options: MemcachedStoreConfig) {
+    this.expire = options.expire
+    this.type = options.type
+    const host = options.host || '127.0.0.1'
+    const port = options.port || 11211
+    const url = `${host}:${port}`
+    this.connect(url)
+  }
+
+  // Create a connection
+  connect = (url: string) => {
+    this.client = new Memcached(url)
+
+    winston.info(`connecting to memcached on ${url}`)
+
+    this.client.on('failure', (error: Memcached.IssueData) => {
+      winston.info('error connecting to memcached', { error })
+    })
+  }
+
+  // Get a file from a key
+  get = (
+    key: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    this.client?.get(key, (error, data: string) => {
+      const value = error ? false : data
+
+      callback(value as string)
+
+      // Update the key so that the expiration is pushed forward
+      if (value && !skipExpire) {
+        this.set(
+          key,
+          data,
+          updateSucceeded => {
+            if (!updateSucceeded) {
+              winston.error('failed to update expiration on GET', { key })
+            }
+          },
+          skipExpire,
+        )
+      }
+    })
+  }
+
+  // Save file in a key
+  set = (
+    key: string,
+    data: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    this.client?.set(key, data, skipExpire ? 0 : this.expire || 0, error => {
+      callback(!error)
+    })
+  }
+}
+
+export default MemcachedDocumentStore

src/lib/document-stores/mongo.ts

@@ -0,0 +1,128 @@
+import * as winston from 'winston'
+import { MongoClient } from 'mongodb'
+
+import type { Callback, Store } from '../../types/store'
+import type { MongoStoreConfig } from '../../types/config'
+
+type ConnectCallback = (error?: Error, db?: MongoClient) => void
+
+class MongoDocumentStore implements Store {
+  type: string
+
+  expire?: number | undefined
+
+  connectionUrl: string
+
+  constructor(options: MongoStoreConfig) {
+    this.expire = options.expire
+    this.type = options.type
+    this.connectionUrl = process.env.DATABASE_URl || options.connectionUrl
+  }
+
+  safeConnect = (callback: ConnectCallback) => {
+    MongoClient.connect(this.connectionUrl, (err, client) => {
+      if (err) {
+        winston.error('error connecting to mongodb', { error: err })
+        callback(err)
+      } else {
+        callback(undefined, client)
+      }
+    })
+  }
+
+  get = (
+    key: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    const now = Math.floor(new Date().getTime() / 1000)
+
+    this.safeConnect((err, client) => {
+      if (err) return callback(false)
+
+      return client
+        ?.db()
+        .collection('entries')
+        .findOne(
+          {
+            entry_id: key,
+            $or: [{ expiration: -1 }, { expiration: { $gt: now } }],
+          },
+          (error?: Error, entry?) => {
+            if (error) {
+              winston.error('error persisting value to mongodb', { error })
+              return callback(false)
+            }
+
+            callback(entry === null ? false : entry?.value)
+
+            if (
+              entry !== null &&
+              entry?.expiration !== -1 &&
+              this.expire &&
+              !skipExpire
+            ) {
+              return client
+                .db()
+                .collection('entries')
+                .update(
+                  {
+                    entry_id: key,
+                  },
+                  {
+                    $set: {
+                      expiration: this.expire + now,
+                    },
+                  },
+                  {},
+                  () => {},
+                )
+            }
+
+            return true
+          },
+        )
+    })
+  }
+
+  set = (
+    key: string,
+    data: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    const now = Math.floor(new Date().getTime() / 1000)
+
+    this.safeConnect((err, client) => {
+      if (err) return callback(false)
+
+      return client
+        ?.db()
+        .collection('entries')
+        .update(
+          {
+            entry_id: key,
+            $or: [{ expiration: -1 }, { expiration: { $gt: now } }],
+          },
+          {
+            entry_id: key,
+            value: data,
+            expiration: this.expire && !skipExpire ? this.expire + now : -1,
+          },
+          {
+            upsert: true,
+          },
+          (error?: Error) => {
+            if (error) {
+              winston.error('error persisting value to mongodb', { error })
+              return callback(false)
+            }
+
+            return callback(true)
+          },
+        )
+    })
+  }
+}
+
+export default MongoDocumentStore

src/lib/document-stores/postgres.ts

@@ -0,0 +1,112 @@
+import * as winston from 'winston'
+import { Pool, PoolClient } from 'pg'
+
+import type { Callback, Store } from '../../types/store'
+import type { PostgresStoreConfig } from '../../types/config'
+
+type ConnectCallback = (
+  error?: Error,
+  client?: PoolClient,
+  done?: () => void,
+) => void
+
+// A postgres document store
+class PostgresDocumentStore implements Store {
+  type: string
+
+  expireJS?: number
+
+  pool: Pool
+
+  constructor(options: PostgresStoreConfig) {
+    this.expireJS = options.expire
+    this.type = options.type
+
+    const connectionString = process.env.DATABASE_URL || options.connectionUrl
+    this.pool = new Pool({ connectionString })
+  }
+
+  // A connection wrapper
+  safeConnect = (callback: ConnectCallback) => {
+    this.pool.connect((error: Error, client: PoolClient, done: () => void) => {
+      if (error) {
+        winston.error('error connecting to postgres', { error })
+        callback(error)
+      } else {
+        callback(undefined, client, done)
+      }
+    })
+  }
+
+  // Get a given key's data
+  get = (
+    key: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    const now = Math.floor(new Date().getTime() / 1000)
+    this.safeConnect((err, client, done): void => {
+      if (err) {
+        return callback(false)
+      }
+
+      return client?.query(
+        'SELECT id,value,expiration from entries where KEY = $1 and (expiration IS NULL or expiration > $2)',
+        [key, now],
+        (error: Error, result) => {
+          if (error) {
+            winston.error('error retrieving value from postgres', {
+              error,
+            })
+            return callback(false)
+          }
+          callback(result.rows.length ? result.rows[0].value : false)
+          if (result.rows.length && this.expireJS && !skipExpire) {
+            return client.query(
+              'UPDATE entries SET expiration = $1 WHERE ID = $2',
+              [this.expireJS + now, result.rows[0].id],
+              (currentErr: Error) => {
+                if (!currentErr) {
+                  return done?.()
+                }
+
+                return callback(false)
+              },
+            )
+          }
+
+          return done?.()
+        },
+      )
+    })
+  }
+
+  // Set a given key
+  set = (
+    key: string,
+    data: string,
+    callback: Callback,
+    skipExpire?: boolean | undefined,
+  ): void => {
+    const now = Math.floor(new Date().getTime() / 1000)
+    this.safeConnect((err, client, done) => {
+      if (err) {
+        return callback(false)
+      }
+      return client?.query(
+        'INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)',
+        [key, data, this.expireJS && !skipExpire ? this.expireJS + now : null],
+        (error: Error) => {
+          if (error) {
+            winston.error('error persisting value to postgres', { error })
+            return callback(false)
+          }
+          callback(true)
+          return done?.()
+        },
+      )
+    })
+  }
+}
+
+export default PostgresDocumentStore

src/lib/document-stores/rethinkdb.ts

@@ -0,0 +1,58 @@
+import * as winston from 'winston'
+import * as crypto from 'crypto'
+
+import rethink, { RethinkClient } from 'rethinkdbdash'
+
+import type { RethinkDbStoreConfig } from '../../types/config'
+import type { Callback } from '../../types/store'
+
+const md5 = (str: string) => {
+  const md5sum = crypto.createHash('md5')
+  md5sum.update(str)
+  return md5sum.digest('hex')
+}
+
+class RethinkDBStore {
+  client: RethinkClient
+
+  constructor(options: RethinkDbStoreConfig) {
+    this.client = rethink({
+      silent: true,
+      host: options.host || '127.0.0.1',
+      port: options.port || 28015,
+      db: options.db || 'haste',
+      user: options.user || 'admin',
+      password: options.password || '',
+    })
+  }
+
+  set = (key: string, data: string, callback: Callback): void => {
+    this.client
+      .table('uploads')
+      .insert({ id: md5(key), data })
+      .run(error => {
+        if (error) {
+          callback(false)
+          winston.error('failed to insert to table', error)
+          return
+        }
+        callback(true)
+      })
+  }
+
+  get = (key: string, callback: Callback): void => {
+    this.client
+      .table('uploads')
+      .get(md5(key))
+      .run((error, result) => {
+        if (error || !result) {
+          callback(false)
+          if (error) winston.error('failed to insert to table', error)
+          return
+        }
+        callback(result.data)
+      })
+  }
+}
+
+export default RethinkDBStore

src/lib/helpers/config.ts

@@ -0,0 +1,21 @@
+import * as fs from 'fs'
+import { Config } from '../../types/config'
+
+const getConfig = (): Config => {
+  const configPath = process.argv.length <= 2 ? 'config.json' : process.argv[2]
+  const config = JSON.parse(fs.readFileSync(configPath, 'utf8'))
+
+  config.port = (process.env.PORT || config.port || 7777) as number
+  config.host = process.env.HOST || config.host || 'localhost'
+
+  if (!config.storage) {
+    config.storage = { type: 'file' }
+  }
+  if (!config.storage.type) {
+    config.storage.type = 'file'
+  }
+
+  return config
+}
+
+export default getConfig

src/lib/helpers/directory.ts

@@ -0,0 +1,7 @@
+import * as path from 'path'
+
+export const getStaticDirectory = (baseDirectory: string) =>
+  path.join(baseDirectory, '..', 'static')
+
+export const getStaticItemDirectory = (baseDirectory: string, item: string) =>
+  path.join(baseDirectory, '..', 'static', item)

src/lib/helpers/log.ts

@@ -0,0 +1,23 @@
+import * as winston from 'winston'
+import type { Config } from '../../types/config'
+
+const addLogging = (config: Config) => {
+  try {
+    winston.remove(winston.transports.Console)
+  } catch (e) {
+    /* was not present */
+  }
+
+  let detail
+  let type
+
+  for (let i = 0; i < config.logging.length; i += 1) {
+    detail = config.logging[i]
+    type = detail.type
+    const transport = winston.transports[type]
+
+    winston.add(transport, detail)
+  }
+}
+
+export default addLogging

src/lib/key-generators/builder.ts

@@ -0,0 +1,13 @@
+import type { KeyGenerator } from '../../types/key-generator'
+import type { Config } from '../../types/config'
+
+const build = async (config: Config): Promise<KeyGenerator> => {
+  const pwOptions = config.keyGenerator
+  pwOptions.type = pwOptions.type || 'random'
+  const Generator = (await import(`../key-generators/${pwOptions.type}`)).default
+  const keyGenerator = new Generator(pwOptions)
+
+  return keyGenerator
+}
+
+export default build

src/lib/key-generators/dictionary.ts

@@ -0,0 +1,41 @@
+import * as fs from 'fs'
+import type { KeyGeneratorConfig } from '../../types/config'
+import type { KeyGenerator } from '../../types/key-generator'
+
+class DictionaryGenerator implements KeyGenerator {
+  type: string
+
+  dictionary: string[]
+
+  constructor(options: KeyGeneratorConfig, readyCallback: () => void) {
+    // Check options format
+    if (!options) throw Error('No options passed to generator')
+    if (!options.path) throw Error('No dictionary path specified in options')
+
+    this.dictionary = []
+    this.type = options.type
+
+    // Load dictionary
+    fs.readFile(options.path, 'utf8', (err, data) => {
+      if (err) throw err
+
+      this.dictionary = data.split(/[\n\r]+/)
+
+      if (readyCallback) readyCallback()
+    })
+  }
+
+  // Generates a dictionary-based key, of keyLength words
+  createKey(keyLength: number): string {
+    let text = ''
+
+    for (let i = 0; i < keyLength; i += 1) {
+      const index = Math.floor(Math.random() * this.dictionary.length)
+      text += this.dictionary[index]
+    }
+
+    return text
+  }
+}
+
+export default DictionaryGenerator

src/lib/key-generators/phonetic.ts

@@ -0,0 +1,34 @@
+// Draws inspiration from pwgen and http://tools.arantius.com/password
+
+import type { KeyGeneratorConfig } from '../../types/config'
+import type { KeyGenerator } from '../../types/key-generator'
+
+const randOf = (collection: string) => () =>
+  collection[Math.floor(Math.random() * collection.length)]
+
+// Helper methods to get an random vowel or consonant
+const randVowel = randOf('aeiou')
+const randConsonant = randOf('bcdfghjklmnpqrstvwxyz')
+
+class PhoneticKeyGenerator implements KeyGenerator {
+  type: string
+
+  constructor(options: KeyGeneratorConfig) {
+    this.type = options.type
+  }
+
+  // Generate a phonetic key of alternating consonant & vowel
+  // eslint-disable-next-line class-methods-use-this
+  createKey(keyLength: number) {
+    let text = ''
+    const start = Math.round(Math.random())
+
+    for (let i = 0; i < keyLength; i += 1) {
+      text += i % 2 === start ? randConsonant() : randVowel()
+    }
+
+    return text
+  }
+}
+
+export default PhoneticKeyGenerator

src/lib/key-generators/random.ts

@@ -0,0 +1,30 @@
+import type { KeyGeneratorConfig } from '../../types/config'
+import type { KeyGenerator } from '../../types/key-generator'
+
+class RandomKeyGenerator implements KeyGenerator {
+  type: string
+
+  keyspace: string
+
+  // Initialize a new generator with the given keySpace
+  constructor(options: KeyGeneratorConfig) {
+    this.keyspace =
+      options.keyspace ||
+      'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
+    this.type = options.type
+  }
+
+  // Generate a key of the given length
+  createKey(keyLength: number): string {
+    let text = ''
+
+    for (let i = 0; i < keyLength; i += 1) {
+      const index = Math.floor(Math.random() * this.keyspace.length)
+      text += this.keyspace.charAt(index)
+    }
+
+    return text
+  }
+}
+
+export default RandomKeyGenerator

src/server.ts

@@ -0,0 +1,8 @@
+import winston = require('winston')
+import App from './app'
+
+const { server, config } = new App()
+
+server.listen(config.port, config.host, () => {
+  winston.info(`listening on ${config.host}:${config.port}`)
+})

src/types/config.ts

@@ -0,0 +1,68 @@
+import { Logging } from './log'
+import { RateLimits } from './rate-limits'
+
+export interface Config {
+  host: string
+  port: number
+  keyLength: number
+  maxLength: number
+  staticMaxAge: number
+  recompressStaticAssets: boolean
+  logging: Logging[]
+  keyGenerator: KeyGeneratorConfig
+  rateLimits: RateLimits
+  storage: StoreConfig
+  documents: Record<string, string>
+}
+
+export type BaseStoreConfig = {
+  type: string
+  expire?: number
+}
+
+export interface MongoStoreConfig extends BaseStoreConfig {
+  connectionUrl: string
+}
+
+export interface MemcachedStoreConfig extends BaseStoreConfig {
+  host: string
+  port: number
+}
+
+export interface FileStoreConfig extends BaseStoreConfig {
+  path: string
+}
+
+export interface AmazonStoreConfig extends BaseStoreConfig {
+  bucket: string
+  region: string
+}
+
+export interface PostgresStoreConfig extends BaseStoreConfig {
+  connectionUrl: string
+}
+
+export interface RethinkDbStoreConfig extends BaseStoreConfig {
+  host: string
+  port: string
+  db: string
+  user: string
+  password: string
+}
+
+export type GoogleStoreConfig = BaseStoreConfig
+
+export type StoreConfig =
+  | MongoStoreConfig
+  | MemcachedStoreConfig
+  | GoogleStoreConfig
+  | AmazonStoreConfig
+  | FileStoreConfig
+  | MongoStoreConfig
+
+export interface KeyGeneratorConfig {
+  type: string
+  keyspace?: string
+  path?: string
+}
+

src/types/document.ts

@@ -0,0 +1,15 @@
+import type { Config } from './config'
+import type { KeyGenerator } from './key-generator'
+import type { Store } from './store'
+
+export type Document = {
+  store: Store
+  config: Config
+  maxLength: number
+  keyLength: number
+  keyGenerator: KeyGenerator
+}
+
+export interface Documents {
+  about: string
+}

src/types/key-generator.ts

@@ -0,0 +1,4 @@
+export interface KeyGenerator {
+  type: string
+  createKey?: (a: number) => string
+}

src/types/log.ts

@@ -0,0 +1,12 @@
+export interface Logging {
+    level: string
+    type:
+      | 'File'
+      | 'Console'
+      | 'Loggly'
+      | 'DailyRotateFile'
+      | 'Http'
+      | 'Memory'
+      | 'Webhook'
+  }
+  

src/types/rate-limits.ts

@@ -0,0 +1,13 @@
+export interface Normal {
+  totalRequests: number
+  every: number
+}
+
+export interface Categories {
+  normal: Normal
+}
+
+export interface RateLimits {
+  end?: boolean
+  categories: Categories
+}

src/types/request.ts

@@ -0,0 +1,5 @@
+import DocumentHandler from '../lib/document-handler'
+
+export type RequestParams = {
+  documentHandler: DocumentHandler
+}

src/types/store.ts

@@ -0,0 +1,13 @@
+export type Callback = (arg0: boolean | string) => void
+
+export interface Store {
+  type: string
+  expire?: number
+  get: (key: string, callback: Callback, skipExpire?: boolean) => void
+  set: (
+    key: string,
+    data: string,
+    callback: Callback,
+    skipExpire?: boolean,
+  ) => void
+}

tsconfig.json

@@ -0,0 +1,39 @@
+{
+  "ts-node": {
+    "files": true
+  },
+  "files": ["src/global.d.ts"],
+  "compilerOptions": {
+    "allowJs": true,
+    "composite": false,
+    "declaration": true,
+    "declarationMap": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "incremental": true,
+    "inlineSources": false,
+    "isolatedModules": true,
+    "lib": ["ES2021", "DOM", "DOM.Iterable"],
+    "moduleResolution": "node",
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "preserveWatchOutput": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "typeRoots": ["node_modules/@types", "src/global.d.ts"],
+    "target": "es6",
+    "noEmit": false,
+    "module": "commonjs",
+    "sourceMap": true,
+    "rootDir": ".",
+    "outDir": "dist",
+    // "baseUrl": "./src",
+    "paths": {
+      // "~/*": ["/src/*"],
+      // "~/lib/*": ["./src/lib/*"]
+    }
+  },
+  "include": ["src", "global.d.ts", "**/*.ts"],
+  "exclude": ["node_modules"]
+}

tsconfig.lint.json

@@ -0,0 +1,5 @@
+{
+    "extends": "./tsconfig.json",
+    "include": ["src"],
+    "exclude": ["node_modules", "dist", "coverage"]
+  }