Feature/recorder service

.devcontainer/docker-compose.yml

@@ -35,6 +35,13 @@ services:
       target: build
     command: ["yarn", "workspace", "@ftl/config-service", "run", "start:dev"]
 
+  recorderservice:
+    build:
+      context: ./
+      dockerfile: ./docker/Dockerfile.services
+      target: build
+    command: ["yarn", "workspace", "@ftl/recorder-service", "run", "start:dev"]
+
   watchservice:
     image: node:16-alpine
     volumes:

docker-compose.yml

@@ -13,12 +13,14 @@ services:
       FTL_NODE_SERVICE: nodeservice:8080
       FTL_STREAM_SERVICE: streamservice:8080
       FTL_CONFIG_SERVICE: configservice:8080
+      FTL_RECORDER_SERVICE: recorderservice:8080
     depends_on:
       - socketservice
       - authservice
       - configservice
       - nodeservice
       - streamservice
+      - recorderservice
     volumes:
       - ./packages/client/dist:/usr/share/nginx/html
   socketservice:
@@ -68,6 +70,18 @@ services:
       MONGO_HOST: authmongo
     volumes:
       - ./:/usr/src/app
+  recorderservice:
+    image: ftlab.utu.fi/app-recorderservice:${TAG:-latest}
+    build:
+      context: ./
+      dockerfile: ./docker/Dockerfile.services
+      target: recorderservice
+    environment:
+      NODE_ENV: development
+      REDIS_HOST: redis
+    volumes:
+      - ./:/usr/src/app
+      - "$HOME/ftl-data:/data/ftl"
   streamservice:
     image: ftlab.utu.fi/app-streamservice:${TAG:-latest}
     build:

docker/Dockerfile.services

@@ -157,3 +157,28 @@ COPY --from=build /usr/src/app/packages/stream-service/resources/ ./packages/str
 USER node
 
 CMD ["yarn", "workspace", "@ftl/stream-service", "run", "start"]
+
+FROM node:16-alpine AS recorderservice
+
+WORKDIR /usr/src/app
+
+RUN mkdir ./packages ./packages/node-service ./packages/common ./packages/api
+COPY ./package.json ./yarn.lock ./lerna.json ./
+COPY ./packages/common/package.json ./packages/common/
+COPY ./packages/types/package.json ./packages/types/
+COPY ./packages/api/package.json ./packages/api/
+COPY ./packages/recorder-service/package.json ./packages/recorder-service/
+
+RUN yarn install --production \
+  && yarn cache clean \
+  && yarn autoclean --init \
+  && yarn autoclean --force
+
+COPY --from=build /usr/src/app/packages/common/dist/ ./packages/common/dist/
+COPY --from=build /usr/src/app/packages/types/dist/ ./packages/types/dist/
+COPY --from=build /usr/src/app/packages/api/dist/ ./packages/api/dist/
+COPY --from=build /usr/src/app/packages/recorder-service/dist/ ./packages/recorder-service/dist/
+
+USER node
+
+CMD ["yarn", "workspace", "@ftl/recorder-service", "run", "start"]

docker/production/docker-compose.yml

@@ -11,6 +11,7 @@ services:
       FTL_CONFIG_SERVICE: configservice:8080
       FTL_NODE_SERVICE: nodeservice:8080
       FTL_STREAM_SERVICE: streamservice:8080
+      FTL_RECORDER_SERVICE: recorderservice:8080
       ASSET_PATH: /lab/
     depends_on:
       - socketservice
@@ -18,6 +19,7 @@ services:
       - configservice
       - nodeservice
       - streamservice
+      - recorderservice
   socketservice:
     image: ftlab.utu.fi/app-socketservice
     restart: unless-stopped
@@ -45,6 +47,14 @@ services:
       NODE_ENV: production
       REDIS_HOST: redis
       MONGO_HOST: authmongo
+  recorderservice:
+    image: ftlab.utu.fi/app-recorderservice
+    restart: unless-stopped
+    environment:
+      NODE_ENV: production
+      REDIS_HOST: redis
+    volumes:
+      - /srv/ftl/webapp/data/recordings:/data/ftl
   authservice:
     image: ftlab.utu.fi/app-authservice:${TAG:-latest}
     restart: unless-stopped

packages/api/src/index.ts

 export * from './nodes';
 export * from './streams';
 export * from './configs';
+export * from './recording';

packages/api/src/recording/events.ts

+import { redisSendEvent } from '@ftl/common';
+import { BaseEvent } from '../events';
+
+export type RecordingEventType = 'start' | 'cancel' | 'complete';
+
+export interface RecordingEvent extends BaseEvent {
+  id: string;
+  event: RecordingEventType;
+  size: number;
+  duration: number;
+  date: Date;
+  filename: string;
+  owner: string;
+}
+
+export function sendRecordingEvent(event: RecordingEvent) {
+  redisSendEvent('event:recording', event);
+}

packages/api/src/recording/index.ts

+export * from './events';

packages/client/reverseproxy.conf

@@ -23,6 +23,10 @@ upstream config_upstream {
     server ${FTL_CONFIG_SERVICE};
 }
 
+upstream recorder_upstream {
+    server ${FTL_RECORDER_SERVICE};
+}
+
 map $http_upgrade $connection_upgrade {
     default upgrade;
     '' close;
@@ -111,6 +115,15 @@ server {
         proxy_set_header   X-Forwarded-Host $server_name;
     }
 
+    location /v1/recorder {
+        proxy_pass         http://recorder_upstream;
+        proxy_redirect     off;
+        proxy_set_header   Host $host;
+        proxy_set_header   X-Real-IP $remote_addr;
+        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header   X-Forwarded-Host $server_name;
+    }
+
     location ~* \.(?:manifest|appcache|html?|xml|json)$ {
         expires -1;
         # access_log logs/static.log; # I don't usually include a static log

packages/client/src/api/paths.ts

@@ -10,3 +10,4 @@ export const OAUTH2_TOKEN = `${OAUTH2}/token`;
 export const NODES = `${API_PATH}/nodes`;
 export const STREAMS = `${API_PATH}/streams`;
 export const CONFIGS = `${API_PATH}/configuration`;
+export const RECORDER = `${API_PATH}/recorder`;

packages/client/src/api/recorder.ts

+import {RECORDER} from './paths';
+import axios from 'axios';
+
+export interface IRecording {
+  id?: string;
+  owner: string;
+  streams: string[];
+  channels: number[];
+  size?: number;
+  startTime: Date;
+  duration?: number;
+  status: string;
+}
+
+export interface ICreateRecording {
+  streams: string[];
+  channels: number[];
+  status?: string;
+}
+
+export async function startRecording(data: ICreateRecording): Promise<IRecording> {
+  try {
+    const res = await axios.post(`${RECORDER}`, data);
+    return res.data;
+  } catch(err) {
+    return null;
+  }
+}
+
+export async function stopRecording(id: string): Promise<IRecording> {
+  try {
+    const res = await axios.put(`${RECORDER}/${id}`, { status: 'stopped' });
+    return res.data;
+  } catch(err) {
+    return null;
+  }
+}
+
+
+export async function getRecordings(): Promise<IRecording[]> {
+  try {
+    const res = await axios.get(RECORDER);
+    return res.data;
+  } catch(err) {
+    return [];
+  }
+}

packages/client/src/services/PeerRoot.tsx

@@ -67,6 +67,10 @@ export function PeerRoot(): React.ReactElement {
             setTimeout(createPeer, 1000);
             console.log('Socket disconnect');
         });
+
+        peer.bind('event', (name: string) => {
+          console.log('Service Event', name);
+        });
     }
 
     useEffect(createPeer, [session]);

packages/client/src/views/Developer/MenuBar.tsx

 import React, {useEffect, useState} from 'react';
 import styled from 'styled-components';
 import {IconButton} from '../../components/IconButton';
-import {FaPlay, FaPause, FaSyncAlt} from 'react-icons/fa';
+import {FaPlay, FaPause, FaSyncAlt, FaCircle} from 'react-icons/fa';
 import { FTLStream } from '@ftl/stream';
+import {startRecording, stopRecording} from '../../api/recorder';
 
 const MenuContainer = styled.nav`
     display: flex;
@@ -15,6 +16,7 @@ interface Props {
 
 export function MenuBar({stream}: Props) {
     const [paused, setPaused] = useState(false);
+    const [recording, setRecording] = useState<string>(null);
 
     useEffect(() => {
         stream.paused = paused;
@@ -27,5 +29,16 @@ export function MenuBar({stream}: Props) {
         <IconButton onClick={() => stream.set(69, "reset")}>
             <FaSyncAlt />
         </IconButton>
+        <IconButton onClick={async () => {
+            if (recording) {
+              stopRecording(recording);
+              setRecording(null);
+            } else {
+              const res = await startRecording({streams: [stream.uri], channels: [0]});
+              setRecording(res.id);
+            }
+          }}>
+            <FaCircle color={recording ? 'red' : 'black'}/>
+        </IconButton>
     </MenuContainer>;
 }

packages/common/src/redis.ts

@@ -120,6 +120,19 @@ export function redisGet<T>(key: string): Promise<T> {
   });
 }
 
+export function redisMGet<T>(keys: string[]): Promise<T[]> {
+  initRedis();
+  return new Promise((resolve) => {
+    redisClient.mget(...keys, (err, reply) => {
+      if (err) {
+        resolve(null);
+      } else {
+        resolve(JSON.parse(reply))
+      }
+    });
+  });
+}
+
 export function redisDelete(key: string): Promise<boolean> {
   initRedis();
   return new Promise((resolve) => {

packages/recorder-service/.eslintrc.json

+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "plugin:react/recommended",
+        "airbnb"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaFeatures": {
+            "jsx": true
+        },
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "react",
+        "@typescript-eslint"
+    ],
+    "rules": {
+        "semi": 0,
+        "import/extensions": ["error", "never"],
+        "import/no-unresolved": 0,
+        "class-methods-use-this": 0
+    }
+}

packages/recorder-service/README.md

+# Recorder Service
+
+Record streams to FTL files on disk.

packages/recorder-service/package.json

+{
+  "name": "@ftl/recorder-service",
+  "version": "1.0.0",
+  "main": "./dist/index.js",
+  "scripts": {
+    "clean": "rm -rf ./dist && rm -rf ./node_modules",
+    "build": "tsc",
+    "start": "node ./dist/index.js",
+    "start:dev": "ts-node-dev --respawn --transpile-only --watch src --ignore-watch node_modules src/index.ts"
+  },
+  "devDependencies": {
+    "@ftl/types": "1.0.0",
+    "@types/compression": "^1.7.2",
+    "@types/express": "^4.17.13",
+    "@types/jest": "^26.0.24",
+    "@types/multer": "^1.4.7",
+    "@types/node": "^16.4.9",
+    "@typescript-eslint/eslint-plugin": "^4.31.1",
+    "@typescript-eslint/parser": "^4.31.1",
+    "eslint": "^7.32.0",
+    "eslint-config-airbnb": "^18.2.1",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-react": "^7.25.2",
+    "eslint-plugin-react-hooks": "^4.2.0",
+    "ts-node-dev": "^1.1.8",
+    "typescript": "^4.3.5"
+  },
+  "dependencies": {
+    "@ftl/api": "1.0.0",
+    "@ftl/common": "1.0.0",
+    "@ftl/types": "1.0.0",
+    "@tsed/ajv": "^6.62.0",
+    "@tsed/common": "^6.62.0",
+    "@tsed/core": "^6.62.0",
+    "@tsed/di": "^6.62.0",
+    "@tsed/exceptions": "^6.62.0",
+    "@tsed/json-mapper": "^6.62.0",
+    "@tsed/logger": "^5.17.0",
+    "@tsed/mongoose": "^6.62.0",
+    "@tsed/platform-express": "^6.62.0",
+    "@tsed/schema": "^6.62.0",
+    "ajv": "^8.6.3",
+    "body-parser": "^1.19.0",
+    "compression": "^1.7.4",
+    "cookie-parser": "^1.4.5",
+    "express": "4",
+    "express-session": "^1.17.2",
+    "msgpack5": "^5.3.2",
+    "uuid": "^8.3.2"
+  }
+}

packages/recorder-service/src/controllers/recordings.ts

+/* eslint-disable class-methods-use-this */
+import {
+  BodyParams, Get, Inject, Post, PathParams, Put,
+} from '@tsed/common';
+import { Description, Groups } from '@tsed/schema';
+import { AccessToken } from '@ftl/types';
+import { Controller, UseToken } from '@ftl/common';
+import RecorderService from '../services/recorder';
+import Recording from '../models/recording';
+
+@Controller('/recorder')
+export default class RecorderController {
+  @Inject()
+  recorderService: RecorderService;
+
+  @Get('/')
+  @Description('Get all active recordings for user')
+  async find(
+    @UseToken() token: AccessToken,
+  ): Promise<Recording[]> {
+    return this.recorderService.find(token.user?.id);
+  }
+
+  @Post('/')
+  async create(@BodyParams() @Groups('creation') request: Recording, @UseToken() token: AccessToken): Promise<Recording> {
+    return this.recorderService.create(request, token.user?.id);
+  }
+
+  @Get('/:id')
+  async get(@PathParams('id') id: string, @UseToken() token: AccessToken): Promise<Recording> {
+    return this.recorderService.get(id, token.user?.id);
+  }
+
+  @Put('/:id')
+  async update(@PathParams('id') id: string, @BodyParams() @Groups('update') request: Recording, @UseToken() token: AccessToken): Promise<Recording> {
+    return this.recorderService.update(id, token.user?.id, request);
+  }
+}

packages/recorder-service/src/index.ts

+import { $log } from '@tsed/common';
+import { PlatformExpress } from '@tsed/platform-express';
+import Server from './server';
+
+async function bootstrap() {
+  try {
+    $log.debug('Start server...');
+    const platform = await PlatformExpress.bootstrap(Server, {
+      // extra settings
+    });
+
+    await platform.listen();
+    $log.debug('Server initialized');
+  } catch (er) {
+    $log.error(er);
+  }
+}
+
+bootstrap();

packages/recorder-service/src/models/recording.ts

+import {
+  Property, Required, Groups, DateTime, CollectionOf, Default,
+} from '@tsed/schema';
+
+export default class Recording {
+    @Required()
+    @Groups('!creation', '!update')
+    id: string;
+
+    @Required()
+    @CollectionOf(String)
+    streams: string[];
+
+    @Required()
+    @CollectionOf(Number)
+    channels: number[];
+
+    @Required()
+    @DateTime()
+    @Default(new Date())
+    startTime: Date;
+
+    @Required()
+    @Groups('!creation', '!update')
+    owner: string;
+
+    filename?: string;
+
+    @Property()
+    @Groups('!creation', '!update')
+    size?: number;
+
+    @Property()
+    @Groups('!creation')
+    status: 'recording' | 'paused' | 'stopped';
+
+    @Property()
+    @Groups('!creation', '!update')
+    duration: number;
+}

packages/recorder-service/src/server.ts

+import { Configuration, Inject, PlatformApplication } from '@tsed/common';
+import express from 'express';
+import compress from 'compression';
+import cookieParser from 'cookie-parser';
+import { redisStreamListen, redisSetGroup } from '@ftl/common';
+
+const rootDir = __dirname;
+
+@Configuration({
+  rootDir,
+  acceptMimes: ['application/json'],
+  port: 8080,
+  debug: false,
+  mount: {
+    '/v1': `${rootDir}/controllers/**/*.ts`,
+  },
+})
+export default class Server {
+  @Inject()
+  app: PlatformApplication;
+
+  @Configuration()
+  settings: Configuration;
+
+  public $beforeInit() {
+    redisSetGroup('recorder-service');
+  }
+
+  public $afterInit() {
+    redisStreamListen('consumer1');
+  }
+
+  /**
+   * This method let you configure the express middleware required by your application to works.
+   * @returns {Server}
+   */
+  public $beforeRoutesInit(): void | Promise<any> {
+    this.app
+      .use(compress({}))
+      .use(cookieParser())
+      .use(express.urlencoded())
+      .use(express.json());
+  }
+}