Feature: Option to add apikeys to the DB instead of api.json. (#2783)

vinodkiran · HenryHengZJ · web-flow · commit 2dd1791ec26d · 2024-07-26T17:52:12.000+01:00
* Feature: Option to add apikeys to the DB instead of api.json.

* add api storage type env variable

* code cleanup and simplification.

* md table fixes

---------

Co-authored-by: Henry &lt;hzj94@hotmail.com&gt;
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -121,7 +121,7 @@ Flowise has 3 different modules in a single mono repository.
 Flowise support different environment variables to configure your instance. You can specify the following variables in the `.env` file inside `packages/server` folder. Read [more](https://docs.flowiseai.com/environment-variables)
 
 | Variable                     | Description                                                                                     | Type                                             | Default                             |
-| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
+| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ |-------------------------------------| 
 | PORT                         | The HTTP port Flowise runs on                                                                   | Number                                           | 3000                                |
 | CORS_ORIGINS                 | The allowed origins for all cross-origin HTTP calls                                             | String                                           |                                     |
 | IFRAME_ORIGINS               | The allowed origins for iframe src embedding                                                    | String                                           |                                     |
@@ -133,7 +133,8 @@ Flowise support different environment variables to configure your instance. You
 | LOG_PATH                     | Location where log files are stored                                                             | String                                           | `your-path/Flowise/logs`            |
 | LOG_LEVEL                    | Different levels of logs                                                                        | Enum String: `error`, `info`, `verbose`, `debug` | `info`                              |
 | LOG_JSON_SPACES              | Spaces to beautify JSON logs                                                                    |                                                  | 2                                   |
-| APIKEY_PATH                  | Location where api keys are saved                                                               | String                                           | `your-path/Flowise/packages/server` |
+| APIKEY_STORAGE_TYPE          | To store api keys on a JSON file or database. Default is `json`                                 | Enum String: `json`, `db`                        | `json`                              | 
+| APIKEY_PATH                  | Location where api keys are saved when `APIKEY_STORAGE_TYPE` is `json`                          | String                                           | `your-path/Flowise/packages/server` |
 | TOOL_FUNCTION_BUILTIN_DEP    | NodeJS built-in modules to be used for Tool Function                                            | String                                           |                                     |
 | TOOL_FUNCTION_EXTERNAL_DEP   | External modules to be used for Tool Function                                                   | String                                           |                                     |
 | DATABASE_TYPE                | Type of database to store the flowise data                                                      | Enum String: `sqlite`, `mysql`, `postgres`       | `sqlite`                            |
@@ -146,8 +147,8 @@ Flowise support different environment variables to configure your instance. You
 | DATABASE_SSL_KEY_BASE64      | Database SSL client cert in base64 (takes priority over DATABASE_SSL)                           | Boolean                                          | false                               |
 | DATABASE_SSL                 | Database connection overssl (When DATABASE_TYPE is postgre)                                     | Boolean                                          | false                               |
 | SECRETKEY_PATH               | Location where encryption key (used to encrypt/decrypt credentials) is saved                    | String                                           | `your-path/Flowise/packages/server` |
-| FLOWISE_SECRETKEY_OVERWRITE  | Encryption key to be used instead of the key stored in SECRETKEY_PATH                           | String                                           |
-| DISABLE_FLOWISE_TELEMETRY    | Turn off telemetry                                                                              | Boolean                                          |
+| FLOWISE_SECRETKEY_OVERWRITE  | Encryption key to be used instead of the key stored in SECRETKEY_PATH                           | String                                           |                                     |
+| DISABLE_FLOWISE_TELEMETRY    | Turn off telemetry                                                                              | Boolean                                          |                                     |
 | MODEL_LIST_CONFIG_JSON       | File path to load list of models from your local config file                                    | String                                           | `/your_model_list_config_file_path` |
 | STORAGE_TYPE                 | Type of storage for uploaded files. default is `local`                                          | Enum String: `s3`, `local`                       | `local`                             |
 | BLOB_STORAGE_PATH            | Local folder path where uploaded files are stored when `STORAGE_TYPE` is `local`                | String                                           | `your-home-dir/.flowise/storage`    |
diff --git a/docker/.env.example b/docker/.env.example
@@ -47,4 +47,6 @@ BLOB_STORAGE_PATH=/root/.flowise/storage
 # S3_STORAGE_ACCESS_KEY_ID=<your-access-key>
 # S3_STORAGE_SECRET_ACCESS_KEY=<your-secret-key>
 # S3_STORAGE_REGION=us-west-2
-# S3_ENDPOINT_URL=<custom-s3-endpoint-url>
+# S3_ENDPOINT_URL=<custom-s3-endpoint-url>
+
+# APIKEY_STORAGE_TYPE=json (json | db)
diff --git a/i18n/CONTRIBUTING-ZH.md b/i18n/CONTRIBUTING-ZH.md
@@ -128,7 +128,8 @@ Flowise 支持不同的环境变量来配置您的实例。您可以在 `package
 | DEBUG                        | 打印组件的日志                                                       | 布尔值                                          |                                     |
 | LOG_PATH                     | 存储日志文件的位置                                                   | 字符串                                          | `your-path/Flowise/logs`            |
 | LOG_LEVEL                    | 日志的不同级别                                                       | 枚举字符串: `error`, `info`, `verbose`, `debug` | `info`                              |
-| APIKEY_PATH                  | 存储 API 密钥的位置                                                  | 字符串                                          | `your-path/Flowise/packages/server` |
+| APIKEY_STORAGE_TYPE          | 存储 API 密钥的存储类型                                              | 枚举字符串: `json`, `db`                        | `json`                              |
+| APIKEY_PATH                  | 存储 API 密钥的位置, 当`APIKEY_STORAGE_TYPE`是`json`                 | 字符串                                          | `your-path/Flowise/packages/server` |
 | TOOL_FUNCTION_BUILTIN_DEP    | 用于工具函数的 NodeJS 内置模块                                       | 字符串                                          |                                     |
 | TOOL_FUNCTION_EXTERNAL_DEP   | 用于工具函数的外部模块                                               | 字符串                                          |                                     |
 | DATABASE_TYPE                | 存储 flowise 数据的数据库类型                                        | 枚举字符串: `sqlite`, `mysql`, `postgres`       | `sqlite`                            |
diff --git a/packages/server/.env.example b/packages/server/.env.example
@@ -47,4 +47,6 @@ PORT=3000
 # S3_STORAGE_ACCESS_KEY_ID=<your-access-key>
 # S3_STORAGE_SECRET_ACCESS_KEY=<your-secret-key>
 # S3_STORAGE_REGION=us-west-2
-# S3_ENDPOINT_URL=<custom-s3-endpoint-url>
+# S3_ENDPOINT_URL=<custom-s3-endpoint-url>
+
+# APIKEY_STORAGE_TYPE=json (json | db)
diff --git a/packages/server/src/AppConfig.ts b/packages/server/src/AppConfig.ts
@@ -0,0 +1,6 @@
+export const appConfig = {
+    apiKeys: {
+        storageType: process.env.APIKEY_STORAGE_TYPE ? process.env.APIKEY_STORAGE_TYPE.toLowerCase() : 'json'
+    }
+    // todo: add more config options here like database, log, storage, credential and allow modification from UI
+}
diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts
@@ -263,5 +263,13 @@ export interface IUploadFileSizeAndTypes {
     maxUploadSize: number
 }
 
+export interface IApiKey {
+    id: string
+    keyName: string
+    apiKey: string
+    apiSecret: string
+    updatedDate: Date
+}
+
 // DocumentStore related
 export * from './Interface.DocumentStore'
diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts
@@ -24,6 +24,7 @@ export default class Start extends Command {
         IFRAME_ORIGINS: Flags.string(),
         DEBUG: Flags.string(),
         BLOB_STORAGE_PATH: Flags.string(),
+        APIKEY_STORAGE_TYPE: Flags.string(),
         APIKEY_PATH: Flags.string(),
         SECRETKEY_PATH: Flags.string(),
         FLOWISE_SECRETKEY_OVERWRITE: Flags.string(),
@@ -100,6 +101,7 @@ export default class Start extends Command {
         // Authorization
         if (flags.FLOWISE_USERNAME) process.env.FLOWISE_USERNAME = flags.FLOWISE_USERNAME
         if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD
+        if (flags.APIKEY_STORAGE_TYPE) process.env.APIKEY_STORAGE_TYPE = flags.APIKEY_STORAGE_TYPE
         if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH
 
         // API Configuration
diff --git a/packages/server/src/controllers/apikey/index.ts b/packages/server/src/controllers/apikey/index.ts
@@ -41,6 +41,19 @@ const updateApiKey = async (req: Request, res: Response, next: NextFunction) =>
     }
 }
 
+// Import Keys from JSON file
+const importKeys = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.body === 'undefined' || !req.body.jsonFile) {
+            throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, `Error: apikeyController.importKeys - body not provided!`)
+        }
+        const apiResponse = await apikeyService.importKeys(req.body)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
 // Delete api key
 const deleteApiKey = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -72,5 +85,6 @@ export default {
     deleteApiKey,
     getAllApiKeys,
     updateApiKey,
-    verifyApiKey
+    verifyApiKey,
+    importKeys
 }
diff --git a/packages/server/src/controllers/chatflows/index.ts b/packages/server/src/controllers/chatflows/index.ts
@@ -1,11 +1,11 @@
 import { NextFunction, Request, Response } from 'express'
 import { StatusCodes } from 'http-status-codes'
+import apiKeyService from '../../services/apikey'
 import { ChatFlow } from '../../database/entities/ChatFlow'
+import { createRateLimiter } from '../../utils/rateLimit'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { ChatflowType } from '../../Interface'
 import chatflowsService from '../../services/chatflows'
-import { getApiKey } from '../../utils/apiKey'
-import { createRateLimiter } from '../../utils/rateLimit'
 
 const checkIfChatflowIsValidForStreaming = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -67,7 +67,7 @@ const getChatflowByApiKey = async (req: Request, res: Response, next: NextFuncti
                 `Error: chatflowsRouter.getChatflowByApiKey - apikey not provided!`
             )
         }
-        const apikey = await getApiKey(req.params.apikey)
+        const apikey = await apiKeyService.getApiKey(req.params.apikey)
         if (!apikey) {
             return res.status(401).send('Unauthorized')
         }
diff --git a/packages/server/src/database/entities/ApiKey.ts b/packages/server/src/database/entities/ApiKey.ts
@@ -0,0 +1,21 @@
+import { Column, Entity, PrimaryColumn, UpdateDateColumn } from 'typeorm'
+import { IApiKey } from '../../Interface'
+
+@Entity('apikey')
+export class ApiKey implements IApiKey {
+    @PrimaryColumn({ type: 'varchar', length: 20 })
+    id: string
+
+    @Column({ type: 'text' })
+    apiKey: string
+
+    @Column({ type: 'text' })
+    apiSecret: string
+
+    @Column({ type: 'text' })
+    keyName: string
+
+    @Column({ type: 'timestamp' })
+    @UpdateDateColumn()
+    updatedDate: Date
+}
diff --git a/packages/server/src/database/entities/index.ts b/packages/server/src/database/entities/index.ts
@@ -9,6 +9,7 @@ import { DocumentStore } from './DocumentStore'
 import { DocumentStoreFileChunk } from './DocumentStoreFileChunk'
 import { Lead } from './Lead'
 import { UpsertHistory } from './UpsertHistory'
+import { ApiKey } from './ApiKey'
 
 export const entities = {
     ChatFlow,
@@ -21,5 +22,6 @@ export const entities = {
     DocumentStore,
     DocumentStoreFileChunk,
     Lead,
-    UpsertHistory
+    UpsertHistory,
+    ApiKey
 }
diff --git a/packages/server/src/database/migrations/mariadb/1720230151480-AddApiKey.ts b/packages/server/src/database/migrations/mariadb/1720230151480-AddApiKey.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddApiKey1720230151480 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`apikey\` (
+                \`id\` varchar(36) NOT NULL,
+                \`apiKey\` varchar(255) NOT NULL,
+                \`apiSecret\` varchar(255) NOT NULL,
+                \`keyName\` varchar(255) NOT NULL,
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE apikey`)
+    }
+}
diff --git a/packages/server/src/database/migrations/mariadb/index.ts b/packages/server/src/database/migrations/mariadb/index.ts
@@ -20,6 +20,7 @@ import { AddLead1710832127079 } from './1710832127079-AddLead'
 import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
 import { AddTypeToChatFlow1766759476232 } from './1766759476232-AddTypeToChatFlow'
+import { AddApiKey1720230151480 } from './1720230151480-AddApiKey'
 import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage'
 
 export const mariadbMigrations = [
@@ -45,5 +46,6 @@ export const mariadbMigrations = [
     AddLeadToChatMessage1711538023578,
     AddAgentReasoningToChatMessage1714679514451,
     AddTypeToChatFlow1766759476232,
+    AddApiKey1720230151480,
     AddActionToChatMessage1721078251523
 ]
diff --git a/packages/server/src/database/migrations/mysql/1720230151480-AddApiKey.ts b/packages/server/src/database/migrations/mysql/1720230151480-AddApiKey.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddApiKey1720230151480 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`apikey\` (
+                \`id\` varchar(36) NOT NULL,
+                \`apiKey\` varchar(255) NOT NULL,
+                \`apiSecret\` varchar(255) NOT NULL,
+                \`keyName\` varchar(255) NOT NULL,
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE apikey`)
+    }
+}
diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts
@@ -20,6 +20,7 @@ import { AddLead1710832127079 } from './1710832127079-AddLead'
 import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
 import { AddTypeToChatFlow1766759476232 } from './1766759476232-AddTypeToChatFlow'
+import { AddApiKey1720230151480 } from './1720230151480-AddApiKey'
 import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage'
 
 export const mysqlMigrations = [
@@ -45,5 +46,6 @@ export const mysqlMigrations = [
     AddLeadToChatMessage1711538023578,
     AddAgentReasoningToChatMessage1714679514451,
     AddTypeToChatFlow1766759476232,
+    AddApiKey1720230151480,
     AddActionToChatMessage1721078251523
 ]
diff --git a/packages/server/src/database/migrations/postgres/1720230151480-AddApiKey.ts b/packages/server/src/database/migrations/postgres/1720230151480-AddApiKey.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddApiKey1720230151480 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS apikey (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "apiKey" varchar NOT NULL,
+                "apiSecret" varchar NOT NULL,
+                "keyName" varchar NOT NULL,
+                "updatedDate" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_96109043dd704f53-9830ab78f0" PRIMARY KEY (id)
+            );`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE apikey`)
+    }
+}
diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts
@@ -21,6 +21,7 @@ import { AddLead1710832137905 } from './1710832137905-AddLead'
 import { AddLeadToChatMessage1711538016098 } from './1711538016098-AddLeadToChatMessage'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
 import { AddTypeToChatFlow1766759476232 } from './1766759476232-AddTypeToChatFlow'
+import { AddApiKey1720230151480 } from './1720230151480-AddApiKey'
 import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage'
 
 export const postgresMigrations = [
@@ -47,5 +48,6 @@ export const postgresMigrations = [
     AddLeadToChatMessage1711538016098,
     AddAgentReasoningToChatMessage1714679514451,
     AddTypeToChatFlow1766759476232,
+    AddApiKey1720230151480,
     AddActionToChatMessage1721078251523
 ]
diff --git a/packages/server/src/database/migrations/sqlite/1720230151480-AddApiKey.ts b/packages/server/src/database/migrations/sqlite/1720230151480-AddApiKey.ts
@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddApiKey1720230151480 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "apikey" ("id" varchar PRIMARY KEY NOT NULL, 
+                "apiKey" varchar NOT NULL, 
+                "apiSecret" varchar NOT NULL, 
+                "keyName" varchar NOT NULL, 
+                "updatedDate" datetime NOT NULL DEFAULT (datetime('now')));`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "apikey";`)
+    }
+}
diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts
@@ -21,6 +21,7 @@ import { AddLeadToChatMessage1711537986113 } from './1711537986113-AddLeadToChat
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
 import { AddTypeToChatFlow1766759476232 } from './1766759476232-AddTypeToChatFlow'
 import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage'
+import { AddApiKey1720230151480 } from './1720230151480-AddApiKey'
 
 export const sqliteMigrations = [
     Init1693835579790,
@@ -45,5 +46,6 @@ export const sqliteMigrations = [
     AddLeadToChatMessage1711537986113,
     AddAgentReasoningToChatMessage1714679514451,
     AddTypeToChatFlow1766759476232,
+    AddApiKey1720230151480,
     AddActionToChatMessage1721078251523
 ]
diff --git a/packages/server/src/routes/apikey/index.ts b/packages/server/src/routes/apikey/index.ts
@@ -4,6 +4,7 @@ const router = express.Router()
 
 // CREATE
 router.post('/', apikeyController.createApiKey)
+router.post('/import', apikeyController.importKeys)
 
 // READ
 router.get('/', apikeyController.getAllApiKeys)
diff --git a/packages/server/src/services/apikey/index.ts b/packages/server/src/services/apikey/index.ts
diff --git a/packages/server/src/utils/apiKey.ts b/packages/server/src/utils/apiKey.ts
diff --git a/packages/server/src/utils/validateKey.ts b/packages/server/src/utils/validateKey.ts
diff --git a/packages/ui/src/api/apikey.js b/packages/ui/src/api/apikey.js
diff --git a/packages/ui/src/views/apikey/UploadJSONFileDialog.jsx b/packages/ui/src/views/apikey/UploadJSONFileDialog.jsx
diff --git a/packages/ui/src/views/apikey/index.jsx b/packages/ui/src/views/apikey/index.jsx
