Files
ai-showcase-platform/lib/database-failover.ts

683 lines
22 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// =====================================================
// 資料庫備援連線服務
// =====================================================
import mysql from 'mysql2/promise';
// 資料庫配置介面
interface DatabaseConfig {
host: string;
port: number;
user: string;
password: string;
database: string;
charset: string;
timezone: string;
acquireTimeout: number;
timeout: number;
reconnect: boolean;
connectionLimit: number;
queueLimit: number;
retryDelay: number;
maxRetries: number;
idleTimeout: number;
maxIdle: number;
ssl: boolean;
}
// 主機資料庫配置
const masterConfig = {
host: process.env.DB_HOST || '122.100.99.161',
port: parseInt(process.env.DB_PORT || '43306'),
user: process.env.DB_USER || 'A999',
password: process.env.DB_PASSWORD || '1023',
database: process.env.DB_NAME || 'db_AI_Platform',
charset: 'utf8mb4',
timezone: '+08:00',
connectionLimit: 5, // 減少連接數,避免 Too many connections
queueLimit: 10, // 允許排隊,避免立即失敗
idleTimeout: 60000, // 1分鐘空閒超時快速釋放連接
acquireTimeout: 10000, // 10秒獲取連接超時
timeout: 10000, // 10秒查詢超時
reconnect: true, // 啟用重連
ssl: false as any,
};
// 備機資料庫配置
const slaveConfig = {
host: process.env.SLAVE_DB_HOST || 'mysql.theaken.com',
port: parseInt(process.env.SLAVE_DB_PORT || '33306'),
user: process.env.SLAVE_DB_USER || 'AI_Platform',
password: process.env.SLAVE_DB_PASSWORD || 'Aa123456',
database: process.env.SLAVE_DB_NAME || 'db_AI_Platform', // 修正為 AI 平台資料庫
charset: 'utf8mb4',
timezone: '+08:00',
connectionLimit: 5, // 減少連接數,避免 Too many connections
queueLimit: 10, // 允許排隊,避免立即失敗
idleTimeout: 60000, // 1分鐘空閒超時快速釋放連接
acquireTimeout: 10000, // 10秒獲取連接超時
timeout: 10000, // 10秒查詢超時
reconnect: true, // 啟用重連
ssl: false as any,
};
// 備援狀態
interface FailoverStatus {
isEnabled: boolean;
currentDatabase: 'master' | 'slave';
masterHealthy: boolean;
slaveHealthy: boolean;
lastHealthCheck: number;
consecutiveFailures: number;
}
// 備援資料庫管理類
export class DatabaseFailover {
private static instance: DatabaseFailover;
private masterPool: mysql.Pool | null = null;
private slavePool: mysql.Pool | null = null;
private status: FailoverStatus;
private healthCheckInterval: NodeJS.Timeout | null = null;
private constructor() {
this.status = {
isEnabled: process.env.DB_FAILOVER_ENABLED === 'true',
currentDatabase: 'master',
masterHealthy: false,
slaveHealthy: false,
lastHealthCheck: Date.now(), // 使用當前時間作為初始值
consecutiveFailures: 0,
};
// 同步初始化連接池
this.initializePoolsSync();
this.startHealthCheck();
}
public static getInstance(): DatabaseFailover {
if (!DatabaseFailover.instance) {
DatabaseFailover.instance = new DatabaseFailover();
}
return DatabaseFailover.instance;
}
// 同步初始化連接池
private initializePoolsSync(): void {
try {
console.log('🚀 開始同步初始化資料庫備援系統...');
// 直接創建連接池,不等待測試
this.masterPool = mysql.createPool(masterConfig);
console.log('✅ 主機資料庫連接池創建成功');
this.slavePool = mysql.createPool(slaveConfig);
console.log('✅ 備機資料庫連接池創建成功');
// 設置默認使用主機
this.status.currentDatabase = 'master';
this.status.masterHealthy = true; // 假設主機健康,後續健康檢查會驗證
console.log('🎯 當前使用資料庫: 主機');
// 異步測試連接
this.testConnectionsAsync();
} catch (error) {
console.error('❌ 資料庫連接池同步初始化失敗:', error);
}
}
// 異步測試連接
private async testConnectionsAsync(): Promise<void> {
try {
await this.testConnections();
} catch (error) {
console.error('❌ 異步連接測試失敗:', error);
}
}
// 初始化連接池
private async initializePools(): Promise<void> {
try {
console.log('🚀 開始初始化資料庫備援系統...');
// 先測試主機連接
console.log('📡 測試主機資料庫連接...');
const masterHealthy = await this.testMasterConnection();
if (masterHealthy) {
console.log('✅ 主機資料庫連接正常,使用主機');
this.status.currentDatabase = 'master';
this.status.masterHealthy = true;
// 初始化主機連接池
this.masterPool = mysql.createPool(masterConfig);
console.log('✅ 主機資料庫連接池初始化成功');
} else {
console.log('❌ 主機資料庫連接失敗,測試備機...');
// 測試備機連接
const slaveHealthy = await this.testSlaveConnection();
if (slaveHealthy) {
console.log('✅ 備機資料庫連接正常,切換到備機');
this.status.currentDatabase = 'slave';
this.status.slaveHealthy = true;
// 初始化備機連接池
this.slavePool = mysql.createPool(slaveConfig);
console.log('✅ 備機資料庫連接池初始化成功');
} else {
console.log('❌ 主機和備機都無法連接,嘗試初始化主機連接池');
this.masterPool = mysql.createPool(masterConfig);
this.status.currentDatabase = 'master';
}
}
// 初始化另一個連接池(用於健康檢查)
if (this.status.currentDatabase === 'master' && !this.slavePool) {
this.slavePool = mysql.createPool(slaveConfig);
console.log('✅ 備機資料庫連接池初始化成功(用於健康檢查)');
} else if (this.status.currentDatabase === 'slave' && !this.masterPool) {
this.masterPool = mysql.createPool(masterConfig);
console.log('✅ 主機資料庫連接池初始化成功(用於健康檢查)');
}
console.log(`🎯 當前使用資料庫: ${this.status.currentDatabase === 'master' ? '主機' : '備機'}`);
} catch (error) {
console.error('❌ 資料庫連接池初始化失敗:', error);
}
}
// 測試主機連接
private async testMasterConnection(): Promise<boolean> {
try {
const connection = await mysql.createConnection(masterConfig);
await connection.ping();
await connection.end();
return true;
} catch (error: any) {
console.error('主機資料庫連接失敗:', error.message);
return false;
}
}
// 測試備機連接
private async testSlaveConnection(): Promise<boolean> {
try {
const connection = await mysql.createConnection(slaveConfig);
await connection.ping();
await connection.end();
return true;
} catch (error: any) {
console.error('備機資料庫連接失敗:', error.message);
return false;
}
}
// 測試資料庫連接
private async testConnections(): Promise<void> {
// 測試主機
try {
if (this.masterPool) {
const connection = await this.masterPool.getConnection();
await connection.ping();
connection.release();
this.status.masterHealthy = true;
console.log('主機資料庫連接正常');
}
} catch (error) {
this.status.masterHealthy = false;
console.error('主機資料庫連接失敗:', error);
}
// 測試備機
try {
if (this.slavePool) {
const connection = await this.slavePool.getConnection();
await connection.ping();
connection.release();
this.status.slaveHealthy = true;
console.log('備機資料庫連接正常');
}
} catch (error) {
this.status.slaveHealthy = false;
console.error('備機資料庫連接失敗:', error);
}
}
// 開始健康檢查
private startHealthCheck(): void {
if (!this.status.isEnabled) return;
const interval = parseInt(process.env.DB_HEALTH_CHECK_INTERVAL || '30000');
this.healthCheckInterval = setInterval(async () => {
await this.performHealthCheck();
}, interval);
}
// 執行健康檢查
private async performHealthCheck(): Promise<void> {
const now = Date.now();
this.status.lastHealthCheck = now;
// 檢查主機
if (this.masterPool) {
try {
const connection = await this.masterPool.getConnection();
await connection.ping();
connection.release();
this.status.masterHealthy = true;
console.log('✅ 主機資料庫健康檢查通過');
} catch (error: any) {
this.status.masterHealthy = false;
console.error('❌ 主機資料庫健康檢查失敗:', error.message);
// 如果是 Too many connections 錯誤,強制標記為不健康
if (error.code === 'ER_CON_COUNT_ERROR' || error.errno === 1040) {
console.log('🚨 主機資料庫 Too many connections強制切換到備機');
}
}
}
// 檢查備機
if (this.slavePool) {
try {
const connection = await this.slavePool.getConnection();
await connection.ping();
connection.release();
this.status.slaveHealthy = true;
console.log('✅ 備機資料庫健康檢查通過');
} catch (error: any) {
this.status.slaveHealthy = false;
console.error('❌ 備機資料庫健康檢查失敗:', error.message);
}
}
// 決定當前使用的資料庫
this.determineCurrentDatabase();
}
// 決定當前使用的資料庫
private determineCurrentDatabase(): void {
const previousDatabase = this.status.currentDatabase;
if (this.status.masterHealthy) {
if (this.status.currentDatabase !== 'master') {
console.log('🔄 主機資料庫恢復,切換回主機');
this.status.currentDatabase = 'master';
this.status.consecutiveFailures = 0;
}
} else if (this.status.slaveHealthy) {
if (this.status.currentDatabase !== 'slave') {
console.log('🔄 主機資料庫故障,切換到備機');
this.status.currentDatabase = 'slave';
this.status.consecutiveFailures++;
}
} else {
this.status.consecutiveFailures++;
console.error('❌ 主機和備機資料庫都無法連接');
}
// 記錄狀態變化
if (previousDatabase !== this.status.currentDatabase) {
console.log(`📊 資料庫狀態變化: ${previousDatabase}${this.status.currentDatabase}`);
console.log(`🔧 當前資料庫: ${this.status.currentDatabase}`);
console.log(`📈 主機健康: ${this.status.masterHealthy ? '✅' : '❌'}`);
console.log(`📈 備機健康: ${this.status.slaveHealthy ? '✅' : '❌'}`);
}
}
// 強制切換到備機(緊急情況)
public forceSwitchToSlave(): void {
console.log('🚨 強制切換到備機資料庫');
this.status.masterHealthy = false;
this.status.currentDatabase = 'slave';
this.status.consecutiveFailures++;
console.log('✅ 已強制切換到備機');
}
// 強制切換到主機
public forceSwitchToMaster(): void {
console.log('🔄 強制切換到主機資料庫');
this.status.slaveHealthy = false;
this.status.currentDatabase = 'master';
this.status.consecutiveFailures = 0;
console.log('✅ 已強制切換到主機');
}
// 獲取當前資料庫
public getCurrentDatabase(): 'master' | 'slave' {
return this.status.currentDatabase;
}
// 獲取當前連接池
private getCurrentPool(): mysql.Pool | null {
if (this.status.currentDatabase === 'master') {
if (this.masterPool) {
return this.masterPool;
} else if (this.slavePool) {
// 主機不可用,嘗試使用備機
console.log('⚠️ 主機連接池不可用,嘗試使用備機');
this.status.currentDatabase = 'slave';
return this.slavePool;
}
} else if (this.status.currentDatabase === 'slave') {
if (this.slavePool) {
return this.slavePool;
} else if (this.masterPool) {
// 備機不可用,嘗試使用主機
console.log('⚠️ 備機連接池不可用,嘗試使用主機');
this.status.currentDatabase = 'master';
return this.masterPool;
}
}
console.error('❌ 沒有可用的資料庫連接池');
return null;
}
// 獲取連接
public async getConnection(): Promise<mysql.PoolConnection> {
let retries = 0;
const maxRetries = parseInt(process.env.DB_RETRY_ATTEMPTS || '3');
const retryDelay = parseInt(process.env.DB_RETRY_DELAY || '2000');
while (retries < maxRetries) {
const pool = this.getCurrentPool();
if (!pool) {
throw new Error('沒有可用的資料庫連接');
}
try {
return await pool.getConnection();
} catch (error: any) {
console.error(`資料庫連接失敗 (嘗試 ${retries + 1}/${maxRetries}):`, error.message);
console.error('錯誤詳情:', {
code: error.code,
errno: error.errno,
sqlState: error.sqlState,
sqlMessage: error.sqlMessage
});
// 嚴格處理 Too many connections 錯誤
if (error.code === 'ER_CON_COUNT_ERROR' || error.errno === 1040) {
console.log('🚨 檢測到 Too many connections立即強制切換到備機');
this.forceSwitchToSlave();
// 立即嘗試使用備機連接
const slavePool = this.getSlavePool();
if (slavePool) {
try {
console.log('🔄 嘗試使用備機連接...');
return await slavePool.getConnection();
} catch (slaveError: any) {
console.error('❌ 備機連接也失敗:', slaveError.message);
}
}
}
// 處理其他連接錯誤
if (error.code === 'ECONNRESET' ||
error.code === 'PROTOCOL_CONNECTION_LOST') {
console.log(`🔄 檢測到連接問題,觸發健康檢查並嘗試切換資料庫...`);
// 標記當前資料庫為不健康
if (this.status.currentDatabase === 'master') {
this.status.masterHealthy = false;
console.log('❌ 主機資料庫標記為不健康');
} else {
this.status.slaveHealthy = false;
console.log('❌ 備機資料庫標記為不健康');
}
// 觸發健康檢查
await this.performHealthCheck();
}
retries++;
if (retries < maxRetries) {
console.log(`等待 ${retryDelay}ms 後重試...`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
continue;
}
throw error;
}
}
throw new Error('資料庫連接失敗,已達到最大重試次數');
}
// 執行查詢
public async query<T = any>(sql: string, params?: any[]): Promise<T[]> {
let retries = 0;
const maxRetries = parseInt(process.env.DB_RETRY_ATTEMPTS || '3');
const retryDelay = parseInt(process.env.DB_RETRY_DELAY || '2000');
while (retries < maxRetries) {
let connection;
try {
connection = await this.getConnection();
const [rows] = await connection.execute(sql, params);
// 完全關閉連線
await connection.destroy();
connection = null;
console.log('✅ 備援查詢連線已完全關閉');
return rows as T[];
} catch (error: any) {
// 確保連線被關閉
if (connection) {
try {
await connection.destroy();
} catch (destroyError) {
console.error('關閉連線時發生錯誤:', destroyError);
}
connection = null;
}
console.error(`資料庫查詢錯誤 (嘗試 ${retries + 1}/${maxRetries}):`, error.message);
console.error('查詢錯誤詳情:', {
code: error.code,
errno: error.errno,
sqlState: error.sqlState,
sqlMessage: error.sqlMessage
});
// 嚴格處理 Too many connections 錯誤
if (error.code === 'ER_CON_COUNT_ERROR' || error.errno === 1040) {
console.log('🚨 查詢時檢測到 Too many connections立即強制切換到備機');
this.forceSwitchToSlave();
// 立即嘗試使用備機連接
const slavePool = this.getSlavePool();
if (slavePool) {
try {
console.log('🔄 查詢時嘗試使用備機連接...');
const slaveConnection = await slavePool.getConnection();
const [rows] = await slaveConnection.execute(sql, params);
slaveConnection.release();
return rows as T[];
} catch (slaveError: any) {
console.error('❌ 備機查詢也失敗:', slaveError.message);
}
}
}
// 處理其他連接錯誤
if (error.code === 'ECONNRESET' ||
error.code === 'PROTOCOL_CONNECTION_LOST') {
console.log(`🔄 查詢時檢測到連接問題,觸發健康檢查並嘗試切換資料庫...`);
// 標記當前資料庫為不健康
if (this.status.currentDatabase === 'master') {
this.status.masterHealthy = false;
console.log('❌ 主機資料庫標記為不健康');
} else {
this.status.slaveHealthy = false;
console.log('❌ 備機資料庫標記為不健康');
}
// 觸發健康檢查
await this.performHealthCheck();
}
retries++;
if (retries < maxRetries) {
console.log(`等待 ${retryDelay}ms 後重試查詢...`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
continue;
}
throw error;
} finally {
if (connection) {
connection.release();
}
}
}
throw new Error('資料庫查詢失敗,已達到最大重試次數');
}
// 執行單一查詢
public async queryOne<T = any>(sql: string, params?: any[]): Promise<T | null> {
try {
const results = await this.query<T>(sql, params);
return results.length > 0 ? results[0] : null;
} catch (error) {
console.error('資料庫單一查詢錯誤:', error);
throw error;
}
}
// 執行插入 - 每次查詢完就完全關閉連線
public async insert(sql: string, params?: any[]): Promise<mysql.ResultSetHeader> {
let connection;
try {
connection = await this.getConnection();
const [result] = await connection.execute(sql, params);
return result as mysql.ResultSetHeader;
} finally {
if (connection) {
try {
await connection.destroy();
console.log('✅ 備援插入連線已完全關閉');
} catch (destroyError) {
console.error('關閉連線時發生錯誤:', destroyError);
}
}
}
}
// 執行更新 - 每次查詢完就完全關閉連線
public async update(sql: string, params?: any[]): Promise<mysql.ResultSetHeader> {
let connection;
try {
connection = await this.getConnection();
const [result] = await connection.execute(sql, params);
return result as mysql.ResultSetHeader;
} finally {
if (connection) {
try {
await connection.destroy();
console.log('✅ 備援更新連線已完全關閉');
} catch (destroyError) {
console.error('關閉連線時發生錯誤:', destroyError);
}
}
}
}
// 執行刪除 - 每次查詢完就完全關閉連線
public async delete(sql: string, params?: any[]): Promise<mysql.ResultSetHeader> {
let connection;
try {
connection = await this.getConnection();
const [result] = await connection.execute(sql, params);
return result as mysql.ResultSetHeader;
} finally {
if (connection) {
try {
await connection.destroy();
console.log('✅ 備援刪除連線已完全關閉');
} catch (destroyError) {
console.error('關閉連線時發生錯誤:', destroyError);
}
}
}
}
// 開始事務
public async beginTransaction(): Promise<mysql.PoolConnection> {
const connection = await this.getConnection();
await connection.beginTransaction();
return connection;
}
// 提交事務
public async commit(connection: mysql.PoolConnection): Promise<void> {
await connection.commit();
connection.release();
}
// 回滾事務
public async rollback(connection: mysql.PoolConnection): Promise<void> {
await connection.rollback();
connection.release();
}
// 獲取備援狀態
public getStatus(): FailoverStatus {
return { ...this.status };
}
// 獲取主機連接池
public getMasterPool(): mysql.Pool | null {
return this.masterPool;
}
// 獲取備機連接池
public getSlavePool(): mysql.Pool | null {
return this.slavePool;
}
// 強制切換到指定資料庫
public async switchToDatabase(database: 'master' | 'slave'): Promise<boolean> {
if (database === 'master' && this.status.masterHealthy) {
this.status.currentDatabase = 'master';
return true;
} else if (database === 'slave' && this.status.slaveHealthy) {
this.status.currentDatabase = 'slave';
return true;
}
return false;
}
// 關閉所有連接池
public async close(): Promise<void> {
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
}
if (this.masterPool) {
await this.masterPool.end();
}
if (this.slavePool) {
await this.slavePool.end();
}
}
}
// 導出單例實例
export const dbFailover = DatabaseFailover.getInstance();
// 導出類型
export type { PoolConnection } from 'mysql2/promise';