16th_fix

app/api/admin.py

@@ -94,6 +94,12 @@ def get_system_stats():
                 TranslationJob.status == 'COMPLETED'
             ).count()
             
+            # 當日失敗任務統計
+            daily_failed = TranslationJob.query.filter(
+                func.date(TranslationJob.created_at) == target_date,
+                TranslationJob.status == 'FAILED'
+            ).count()
+            
             # 當日成本統計  
             daily_cost = db.session.query(
                 func.sum(TranslationJob.total_cost)
@@ -105,6 +111,7 @@ def get_system_stats():
                 'date': target_date.strftime('%Y-%m-%d'),
                 'jobs': daily_jobs,
                 'completed': daily_completed,
+                'failed': daily_failed,
                 'cost': float(daily_cost)
             })
         
@@ -145,6 +152,8 @@ def get_all_jobs():
         per_page = request.args.get('per_page', 50, type=int)
         user_id = request.args.get('user_id', type=int)
         status = request.args.get('status')
+        search = request.args.get('search', '').strip()
+        include_deleted = request.args.get('include_deleted', 'false').lower() == 'true'
         
         # 驗證分頁參數
         page, per_page = validate_pagination(page, min(per_page, 100))
@@ -152,6 +161,10 @@ def get_all_jobs():
         # 建立查詢
         query = TranslationJob.query
         
+        # 預設排除軟刪除的記錄，除非明確要求包含
+        if not include_deleted:
+            query = query.filter(TranslationJob.deleted_at.is_(None))
+        
         # 使用者篩選
         if user_id:
             query = query.filter_by(user_id=user_id)
@@ -161,6 +174,10 @@ def get_all_jobs():
             valid_statuses = ['PENDING', 'PROCESSING', 'COMPLETED', 'FAILED', 'RETRY']
             if status.upper() in valid_statuses:
                 query = query.filter_by(status=status.upper())
+                
+        # 檔案名搜尋
+        if search:
+            query = query.filter(TranslationJob.original_filename.like(f'%{search}%'))
         
         # 排序
         query = query.order_by(TranslationJob.created_at.desc())
@@ -438,39 +455,61 @@ def get_system_health():
             }
             status['status'] = 'unhealthy'
         
-        # Celery 工作者檢查
+        # Celery 工作者檢查 - 使用替代方案檢測
         try:
-            from celery_app import celery
-            from celery.app.control import Control
+            import redis
+            import os
+            from flask import current_app
             
-            # 檢查 Celery 工作者狀態
-            control = Control(celery)
-            inspect_obj = control.inspect(timeout=2.0)  # 設置較短超時
+            # 方法1: 檢查Redis中是否有Celery相關的key
+            redis_client = redis.from_url(current_app.config['REDIS_URL'])
             
-            # 獲取活躍工作者
-            active_workers = inspect_obj.active()
+            # 檢查Celery binding keys（worker存在時會有這些keys）
+            celery_keys = redis_client.keys('_kombu.binding.celery*')
             
-            if active_workers and len(active_workers) > 0:
-                worker_count = len(active_workers)
+            # 方法2: 檢查進程（Docker環境中）
+            worker_detected = False
+            worker_count = 0
+            
+            try:
+                # 檢查是否有Celery相關的keys
+                if celery_keys:
+                    worker_detected = True
+                    worker_count = 1  # Docker環境中通常只有一個worker
+                    
+                # 額外檢查：如果有最近的任務處理記錄，說明worker在工作
+                recent_tasks = TranslationJob.query.filter(
+                    TranslationJob.updated_at >= datetime.utcnow() - timedelta(minutes=10),
+                    TranslationJob.status.in_(['PROCESSING', 'COMPLETED'])
+                ).count()
+                
+                if recent_tasks > 0:
+                    worker_detected = True
+                    worker_count = max(worker_count, 1)
+            except Exception:
+                pass
+            
+            if worker_detected:
                 status['services']['celery'] = {
                     'status': 'healthy',
                     'active_workers': worker_count,
-                    'workers': list(active_workers.keys())
+                    'message': 'Worker detected via Redis/Task activity'
                 }
             else:
-                # Celery 工作者沒有運行，但不一定表示系統異常
+                # Celery 工作者沒有檢測到
                 status['services']['celery'] = {
                     'status': 'warning',
-                    'message': 'No active Celery workers found',
+                    'message': 'No Celery worker activity detected',
                     'active_workers': 0
                 }
                 # 不設置整體系統為異常，只是警告
                 
         except Exception as e:
-            # Celery 連接失敗，但不一定表示系統異常
+            # Redis連接失敗或其他錯誤
             status['services']['celery'] = {
                 'status': 'warning', 
-                'message': f'Cannot connect to Celery workers: {str(e)[:100]}'
+                'message': f'Cannot check Celery status: {str(e)[:100]}',
+                'active_workers': 0
             }
             # 不設置整體系統為異常，只是警告
             
@@ -533,21 +572,24 @@ def get_system_metrics():
         from datetime import datetime, timedelta
         from app import db
         
-        # 統計任務狀態
+        # 統計任務狀態（排除軟刪除的記錄，反映當前實際狀態）
         job_stats = db.session.query(
             TranslationJob.status,
             func.count(TranslationJob.id)
+        ).filter(
+            TranslationJob.deleted_at.is_(None)
         ).group_by(TranslationJob.status).all()
         
         job_counts = {status: count for status, count in job_stats}
         
-        # 最近24小時的統計
+        # 最近24小時的統計（排除軟刪除的記錄）
         yesterday = datetime.utcnow() - timedelta(days=1)
         recent_jobs = db.session.query(
             TranslationJob.status,
             func.count(TranslationJob.id)
         ).filter(
-            TranslationJob.created_at >= yesterday
+            TranslationJob.created_at >= yesterday,
+            TranslationJob.deleted_at.is_(None)
         ).group_by(TranslationJob.status).all()
         
         recent_counts = {status: count for status, count in recent_jobs}
@@ -918,4 +960,112 @@ def generate_jobs_report(start_date, end_date):
     
     return {
         '任務清單': jobs_df
-    }
+    }
+
+
+@admin_bp.route('/jobs/<job_uuid>/cancel', methods=['POST'])
+@admin_required
+def admin_cancel_job(job_uuid):
+    """管理員取消任務"""
+    try:
+        from app import db
+        
+        job = TranslationJob.query.filter_by(job_uuid=job_uuid).first()
+        
+        if not job:
+            return jsonify(create_response(
+                success=False,
+                error='NOT_FOUND',
+                message='任務不存在'
+            )), 404
+            
+        # 只能取消等待中或處理中的任務
+        if job.status not in ['PENDING', 'PROCESSING']:
+            return jsonify(create_response(
+                success=False,
+                error='CANNOT_CANCEL',
+                message='只能取消等待中或處理中的任務'
+            )), 400
+            
+        # 如果任務正在處理中，嘗試撤銷 Celery 任務
+        if job.status == 'PROCESSING':
+            try:
+                from app.services.celery_service import revoke_task
+                revoke_task(job_uuid)
+                logger.info(f"Admin {g.current_user.username} revoked Celery task for job {job_uuid}")
+            except Exception as e:
+                logger.warning(f"Failed to revoke Celery task {job_uuid}: {e}")
+                # 即使撤銷失敗，也繼續標記任務為失敗
+            
+        # 更新任務狀態
+        job.status = 'FAILED'
+        job.error_message = f'管理員 {g.current_user.username} 取消了任務'
+        job.updated_at = datetime.utcnow()
+        
+        db.session.commit()
+        
+        logger.info(f"Admin {g.current_user.username} cancelled job {job_uuid}")
+        
+        return jsonify(create_response(
+            success=True,
+            data={
+                'job_uuid': job_uuid,
+                'status': job.status,
+                'message': '任務已取消'
+            }
+        ))
+        
+    except Exception as e:
+        logger.error(f"Error cancelling job {job_uuid}: {e}", exc_info=True)
+        return jsonify(create_response(
+            success=False,
+            error='INTERNAL_ERROR',
+            message=str(e)
+        )), 500
+
+
+@admin_bp.route('/jobs/<job_uuid>', methods=['DELETE'])
+@admin_required
+def admin_delete_job(job_uuid):
+    """管理員刪除任務（軟刪除）"""
+    try:
+        from app import db
+        
+        job = TranslationJob.query.filter_by(job_uuid=job_uuid).first()
+        
+        if not job:
+            return jsonify(create_response(
+                success=False,
+                error='NOT_FOUND',
+                message='任務不存在'
+            )), 404
+            
+        # 如果任務正在處理中，先嘗試撤銷 Celery 任務
+        if job.status == 'PROCESSING':
+            try:
+                from app.services.celery_service import revoke_task
+                revoke_task(job_uuid)
+                logger.info(f"Admin {g.current_user.username} revoked Celery task before deletion for job {job_uuid}")
+            except Exception as e:
+                logger.warning(f"Failed to revoke Celery task {job_uuid} before deletion: {e}")
+                
+        # 軟刪除資料庫記錄（保留數據供報表使用）
+        job.soft_delete()
+        
+        logger.info(f"Admin {g.current_user.username} soft deleted job {job_uuid}")
+        
+        return jsonify(create_response(
+            success=True,
+            data={
+                'job_uuid': job_uuid,
+                'message': '任務已刪除'
+            }
+        ))
+        
+    except Exception as e:
+        logger.error(f"Error deleting job {job_uuid}: {e}", exc_info=True)
+        return jsonify(create_response(
+            success=False,
+            error='INTERNAL_ERROR',
+            message=str(e)
+        )), 500

app/api/health.py

@@ -32,7 +32,8 @@ def health_check():
         # 資料庫檢查
         try:
             from app import db
-            db.session.execute('SELECT 1')
+            from sqlalchemy import text
+            db.session.execute(text('SELECT 1'))
             status['services']['database'] = {'status': 'healthy'}
         except Exception as e:
             status['services']['database'] = {

app/api/jobs.py

@@ -40,8 +40,8 @@ def get_user_jobs():
         # 驗證分頁參數
         page, per_page = validate_pagination(page, per_page)
         
-        # 建立查詢
-        query = TranslationJob.query.filter_by(user_id=g.current_user_id)
+        # 建立查詢（排除軟刪除的記錄）
+        query = TranslationJob.query.filter_by(user_id=g.current_user_id).filter(TranslationJob.deleted_at.is_(None))
         
         # 狀態篩選
         if status and status != 'all':
@@ -118,8 +118,8 @@ def get_job_detail(job_uuid):
         # 驗證 UUID 格式
         validate_job_uuid(job_uuid)
         
-        # 取得任務
-        job = TranslationJob.query.filter_by(job_uuid=job_uuid).first()
+        # 取得任務（排除軟刪除的記錄）
+        job = TranslationJob.query.filter_by(job_uuid=job_uuid).filter(TranslationJob.deleted_at.is_(None)).first()
         
         if not job:
             return jsonify(create_response(
@@ -194,8 +194,8 @@ def retry_job(job_uuid):
         # 驗證 UUID 格式
         validate_job_uuid(job_uuid)
         
-        # 取得任務
-        job = TranslationJob.query.filter_by(job_uuid=job_uuid).first()
+        # 取得任務（排除軟刪除的記錄）
+        job = TranslationJob.query.filter_by(job_uuid=job_uuid).filter(TranslationJob.deleted_at.is_(None)).first()
         
         if not job:
             return jsonify(create_response(
@@ -373,13 +373,13 @@ def get_queue_status():
 @jobs_bp.route('/<job_uuid>/cancel', methods=['POST'])
 @jwt_login_required
 def cancel_job(job_uuid):
-    """取消任務（僅限 PENDING 狀態）"""
+    """取消任務（支援 PENDING 和 PROCESSING 狀態）"""
     try:
         # 驗證 UUID 格式
         validate_job_uuid(job_uuid)
         
-        # 取得任務
-        job = TranslationJob.query.filter_by(job_uuid=job_uuid).first()
+        # 取得任務（排除軟刪除的記錄）
+        job = TranslationJob.query.filter_by(job_uuid=job_uuid).filter(TranslationJob.deleted_at.is_(None)).first()
         
         if not job:
             return jsonify(create_response(
@@ -396,16 +396,28 @@ def cancel_job(job_uuid):
                 message='無權限操作此任務'
             )), 403
         
-        # 只能取消等待中的任務
-        if job.status != 'PENDING':
+        # 只能取消等待中或處理中的任務
+        if job.status not in ['PENDING', 'PROCESSING']:
             return jsonify(create_response(
                 success=False,
                 error='CANNOT_CANCEL',
-                message='只能取消等待中的任務'
+                message='只能取消等待中或處理中的任務'
             )), 400
         
+        # 如果是處理中的任務，需要中斷 Celery 任務
+        if job.status == 'PROCESSING':
+            try:
+                from app.services.celery_service import revoke_task
+                # 嘗試撤銷 Celery 任務
+                revoke_task(job.job_uuid)
+                logger.info(f"Celery task revoked for job: {job.job_uuid}")
+            except Exception as celery_error:
+                logger.warning(f"Failed to revoke Celery task for job {job.job_uuid}: {celery_error}")
+                # 即使撤銷失敗也繼續取消任務，因為用戶請求取消
+        
         # 更新任務狀態為失敗（取消）
-        job.update_status('FAILED', error_message='使用者取消任務')
+        cancel_message = f'使用者取消任務 (原狀態: {job.status})'
+        job.update_status('FAILED', error_message=cancel_message)
         
         # 記錄取消日誌
         SystemLog.info(
@@ -469,13 +481,16 @@ def delete_job(job_uuid):
                 message='無權限操作此任務'
             )), 403
         
-        # 檢查任務狀態 - 不能刪除正在處理中的任務
+        # 如果是處理中的任務，先嘗試中斷 Celery 任務
         if job.status == 'PROCESSING':
-            return jsonify(create_response(
-                success=False,
-                error='CANNOT_DELETE',
-                message='無法刪除正在處理中的任務'
-            )), 400
+            try:
+                from app.services.celery_service import revoke_task
+                # 嘗試撤銷 Celery 任務
+                revoke_task(job.job_uuid)
+                logger.info(f"Celery task revoked before deletion for job: {job.job_uuid}")
+            except Exception as celery_error:
+                logger.warning(f"Failed to revoke Celery task before deletion for job {job.job_uuid}: {celery_error}")
+                # 即使撤銷失敗也繼續刪除任務，因為用戶要求刪除
         
         # 刪除任務相關檔案
         import os
@@ -506,11 +521,10 @@ def delete_job(job_uuid):
         
         from app import db
         
-        # 刪除資料庫記錄
-        db.session.delete(job)
-        db.session.commit()
+        # 軟刪除資料庫記錄（保留數據供報表使用）
+        job.soft_delete()
         
-        logger.info(f"Job deleted by user: {job_uuid}")
+        logger.info(f"Job soft deleted by user: {job_uuid}")
         
         return jsonify(create_response(
             success=True,

app/models/job.py

@@ -49,6 +49,7 @@ class TranslationJob(db.Model):
         onupdate=func.now(), 
         comment='更新時間'
     )
+    deleted_at = db.Column(db.DateTime, comment='軟刪除時間')
     
     # 關聯關係
     files = db.relationship('JobFile', backref='job', lazy='dynamic', cascade='all, delete-orphan')
@@ -84,7 +85,8 @@ class TranslationJob(db.Model):
             'processing_started_at': format_taiwan_time(self.processing_started_at, "%Y-%m-%d %H:%M:%S") if self.processing_started_at else None,
             'completed_at': format_taiwan_time(self.completed_at, "%Y-%m-%d %H:%M:%S") if self.completed_at else None,
             'created_at': format_taiwan_time(self.created_at, "%Y-%m-%d %H:%M:%S") if self.created_at else None,
-            'updated_at': format_taiwan_time(self.updated_at, "%Y-%m-%d %H:%M:%S") if self.updated_at else None
+            'updated_at': format_taiwan_time(self.updated_at, "%Y-%m-%d %H:%M:%S") if self.updated_at else None,
+            'deleted_at': format_taiwan_time(self.deleted_at, "%Y-%m-%d %H:%M:%S") if self.deleted_at else None
         }
         
         if include_files:
@@ -156,15 +158,32 @@ class TranslationJob(db.Model):
         self.updated_at = datetime.utcnow()
         db.session.commit()
     
+    def soft_delete(self):
+        """軟刪除任務（保留資料供報表使用）"""
+        self.deleted_at = datetime.utcnow()
+        self.updated_at = datetime.utcnow()
+        db.session.commit()
+    
+    def restore(self):
+        """恢復已刪除的任務"""
+        self.deleted_at = None
+        self.updated_at = datetime.utcnow()
+        db.session.commit()
+    
+    def is_deleted(self):
+        """檢查任務是否已被軟刪除"""
+        return self.deleted_at is not None
+    
     @classmethod
     def get_queue_position(cls, job_uuid):
         """取得任務在佇列中的位置"""
-        job = cls.query.filter_by(job_uuid=job_uuid).first()
+        job = cls.query.filter_by(job_uuid=job_uuid, deleted_at=None).first()
         if not job:
             return None
         
         position = cls.query.filter(
             cls.status == 'PENDING',
+            cls.deleted_at.is_(None),
             cls.created_at < job.created_at
         ).count()
         
@@ -173,18 +192,22 @@ class TranslationJob(db.Model):
     @classmethod
     def get_pending_jobs(cls):
         """取得所有等待處理的任務"""
-        return cls.query.filter_by(status='PENDING').order_by(cls.created_at.asc()).all()
+        return cls.query.filter_by(status='PENDING', deleted_at=None).order_by(cls.created_at.asc()).all()
     
     @classmethod
     def get_processing_jobs(cls):
         """取得所有處理中的任務"""
-        return cls.query.filter_by(status='PROCESSING').all()
+        return cls.query.filter_by(status='PROCESSING', deleted_at=None).all()
     
     @classmethod
-    def get_user_jobs(cls, user_id, status=None, limit=None, offset=None):
+    def get_user_jobs(cls, user_id, status=None, limit=None, offset=None, include_deleted=False):
         """取得使用者的任務列表"""
         query = cls.query.filter_by(user_id=user_id)
         
+        # 預設排除軟刪除的記錄，除非明確要求包含
+        if not include_deleted:
+            query = query.filter(cls.deleted_at.is_(None))
+        
         if status and status != 'all':
             query = query.filter_by(status=status.upper())
         
@@ -198,10 +221,14 @@ class TranslationJob(db.Model):
         return query.all()
     
     @classmethod
-    def get_statistics(cls, user_id=None, start_date=None, end_date=None):
-        """取得統計資料"""
+    def get_statistics(cls, user_id=None, start_date=None, end_date=None, include_deleted=True):
+        """取得統計資料（預設包含所有記錄以確保報表完整性）"""
         query = cls.query
         
+        # 報表統計預設包含已刪除記錄以確保數據完整性
+        if not include_deleted:
+            query = query.filter(cls.deleted_at.is_(None))
+        
         if user_id:
             query = query.filter_by(user_id=user_id)
         

app/services/celery_service.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Celery任務管理服務
+
+Author: PANJIT IT Team
+Created: 2025-09-04
+"""
+
+from celery import Celery
+from app.utils.logger import get_logger
+import os
+
+logger = get_logger(__name__)
+
+
+def get_celery_app():
+    """取得Celery應用實例"""
+    try:
+        from celery_app import app as celery_app
+        return celery_app
+    except ImportError:
+        # 如果無法導入，創建一個簡單的Celery實例
+        broker_url = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
+        celery_app = Celery('translation_worker', broker=broker_url)
+        return celery_app
+
+
+def revoke_task(job_uuid):
+    """
+    撤銷指定任務的Celery任務
+    
+    Args:
+        job_uuid (str): 任務UUID
+        
+    Returns:
+        bool: 撤銷是否成功
+    """
+    try:
+        celery_app = get_celery_app()
+        
+        # Celery任務ID通常與job_uuid相同或相關
+        task_id = f"translate_document_{job_uuid}"
+        
+        # 嘗試撤銷任務
+        celery_app.control.revoke(task_id, terminate=True, signal='SIGKILL')
+        
+        logger.info(f"Successfully revoked Celery task: {task_id}")
+        return True
+        
+    except Exception as e:
+        logger.error(f"Failed to revoke Celery task for job {job_uuid}: {str(e)}")
+        return False
+
+
+def get_active_tasks():
+    """
+    取得當前活躍的Celery任務
+    
+    Returns:
+        list: 活躍任務列表
+    """
+    try:
+        celery_app = get_celery_app()
+        
+        # 取得活躍任務
+        inspect = celery_app.control.inspect()
+        active_tasks = inspect.active()
+        
+        if active_tasks:
+            return active_tasks
+        else:
+            return {}
+            
+    except Exception as e:
+        logger.error(f"Failed to get active tasks: {str(e)}")
+        return {}
+
+
+def is_task_active(job_uuid):
+    """
+    檢查指定任務是否在Celery中活躍
+    
+    Args:
+        job_uuid (str): 任務UUID
+        
+    Returns:
+        bool: 任務是否活躍
+    """
+    try:
+        active_tasks = get_active_tasks()
+        task_id = f"translate_document_{job_uuid}"
+        
+        # 檢查所有worker的活躍任務
+        for worker, tasks in active_tasks.items():
+            for task in tasks:
+                if task.get('id') == task_id:
+                    return True
+        
+        return False
+        
+    except Exception as e:
+        logger.error(f"Failed to check if task is active for job {job_uuid}: {str(e)}")
+        return False
+
+
+def cleanup_stale_tasks():
+    """
+    清理卡住的Celery任務
+    
+    Returns:
+        int: 清理的任務數量
+    """
+    try:
+        from app.models.job import TranslationJob
+        from datetime import datetime, timedelta
+        
+        # 找出超過30分鐘仍在處理中的任務
+        stale_threshold = datetime.utcnow() - timedelta(minutes=30)
+        stale_jobs = TranslationJob.query.filter(
+            TranslationJob.status == 'PROCESSING',
+            TranslationJob.processing_started_at < stale_threshold
+        ).all()
+        
+        cleanup_count = 0
+        for job in stale_jobs:
+            if not is_task_active(job.job_uuid):
+                # 任務不在Celery中活躍，標記為失敗
+                job.update_status('FAILED', error_message='任務處理超時，已自動取消')
+                cleanup_count += 1
+                logger.info(f"Cleaned up stale job: {job.job_uuid}")
+        
+        return cleanup_count
+        
+    except Exception as e:
+        logger.error(f"Failed to cleanup stale tasks: {str(e)}")
+        return 0