Files
OCR/backend/app/core/config.py
egg 8b9a364452 feat: add GPU optimization and fix TableData consistency
GPU Optimization (Section 3.1):
- Add comprehensive memory management for RTX 4060 8GB
- Enable all recognition features (chart, formula, table, seal, text)
- Implement model cache with auto-unload for idle models
- Add memory monitoring and warning system

Bug Fix (Section 3.3):
- Fix TableData field inconsistency: 'columns' -> 'cols'
- Remove invalid 'html' and 'extracted_text' parameters
- Add proper TableCell conversion in _convert_table_data

Documentation:
- Add Future Improvements section for batch processing enhancement

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-19 09:17:27 +08:00

187 lines
7.7 KiB
Python

"""
Tool_OCR - Configuration Management
Loads environment variables and provides centralized configuration
"""
from typing import List
from pydantic_settings import BaseSettings
from pydantic import Field
from pathlib import Path
class Settings(BaseSettings):
"""Application settings loaded from environment variables"""
# ===== Database Configuration =====
mysql_host: str = Field(default="mysql.theaken.com")
mysql_port: int = Field(default=33306)
mysql_user: str = Field(default="A060")
mysql_password: str = Field(default="")
mysql_database: str = Field(default="db_A060")
@property
def database_url(self) -> str:
"""Construct SQLAlchemy database URL"""
return (
f"mysql+pymysql://{self.mysql_user}:{self.mysql_password}"
f"@{self.mysql_host}:{self.mysql_port}/{self.mysql_database}"
)
# ===== Application Configuration =====
backend_port: int = Field(default=12010)
frontend_port: int = Field(default=12011)
secret_key: str = Field(default="your-secret-key-change-this")
algorithm: str = Field(default="HS256")
access_token_expire_minutes: int = Field(default=1440) # 24 hours
# ===== External Authentication Configuration =====
external_auth_api_url: str = Field(default="https://pj-auth-api.vercel.app")
external_auth_endpoint: str = Field(default="/api/auth/login")
external_auth_timeout: int = Field(default=30)
token_refresh_buffer: int = Field(default=300) # Refresh tokens 5 minutes before expiry
@property
def external_auth_full_url(self) -> str:
"""Construct full external authentication URL"""
return f"{self.external_auth_api_url.rstrip('/')}{self.external_auth_endpoint}"
# ===== Task Management Configuration =====
database_table_prefix: str = Field(default="tool_ocr_")
enable_task_history: bool = Field(default=True)
task_retention_days: int = Field(default=30)
max_tasks_per_user: int = Field(default=1000)
# ===== OCR Configuration =====
paddleocr_model_dir: str = Field(default="./models/paddleocr")
ocr_languages: str = Field(default="ch,en,japan,korean")
ocr_confidence_threshold: float = Field(default=0.5)
max_ocr_workers: int = Field(default=4)
@property
def ocr_languages_list(self) -> List[str]:
"""Get OCR languages as list"""
return [lang.strip() for lang in self.ocr_languages.split(",")]
# ===== GPU Acceleration Configuration =====
# Basic GPU settings
force_cpu_mode: bool = Field(default=False)
gpu_memory_fraction: float = Field(default=0.7) # Optimized for RTX 4060 8GB
gpu_device_id: int = Field(default=0)
# Memory management for RTX 4060 8GB
gpu_memory_limit_mb: int = Field(default=6144) # 6GB max for models (leave 2GB buffer)
gpu_memory_reserve_mb: int = Field(default=512) # Reserve for CUDA overhead
enable_memory_optimization: bool = Field(default=True)
# Model loading and caching
enable_lazy_model_loading: bool = Field(default=True) # Load models on demand
enable_model_cache: bool = Field(default=True)
model_cache_limit_mb: int = Field(default=4096) # Max 4GB for cached models
auto_unload_unused_models: bool = Field(default=True) # Unload unused language models
model_idle_timeout_seconds: int = Field(default=300) # Unload after 5 min idle
# Batch processing configuration
enable_batch_processing: bool = Field(default=True)
inference_batch_size: int = Field(default=1) # Conservative for 8GB VRAM
max_concurrent_pages: int = Field(default=2) # Process 2 pages concurrently
# PP-StructureV3 optimization
enable_chart_recognition: bool = Field(default=True) # Chart/diagram recognition
enable_formula_recognition: bool = Field(default=True) # Math formula recognition
enable_table_recognition: bool = Field(default=True) # Table structure recognition
enable_seal_recognition: bool = Field(default=True) # Seal/stamp recognition
enable_text_recognition: bool = Field(default=True) # General text recognition
layout_detection_threshold: float = Field(default=0.5)
# Performance tuning
use_fp16_inference: bool = Field(default=False) # Half-precision (if supported)
enable_cudnn_benchmark: bool = Field(default=True) # Optimize convolution algorithms
num_threads: int = Field(default=4) # CPU threads for preprocessing
# ===== File Upload Configuration =====
max_upload_size: int = Field(default=52428800) # 50MB
allowed_extensions: str = Field(default="png,jpg,jpeg,pdf,bmp,tiff,doc,docx,ppt,pptx")
upload_dir: str = Field(default="./uploads")
temp_dir: str = Field(default="./uploads/temp")
processed_dir: str = Field(default="./uploads/processed")
images_dir: str = Field(default="./uploads/images")
@property
def allowed_extensions_list(self) -> List[str]:
"""Get allowed extensions as list"""
return [ext.strip() for ext in self.allowed_extensions.split(",")]
# ===== Export Configuration =====
storage_dir: str = Field(default="./storage")
markdown_dir: str = Field(default="./storage/markdown")
json_dir: str = Field(default="./storage/json")
exports_dir: str = Field(default="./storage/exports")
result_dir: str = Field(default="./storage/results")
# ===== PDF Generation Configuration =====
pandoc_path: str = Field(default="/opt/homebrew/bin/pandoc")
font_dir: str = Field(default="/System/Library/Fonts")
pdf_page_size: str = Field(default="A4")
pdf_margin_top: int = Field(default=20)
pdf_margin_bottom: int = Field(default=20)
pdf_margin_left: int = Field(default=20)
pdf_margin_right: int = Field(default=20)
# ===== Layout-Preserving PDF Configuration =====
chinese_font_path: str = Field(default="./backend/fonts/NotoSansSC-Regular.ttf")
pdf_font_size_base: int = Field(default=12)
pdf_enable_bbox_debug: bool = Field(default=False) # Draw bounding boxes for debugging
# ===== Translation Configuration (Reserved) =====
enable_translation: bool = Field(default=False)
translation_engine: str = Field(default="offline")
argostranslate_models_dir: str = Field(default="./models/argostranslate")
# ===== Background Tasks Configuration =====
task_queue_type: str = Field(default="memory")
redis_url: str = Field(default="redis://localhost:6379/0")
# ===== CORS Configuration =====
cors_origins: str = Field(default="http://localhost:12011,http://127.0.0.1:12011")
@property
def cors_origins_list(self) -> List[str]:
"""Get CORS origins as list"""
return [origin.strip() for origin in self.cors_origins.split(",")]
# ===== Logging Configuration =====
log_level: str = Field(default="INFO")
log_file: str = Field(default="./logs/app.log")
class Config:
# Look for .env in project root (one level up from backend/)
env_file = str(Path(__file__).resolve().parent.parent.parent.parent / ".env")
env_file_encoding = "utf-8"
case_sensitive = False
def ensure_directories(self):
"""Create all necessary directories if they don't exist"""
dirs = [
self.upload_dir,
self.temp_dir,
self.processed_dir,
self.images_dir,
self.storage_dir,
self.markdown_dir,
self.json_dir,
self.exports_dir,
self.result_dir,
self.paddleocr_model_dir,
Path(self.log_file).parent,
]
if self.enable_translation and self.translation_engine == "offline":
dirs.append(self.argostranslate_models_dir)
for dir_path in dirs:
Path(dir_path).mkdir(parents=True, exist_ok=True)
# Global settings instance
settings = Settings()