egg/PROJECT-CONTORL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement security, error resilience, and query optimization proposals

Browse Source 
Security Validation (enhance-security-validation):
- JWT secret validation with entropy checking and pattern detection
- CSRF protection middleware with token generation/validation
- Frontend CSRF token auto-injection for DELETE/PUT/PATCH requests
- MIME type validation with magic bytes detection for file uploads

Error Resilience (add-error-resilience):
- React ErrorBoundary component with fallback UI and retry functionality
- ErrorBoundaryWithI18n wrapper for internationalization support
- Page-level and section-level error boundaries in App.tsx

Query Performance (optimize-query-performance):
- Query monitoring utility with threshold warnings
- N+1 query fixes using joinedload/selectinload
- Optimized project members, tasks, and subtasks endpoints

Bug Fixes:
- WebSocket session management (P0): Return primitives instead of ORM objects
- LIKE query injection (P1): Escape special characters in search queries

Tests: 543 backend tests, 56 frontend tests passing

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
beabigegg
2026-01-11 18:41:19 +08:00
parent 2cb591ef23
commit 679b89ae4c
41 changed files with 3673 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
backend/app/services/file_storage_service.py
View File

@@ -286,11 +286,15 @@ class FileStorageService:
return filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
@staticmethod
def validate_file(file: UploadFile) -> Tuple[str, str]:
def validate_file(file: UploadFile, validate_mime: bool = True) -> Tuple[str, str]:
"""
Validate file size and type.
Validate file size, type, and optionally MIME content.
Returns (extension, mime_type) if valid.
Raises HTTPException if invalid.
Args:
file: The uploaded file
validate_mime: If True, validate MIME type using magic bytes detection
"""
# Check file size
file.file.seek(0, 2) # Seek to end
@@ -323,7 +327,35 @@ class FileStorageService:
detail=f"File type '.{extension}' is not supported"
)
mime_type = file.content_type or "application/octet-stream"
# Validate MIME type using magic bytes detection
if validate_mime:
from app.services.mime_validation_service import mime_validation_service
# Read first 16 bytes for magic detection (enough for most signatures)
file_header = file.file.read(16)
file.file.seek(0) # Reset
is_valid, detected_mime, error_message = mime_validation_service.validate_file_content(
file_content=file_header,
declared_extension=extension,
declared_mime_type=file.content_type
)
if not is_valid:
logger.warning(
"MIME validation failed for file '%s': %s (detected: %s)",
file.filename, error_message, detected_mime
)
raise HTTPException(
status_code=400,
detail=error_message or "File type validation failed"
)
# Use detected MIME type if available, otherwise fall back to declared
mime_type = detected_mime if detected_mime else (file.content_type or "application/octet-stream")
else:
mime_type = file.content_type or "application/octet-stream"
return extension, mime_type
async def save_file(

314
backend/app/services/mime_validation_service.py Normal file
View File

@@ -0,0 +1,314 @@
"""
MIME Type Validation Service using Magic Bytes Detection.
This module provides file content type validation by examining
the actual file content (magic bytes) rather than trusting
the file extension or Content-Type header.
"""
import logging
from typing import Optional, Tuple, Dict, Set, BinaryIO
from io import BytesIO
logger = logging.getLogger(__name__)
class MimeValidationError(Exception):
"""Raised when MIME type validation fails."""
pass
class FileMismatchError(MimeValidationError):
"""Raised when file extension doesn't match actual content type."""
pass
class UnsupportedMimeError(MimeValidationError):
"""Raised when file has an unsupported MIME type."""
pass
# Magic bytes signatures for common file types
# Format: { bytes_pattern: (mime_type, extensions) }
MAGIC_SIGNATURES: Dict[bytes, Tuple[str, Set[str]]] = {
# Images
b'\xFF\xD8\xFF': ('image/jpeg', {'jpg', 'jpeg', 'jpe'}),
b'\x89PNG\r\n\x1a\n': ('image/png', {'png'}),
b'GIF87a': ('image/gif', {'gif'}),
b'GIF89a': ('image/gif', {'gif'}),
b'RIFF': ('image/webp', {'webp'}), # WebP starts with RIFF, then WEBP
b'BM': ('image/bmp', {'bmp'}),
# PDF
b'%PDF': ('application/pdf', {'pdf'}),
# Microsoft Office (Modern formats - ZIP-based)
b'PK\x03\x04': ('application/zip', {'zip', 'docx', 'xlsx', 'pptx', 'odt', 'ods', 'odp', 'jar'}),
# Microsoft Office (Legacy formats - Compound Document)
b'\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1': ('application/msword', {'doc', 'xls', 'ppt', 'msi'}),
# Archives
b'\x1f\x8b': ('application/gzip', {'gz', 'tgz'}),
b'\x42\x5a\x68': ('application/x-bzip2', {'bz2'}),
b'\x37\x7A\xBC\xAF\x27\x1C': ('application/x-7z-compressed', {'7z'}),
b'Rar!\x1a\x07': ('application/x-rar-compressed', {'rar'}),
# Text/Data formats - these are harder to detect, usually fallback to extension
b'<?xml': ('application/xml', {'xml', 'svg'}),
b'{': ('application/json', {'json'}), # JSON typically starts with { or [
b'[': ('application/json', {'json'}),
# Executables (dangerous - should be blocked)
b'MZ': ('application/x-executable', {'exe', 'dll', 'com', 'scr'}),
b'\x7fELF': ('application/x-executable', {'elf', 'so', 'bin'}),
}
# Map extensions to expected MIME types
EXTENSION_TO_MIME: Dict[str, Set[str]] = {
# Images
'jpg': {'image/jpeg'},
'jpeg': {'image/jpeg'},
'jpe': {'image/jpeg'},
'png': {'image/png'},
'gif': {'image/gif'},
'bmp': {'image/bmp'},
'webp': {'image/webp'},
'svg': {'image/svg+xml', 'application/xml', 'text/xml'},
# Documents
'pdf': {'application/pdf'},
'doc': {'application/msword'},
'docx': {'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/zip'},
'xls': {'application/vnd.ms-excel', 'application/msword'},
'xlsx': {'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/zip'},
'ppt': {'application/vnd.ms-powerpoint', 'application/msword'},
'pptx': {'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'application/zip'},
# Text
'txt': {'text/plain'},
'csv': {'text/csv', 'text/plain'},
'json': {'application/json', 'text/plain'},
'xml': {'application/xml', 'text/xml', 'text/plain'},
'yaml': {'application/yaml', 'text/plain'},
'yml': {'application/yaml', 'text/plain'},
# Archives
'zip': {'application/zip'},
'rar': {'application/x-rar-compressed'},
'7z': {'application/x-7z-compressed'},
'tar': {'application/x-tar'},
'gz': {'application/gzip'},
}
# MIME types that should always be blocked (dangerous executables)
BLOCKED_MIME_TYPES: Set[str] = {
'application/x-executable',
'application/x-msdownload',
'application/x-msdos-program',
'application/x-sh',
'application/x-csh',
'application/x-dosexec',
}
# Configurable allowed MIME type categories
ALLOWED_MIME_CATEGORIES: Dict[str, Set[str]] = {
'images': {
'image/jpeg', 'image/png', 'image/gif', 'image/bmp', 'image/webp', 'image/svg+xml'
},
'documents': {
'application/pdf',
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.ms-excel',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.ms-powerpoint',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'text/plain', 'text/csv',
},
'archives': {
'application/zip', 'application/x-rar-compressed',
'application/x-7z-compressed', 'application/gzip',
'application/x-tar',
},
'data': {
'application/json', 'application/xml', 'text/xml',
'application/yaml', 'text/plain',
},
}
class MimeValidationService:
"""Service for validating file MIME types using magic bytes."""
def __init__(
self,
allowed_categories: Optional[Set[str]] = None,
bypass_for_trusted: bool = False
):
"""
Initialize the MIME validation service.
Args:
allowed_categories: Set of allowed MIME categories ('images', 'documents', etc.)
If None, all categories are allowed.
bypass_for_trusted: If True, validation can be bypassed for trusted sources.
"""
self.bypass_for_trusted = bypass_for_trusted
# Build set of allowed MIME types
if allowed_categories is None:
self.allowed_mime_types = set()
for category_mimes in ALLOWED_MIME_CATEGORIES.values():
self.allowed_mime_types.update(category_mimes)
else:
self.allowed_mime_types = set()
for category in allowed_categories:
if category in ALLOWED_MIME_CATEGORIES:
self.allowed_mime_types.update(ALLOWED_MIME_CATEGORIES[category])
def detect_mime_type(self, file_content: bytes) -> Optional[str]:
"""
Detect MIME type from file content using magic bytes.
Args:
file_content: The raw file bytes (at least first 16 bytes needed)
Returns:
Detected MIME type or None if unknown
"""
if len(file_content) < 2:
return None
# Check each magic signature
for magic_bytes, (mime_type, _) in MAGIC_SIGNATURES.items():
if file_content.startswith(magic_bytes):
# Special case for WebP: check for WEBP after RIFF
if magic_bytes == b'RIFF' and len(file_content) >= 12:
if file_content[8:12] == b'WEBP':
return 'image/webp'
else:
continue # Not WebP, might be something else
return mime_type
return None
def validate_file_content(
self,
file_content: bytes,
declared_extension: str,
declared_mime_type: Optional[str] = None,
trusted_source: bool = False
) -> Tuple[bool, str, Optional[str]]:
"""
Validate file content against declared extension and MIME type.
Args:
file_content: The raw file bytes
declared_extension: The file extension (without dot)
declared_mime_type: The Content-Type header value (optional)
trusted_source: If True and bypass_for_trusted is enabled, skip validation
Returns:
Tuple of (is_valid, detected_mime_type, error_message)
"""
# Bypass for trusted sources if configured
if trusted_source and self.bypass_for_trusted:
logger.debug("MIME validation bypassed for trusted source")
return True, declared_mime_type or 'application/octet-stream', None
# Detect actual MIME type
detected_mime = self.detect_mime_type(file_content)
ext_lower = declared_extension.lower()
# Check if detected MIME is blocked (dangerous executable)
if detected_mime in BLOCKED_MIME_TYPES:
logger.warning(
"Blocked dangerous file type detected: %s (claimed extension: %s)",
detected_mime, ext_lower
)
return False, detected_mime, "File type not allowed for security reasons"
# If we couldn't detect the MIME type, fall back to extension-based check
if detected_mime is None:
# For text/data files, detection is unreliable
# Trust the extension if it's in our allowed list
if ext_lower in EXTENSION_TO_MIME:
expected_mimes = EXTENSION_TO_MIME[ext_lower]
# Check if any expected MIME is in allowed set
if expected_mimes & self.allowed_mime_types:
logger.debug(
"MIME detection inconclusive for extension %s, allowing based on extension",
ext_lower
)
# Return the first expected MIME type
return True, next(iter(expected_mimes)), None
# Unknown extension or MIME type
logger.warning(
"Could not detect MIME type for file with extension: %s",
ext_lower
)
return True, 'application/octet-stream', None
# Check if detected MIME is in allowed set
if detected_mime not in self.allowed_mime_types:
logger.warning(
"Unsupported MIME type detected: %s (extension: %s)",
detected_mime, ext_lower
)
return False, detected_mime, f"Unsupported file type: {detected_mime}"
# Verify extension matches detected MIME type
if ext_lower in EXTENSION_TO_MIME:
expected_mimes = EXTENSION_TO_MIME[ext_lower]
# Special handling for ZIP-based formats (docx, xlsx, pptx)
if detected_mime == 'application/zip' and ext_lower in {'docx', 'xlsx', 'pptx', 'odt', 'ods', 'odp'}:
# These are valid - ZIP container with specific extension
return True, detected_mime, None
# Check if detected MIME matches any expected MIME for this extension
if detected_mime not in expected_mimes:
# Mismatch detected!
logger.warning(
"File type mismatch: extension '%s' but detected '%s'",
ext_lower, detected_mime
)
return False, detected_mime, f"File type mismatch: extension indicates {ext_lower} but content is {detected_mime}"
return True, detected_mime, None
async def validate_upload_file(
self,
file_content: bytes,
filename: str,
content_type: Optional[str] = None,
trusted_source: bool = False
) -> Tuple[bool, str, Optional[str]]:
"""
Validate an uploaded file.
Args:
file_content: The raw file bytes
filename: The uploaded filename
content_type: The Content-Type header value
trusted_source: If True and bypass is enabled, skip validation
Returns:
Tuple of (is_valid, detected_mime_type, error_message)
"""
# Extract extension
extension = filename.rsplit('.', 1)[-1] if '.' in filename else ''
return self.validate_file_content(
file_content=file_content,
declared_extension=extension,
declared_mime_type=content_type,
trusted_source=trusted_source
)
# Singleton instance with default configuration
mime_validation_service = MimeValidationService()

6
backend/app/services/notification_service.py
View File

@@ -14,6 +14,7 @@ from sqlalchemy import event
from app.models import User, Notification, Task, Comment, Mention
from app.core.redis_pubsub import publish_notification as redis_publish, get_channel_name
from app.core.redis import get_redis_sync
from app.core.database import escape_like
logger = logging.getLogger(__name__)
@@ -427,9 +428,12 @@ class NotificationService:
# Find users by email or name
for username in mentioned_usernames:
# Escape special LIKE characters to prevent injection
escaped_username = escape_like(username)
# Try to find user by email first
user = db.query(User).filter(
(User.email.ilike(f"{username}%")) | (User.name.ilike(f"%{username}%"))
(User.email.ilike(f"{escaped_username}%", escape="\\")) |
(User.name.ilike(f"%{escaped_username}%", escape="\\"))
).first()
if user and user.id != author.id: