From a275c30c0ee33a1d7239406ae98969d58d49bb76 Mon Sep 17 00:00:00 2001 From: egg Date: Tue, 3 Mar 2026 14:00:07 +0800 Subject: [PATCH] feat(reject-history): fix silent data loss by propagating partial failure metadata to frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chunk failures in BatchQueryEngine were silently discarded — `has_partial_failure` was tracked in Redis but never surfaced to the API response or frontend. Users could see incomplete data without any warning. This commit closes the gap end-to-end: Backend: - Track failed chunk time ranges (`failed_ranges`) in batch engine progress metadata - Add single retry for transient Oracle errors (timeout, connection) in `_execute_single_chunk` - Read `get_batch_progress()` after merge but before `redis_clear_batch()` cleanup - Inject `has_partial_failure`, `failed_chunk_count`, `failed_ranges` into API response meta - Persist partial failure flag to independent Redis key with TTL aligned to data storage layer - Add shared container-resolution policy module with wildcard/expansion guardrails - Refactor reason filter from single-value to multi-select (`reason` → `reasons`) Frontend: - Add client-side date range validation (730-day limit) before API submission - Display amber warning banner on partial failure with specific failed date ranges - Support generic fallback message for container-mode queries without date ranges - Update FilterPanel to support multi-select reason chips Specs & tests: - Create batch-query-resilience spec; update reject-history-api and reject-history-page specs - Add 7 new tests for retry, memory guard, failed ranges, partial failure propagation, TTL - Cross-service regression verified (hold, resource, job, msd — 411 tests pass) Co-Authored-By: Claude Opus 4.6 --- .env.example | 22 +- README.md | 9 + frontend/src/core/reject-history-filters.js | 57 +- frontend/src/reject-history/App.vue | 63 +- .../reject-history/components/FilterPanel.vue | 144 +- frontend/src/reject-history/style.css | 22 + .../.openspec.yaml | 2 + .../design.md | 80 + .../proposal.md | 34 + .../specs/batch-query-resilience/spec.md | 82 + .../specs/reject-history-api/spec.md | 36 + .../specs/reject-history-page/spec.md | 58 + .../tasks.md | 46 + openspec/specs/batch-query-resilience/spec.md | 86 + openspec/specs/reject-history-api/spec.md | 22 + openspec/specs/reject-history-page/spec.md | 57 + .../routes/reject_history_routes.py | 17 +- .../services/batch_query_engine.py | 162 +- .../services/container_resolution_policy.py | 152 + src/mes_dashboard/services/event_fetcher.py | 13 +- .../services/job_query_service.py | 2 +- .../services/mid_section_defect_service.py | 2 +- .../services/query_tool_service.py | 93 +- .../services/reject_dataset_cache.py | 2552 +++++++++-------- .../sql/reject_history/performance_daily.sql | 27 +- .../reject_history/performance_daily_lot.sql | 37 +- tests/test_batch_query_engine.py | 144 +- tests/test_container_resolution_policy.py | 73 + tests/test_event_fetcher.py | 57 + tests/test_job_query_engine.py | 2 +- tests/test_job_query_service.py | 2 +- tests/test_mid_section_defect_engine.py | 2 +- tests/test_query_tool_routes.py | 15 +- tests/test_query_tool_service.py | 63 +- tests/test_reject_dataset_cache.py | 253 +- 35 files changed, 3028 insertions(+), 1460 deletions(-) create mode 100644 openspec/changes/archive/2026-03-03-fix-silent-data-loss-reject-history/.openspec.yaml create mode 100644 openspec/changes/archive/2026-03-03-fix-silent-data-loss-reject-history/design.md create mode 100644 openspec/changes/archive/2026-03-03-fix-silent-data-loss-reject-history/proposal.md create mode 100644 openspec/changes/archive/2026-03-03-fix-silent-data-loss-reject-history/specs/batch-query-resilience/spec.md create mode 100644 openspec/changes/archive/2026-03-03-fix-silent-data-loss-reject-history/specs/reject-history-api/spec.md create mode 100644 openspec/changes/archive/2026-03-03-fix-silent-data-loss-reject-history/specs/reject-history-page/spec.md create mode 100644 openspec/changes/archive/2026-03-03-fix-silent-data-loss-reject-history/tasks.md create mode 100644 openspec/specs/batch-query-resilience/spec.md create mode 100644 src/mes_dashboard/services/container_resolution_policy.py create mode 100644 tests/test_container_resolution_policy.py diff --git a/.env.example b/.env.example index 0ddd57f..92f5907 100644 --- a/.env.example +++ b/.env.example @@ -59,6 +59,16 @@ QUERY_TOOL_MAX_CONTAINER_IDS=200 RESOURCE_DETAIL_DEFAULT_LIMIT=500 RESOURCE_DETAIL_MAX_LIMIT=500 +# Shared container-resolution guardrails +# 0 = disable raw input count cap (recommended: rely on expansion limits instead) +CONTAINER_RESOLVE_INPUT_MAX_VALUES=0 +# Wildcard pattern must include this many literal-prefix chars before %/_ (e.g., GA%) +CONTAINER_RESOLVE_PATTERN_MIN_PREFIX_LEN=4 +# Per-token expansion guard (avoid one wildcard exploding into too many container IDs) +CONTAINER_RESOLVE_MAX_EXPANSION_PER_TOKEN=2000 +# Total resolved container-ID guard for a single resolve request +CONTAINER_RESOLVE_MAX_CONTAINER_IDS=30000 + # Trust boundary for forwarded headers (safe default: false) # Direct-exposure deployment (no reverse proxy): keep this false TRUST_PROXY_HEADERS=false @@ -101,14 +111,14 @@ GUNICORN_WORKERS=2 GUNICORN_THREADS=4 # Worker timeout (seconds): should stay above DB/query-tool slow paths -GUNICORN_TIMEOUT=130 +GUNICORN_TIMEOUT=360 # Graceful shutdown timeout for worker reloads (seconds) -GUNICORN_GRACEFUL_TIMEOUT=60 +GUNICORN_GRACEFUL_TIMEOUT=300 # Worker recycle policy (set 0 to disable) -GUNICORN_MAX_REQUESTS=5000 -GUNICORN_MAX_REQUESTS_JITTER=500 +GUNICORN_MAX_REQUESTS=1200 +GUNICORN_MAX_REQUESTS_JITTER=300 # ============================================================ # Redis Configuration (for WIP cache) @@ -201,6 +211,8 @@ TRACE_EVENTS_MAX_WORKERS=2 # Max parallel workers for EventFetcher batch queries (per domain) # Recommend: 2 (peak concurrent slow queries = TRACE_EVENTS_MAX_WORKERS × this) EVENT_FETCHER_MAX_WORKERS=2 +# false = any failed batch raises error (avoid silent partial data) +EVENT_FETCHER_ALLOW_PARTIAL_RESULTS=false # Max parallel workers for forward pipeline WIP+rejects fetching FORWARD_PIPELINE_MAX_WORKERS=2 @@ -351,7 +363,7 @@ REJECT_ENGINE_SPOOL_CLEANUP_INTERVAL_SECONDS=300 REJECT_ENGINE_SPOOL_ORPHAN_GRACE_SECONDS=600 # Batch query engine thresholds -BATCH_QUERY_TIME_THRESHOLD_DAYS=60 +BATCH_QUERY_TIME_THRESHOLD_DAYS=10 BATCH_QUERY_ID_THRESHOLD=1000 BATCH_CHUNK_MAX_MEMORY_MB=256 diff --git a/README.md b/README.md index 8ab8360..9c801ae 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,15 @@ QUERY_TOOL_MAX_CONTAINER_IDS=200 RESOURCE_DETAIL_DEFAULT_LIMIT=500 RESOURCE_DETAIL_MAX_LIMIT=500 +# 共用解析防護（LOT/WAFER/工單） +CONTAINER_RESOLVE_INPUT_MAX_VALUES=0 # 0=不限制輸入筆數 +CONTAINER_RESOLVE_PATTERN_MIN_PREFIX_LEN=4 # 萬用字元前最少字首長度（例如 GA25%） +CONTAINER_RESOLVE_MAX_EXPANSION_PER_TOKEN=2000 +CONTAINER_RESOLVE_MAX_CONTAINER_IDS=30000 + +# EventFetcher 批次容錯策略 +EVENT_FETCHER_ALLOW_PARTIAL_RESULTS=false # false=任一批次失敗即整體失敗，避免靜默缺資料 + # 反向代理信任邊界（無反向代理時務必維持 false） TRUST_PROXY_HEADERS=false TRUSTED_PROXY_IPS=127.0.0.1 diff --git a/frontend/src/core/reject-history-filters.js b/frontend/src/core/reject-history-filters.js index 53eed69..01308d1 100644 --- a/frontend/src/core/reject-history-filters.js +++ b/frontend/src/core/reject-history-filters.js @@ -35,7 +35,7 @@ export function toRejectFilterSnapshot(input = {}) { endDate: normalizeText(input.endDate), workcenterGroups: normalizeArray(input.workcenterGroups), packages: normalizeArray(input.packages), - reason: normalizeText(input.reason), + reasons: normalizeArray(input.reasons), includeExcludedScrap: normalizeBoolean(input.includeExcludedScrap, false), excludeMaterialScrap: normalizeBoolean(input.excludeMaterialScrap, true), excludePbDiode: normalizeBoolean(input.excludePbDiode, true), @@ -77,7 +77,7 @@ export function pruneRejectFilterSelections(filters = {}, options = {}) { const removed = { workcenterGroups: [], packages: [], - reason: '', + reasons: [], }; if (hasWorkcenterOptions) { @@ -100,9 +100,14 @@ export function pruneRejectFilterSelections(filters = {}, options = {}) { }); } - if (next.reason && hasReasonOptions && !validReasons.has(next.reason)) { - removed.reason = next.reason; - next.reason = ''; + if (hasReasonOptions) { + next.reasons = next.reasons.filter((value) => { + if (validReasons.has(value)) { + return true; + } + removed.reasons.push(value); + return false; + }); } return { @@ -111,7 +116,7 @@ export function pruneRejectFilterSelections(filters = {}, options = {}) { removedCount: removed.workcenterGroups.length + removed.packages.length + - (removed.reason ? 1 : 0), + removed.reasons.length, }; } @@ -126,13 +131,13 @@ export function buildRejectOptionsRequestParams(filters = {}) { exclude_material_scrap: next.excludeMaterialScrap, exclude_pb_diode: next.excludePbDiode, }; - if (next.reason) { - params.reason = next.reason; + if (next.reasons.length > 0) { + params.reasons = next.reasons; } return params; } -export function buildRejectCommonQueryParams(filters = {}, { reason = '' } = {}) { +export function buildRejectCommonQueryParams(filters = {}, { reasons: extraReasons = [] } = {}) { const next = toRejectFilterSnapshot(filters); const params = { start_date: next.startDate, @@ -143,9 +148,9 @@ export function buildRejectCommonQueryParams(filters = {}, { reason = '' } = {}) exclude_material_scrap: next.excludeMaterialScrap, exclude_pb_diode: next.excludePbDiode, }; - const effectiveReason = normalizeText(reason) || next.reason; - if (effectiveReason) { - params.reasons = [effectiveReason]; + const merged = normalizeArray([...next.reasons, ...normalizeArray(extraReasons)]); + if (merged.length > 0) { + params.reasons = merged; } return params; } @@ -168,6 +173,30 @@ export function parseMultiLineInput(text) { return result; } +export function validateDateRange(startDate, endDate) { + const MAX_QUERY_DAYS = 730; + const start = normalizeText(startDate); + const end = normalizeText(endDate); + if (!start || !end) { + return '請先設定開始與結束日期'; + } + + const startDt = new Date(`${start}T00:00:00`); + const endDt = new Date(`${end}T00:00:00`); + if (Number.isNaN(startDt.getTime()) || Number.isNaN(endDt.getTime())) { + return '日期格式不正確'; + } + if (endDt < startDt) { + return '結束日期必須大於起始日期'; + } + const dayMs = 24 * 60 * 60 * 1000; + const days = Math.floor((endDt - startDt) / dayMs) + 1; + if (days > MAX_QUERY_DAYS) { + return '查詢範圍不可超過 730 天（約兩年）'; + } + return ''; +} + export function buildViewParams(queryId, { supplementaryFilters = {}, metricFilter = 'all', @@ -185,8 +214,8 @@ export function buildViewParams(queryId, { if (supplementaryFilters.workcenterGroups?.length > 0) { params.workcenter_groups = supplementaryFilters.workcenterGroups; } - if (supplementaryFilters.reason) { - params.reason = supplementaryFilters.reason; + if (supplementaryFilters.reasons?.length > 0) { + params.reasons = supplementaryFilters.reasons; } if (metricFilter && metricFilter !== 'all') { params.metric_filter = metricFilter; diff --git a/frontend/src/reject-history/App.vue b/frontend/src/reject-history/App.vue index 5482264..feac24b 100644 --- a/frontend/src/reject-history/App.vue +++ b/frontend/src/reject-history/App.vue @@ -5,6 +5,7 @@ import { apiGet, apiPost } from '../core/api.js'; import { buildViewParams, parseMultiLineInput, + validateDateRange, } from '../core/reject-history-filters.js'; import { replaceRuntimeHistory } from '../core/shell-navigation.js'; @@ -104,14 +105,14 @@ const availableFilters = ref({ workcenterGroups: [], packages: [], reasons: [] } const supplementaryFilters = reactive({ packages: [], workcenterGroups: [], - reason: '', + reasons: [], }); // ---- Interactive state ---- const page = ref(1); const selectedTrendDates = ref([]); const trendLegendSelected = ref({ '扣帳報廢量': true, '不扣帳報廢量': true }); -const paretoDisplayScope = ref('all'); +const paretoDisplayScope = ref('top20'); const paretoSelections = reactive(createEmptyParetoSelections()); const paretoData = reactive(createEmptyParetoData()); @@ -146,6 +147,7 @@ const loading = reactive({ exporting: false, }); const errorMessage = ref(''); +const partialFailureWarning = ref(''); const lastQueryAt = ref(''); // ---- Request staleness tracking ---- @@ -241,8 +243,8 @@ function buildBatchParetoParams() { if (supplementaryFilters.workcenterGroups.length > 0) { params.workcenter_groups = supplementaryFilters.workcenterGroups; } - if (supplementaryFilters.reason) { - params.reason = supplementaryFilters.reason; + if (supplementaryFilters.reasons.length > 0) { + params.reasons = supplementaryFilters.reasons; } if (selectedTrendDates.value.length > 0) { params.trend_dates = selectedTrendDates.value; @@ -301,11 +303,20 @@ async function executePrimaryQuery() { loading.querying = true; loading.list = true; errorMessage.value = ''; + partialFailureWarning.value = ''; try { const body = { mode: queryMode.value }; if (queryMode.value === 'date_range') { + const dateValidationError = validateDateRange( + draftFilters.startDate, + draftFilters.endDate, + ); + if (dateValidationError) { + errorMessage.value = dateValidationError; + return; + } body.start_date = draftFilters.startDate; body.end_date = draftFilters.endDate; } else { @@ -321,6 +332,19 @@ async function executePrimaryQuery() { if (isStaleRequest(requestId)) return; const result = unwrapApiResult(resp, '主查詢執行失敗'); + const meta = result.meta || {}; + if (meta.has_partial_failure) { + const failedChunkCount = Number(meta.failed_chunk_count || 0); + const failedRanges = Array.isArray(meta.failed_ranges) ? meta.failed_ranges : []; + if (failedRanges.length > 0) { + const rangesText = failedRanges + .map((item) => `${item.start} ~ ${item.end}`) + .join('、'); + partialFailureWarning.value = `警告：以下日期區間的資料擷取失敗（${failedChunkCount} 個批次）：${rangesText}。目前顯示結果可能不完整。`; + } else { + partialFailureWarning.value = `警告：${failedChunkCount} 個查詢批次的資料擷取失敗。目前顯示結果可能不完整。`; + } + } committedPrimary.mode = queryMode.value; committedPrimary.startDate = draftFilters.startDate; @@ -344,7 +368,7 @@ async function executePrimaryQuery() { supplementaryFilters.packages = []; supplementaryFilters.workcenterGroups = []; - supplementaryFilters.reason = ''; + supplementaryFilters.reasons = []; page.value = 1; selectedTrendDates.value = []; resetParetoSelections(); @@ -445,7 +469,7 @@ function clearFilters() { draftFilters.excludeMaterialScrap = true; draftFilters.excludePbDiode = true; draftFilters.paretoTop80 = true; - paretoDisplayScope.value = 'all'; + paretoDisplayScope.value = 'top20'; resetParetoSelections(); void executePrimaryQuery(); } @@ -520,7 +544,7 @@ function clearParetoSelection() { function onSupplementaryChange(filters) { supplementaryFilters.packages = filters.packages || []; supplementaryFilters.workcenterGroups = filters.workcenterGroups || []; - supplementaryFilters.reason = filters.reason || ''; + supplementaryFilters.reasons = filters.reasons || []; page.value = 1; selectedTrendDates.value = []; resetParetoSelections(); @@ -545,7 +569,7 @@ function removeFilterChip(chip) { } if (chip.type === 'reason') { - supplementaryFilters.reason = ''; + supplementaryFilters.reasons = supplementaryFilters.reasons.filter((r) => r !== chip.value); page.value = 1; updateUrlState(); void Promise.all([refreshView(), fetchBatchPareto()]); @@ -584,7 +608,7 @@ async function exportCsv() { params.set('query_id', queryId.value); for (const pkg of supplementaryFilters.packages) params.append('packages', pkg); for (const wc of supplementaryFilters.workcenterGroups) params.append('workcenter_groups', wc); - if (supplementaryFilters.reason) params.set('reason', supplementaryFilters.reason); + for (const r of supplementaryFilters.reasons) params.append('reasons', r); params.set('metric_filter', metricFilterParam()); for (const date of selectedTrendDates.value) params.append('trend_dates', date); for (const [dimension, key] of Object.entries(PARETO_SELECTION_PARAM_MAP)) { @@ -760,13 +784,13 @@ const activeFilterChips = computed(() => { value: '', }); - if (supplementaryFilters.reason) { + for (const reason of supplementaryFilters.reasons) { chips.push({ - key: `reason:${supplementaryFilters.reason}`, - label: `原因: ${supplementaryFilters.reason}`, + key: `reason:${reason}`, + label: `原因: ${reason}`, removable: true, type: 'reason', - value: supplementaryFilters.reason, + value: reason, }); } @@ -866,16 +890,14 @@ function updateUrlState() { appendArrayParams(params, 'packages', supplementaryFilters.packages); appendArrayParams(params, 'workcenter_groups', supplementaryFilters.workcenterGroups); - if (supplementaryFilters.reason) { - params.set('reason', supplementaryFilters.reason); - } + appendArrayParams(params, 'reasons', supplementaryFilters.reasons); appendArrayParams(params, 'trend_dates', selectedTrendDates.value); for (const [dimension, key] of Object.entries(PARETO_SELECTION_PARAM_MAP)) { appendArrayParams(params, key, paretoSelections[dimension] || []); } - if (paretoDisplayScope.value !== 'all') { + if (paretoDisplayScope.value !== 'top20') { params.set('pareto_display_scope', paretoDisplayScope.value); } if (!committedPrimary.paretoTop80) { @@ -945,7 +967,7 @@ function restoreFromUrl() { supplementaryFilters.packages = readArrayParam(params, 'packages'); supplementaryFilters.workcenterGroups = readArrayParam(params, 'workcenter_groups'); - supplementaryFilters.reason = String(params.get('reason') || '').trim(); + supplementaryFilters.reasons = readArrayParam(params, 'reasons'); selectedTrendDates.value = readArrayParam(params, 'trend_dates'); @@ -969,7 +991,7 @@ function restoreFromUrl() { } const urlParetoDisplayScope = String(params.get('pareto_display_scope') || '').trim().toLowerCase(); - paretoDisplayScope.value = urlParetoDisplayScope === 'top20' ? 'top20' : 'all'; + paretoDisplayScope.value = urlParetoDisplayScope === 'all' ? 'all' : 'top20'; const parsedPage = Number(params.get('page') || '1'); page.value = Number.isFinite(parsedPage) && parsedPage > 0 ? parsedPage : 1; @@ -1001,6 +1023,9 @@ onMounted(() => {
{{ errorMessage }}
+
+ {{ partialFailureWarning }} +
({}) }, supplementaryFilters: { type: Object, default: () => ({}) }, - queryId: { type: String, default: '' }, - resolutionInfo: { type: Object, default: null }, - loading: { type: Object, required: true }, - activeFilterChips: { type: Array, default: () => [] }, - paretoDisplayScope: { type: String, default: 'all' }, -}); + queryId: { type: String, default: '' }, + resolutionInfo: { type: Object, default: null }, + loading: { type: Object, required: true }, + activeFilterChips: { type: Array, default: () => [] }, + paretoDisplayScope: { type: String, default: 'all' }, +}); const emit = defineEmits([ 'apply', 'clear', 'export-csv', - 'remove-chip', - 'pareto-scope-toggle', - 'pareto-display-scope-change', - 'update:queryMode', - 'update:containerInputType', - 'update:containerInput', + 'remove-chip', + 'pareto-scope-toggle', + 'pareto-display-scope-change', + 'update:queryMode', + 'update:containerInputType', + 'update:containerInput', 'supplementary-change', ]); @@ -32,7 +32,7 @@ function emitSupplementary(patch) { emit('supplementary-change', { packages: props.supplementaryFilters.packages || [], workcenterGroups: props.supplementaryFilters.workcenterGroups || [], - reason: props.supplementaryFilters.reason || '', + reasons: props.supplementaryFilters.reasons || [], ...patch, }); } @@ -86,23 +86,23 @@ function emitSupplementary(patch) {