From 71c8102de68cf1f947388b4913fa37f534f93c14 Mon Sep 17 00:00:00 2001 From: egg Date: Wed, 25 Feb 2026 13:15:02 +0800 Subject: [PATCH] feat: dataset cache for hold/resource history + slow connection migration Two changes combined: 1. historical-query-slow-connection: Migrate all historical query pages to read_sql_df_slow with semaphore concurrency control (max 3), raise DB slow timeout to 300s, gunicorn timeout to 360s, and unify frontend timeouts to 360s for all historical pages. 2. hold-resource-history-dataset-cache: Convert hold-history and resource-history from multi-query to single-query + dataset cache pattern (L1 ProcessLevelCache + L2 Redis parquet/base64, TTL=900s). Replace old GET endpoints with POST /query + GET /view two-phase API. Frontend auto-retries on 410 cache_expired. Co-Authored-By: Claude Opus 4.6 --- .../composables/useExcelQueryData.js | 4 +- frontend/src/hold-history/App.vue | 228 +++--- .../job-query/composables/useJobQueryData.js | 6 +- frontend/src/mid-section-defect/App.vue | 2 +- .../query-tool/components/LotJobsTable.vue | 2 +- .../composables/useEquipmentQuery.js | 4 +- .../query-tool/composables/useLotDetail.js | 8 +- .../query-tool/composables/useLotLineage.js | 2 +- .../query-tool/composables/useLotResolve.js | 2 +- .../composables/useReverseLineage.js | 2 +- frontend/src/reject-history/App.vue | 18 +- frontend/src/resource-history/App.vue | 161 +++- .../shared-composables/useTraceProgress.js | 2 +- gunicorn.conf.py | 6 +- .../.openspec.yaml | 2 + .../design.md | 84 +++ .../proposal.md | 31 + .../specs/hold-dataset-cache/spec.md | 64 ++ .../specs/hold-history-api/spec.md | 62 ++ .../specs/hold-history-page/spec.md | 34 + .../specs/resource-dataset-cache/spec.md | 71 ++ .../specs/resource-history-page/spec.md | 46 ++ .../tasks.md | 42 ++ .../.openspec.yaml | 2 + .../design.md | 78 ++ .../proposal.md | 42 ++ .../specs/hold-history-api/spec.md | 9 + .../specs/hold-history-page/spec.md | 8 + .../specs/progressive-trace-ux/spec.md | 8 + .../specs/query-tool-equipment/spec.md | 8 + .../specs/query-tool-lot-trace/spec.md | 9 + .../specs/reject-history-api/spec.md | 10 + .../specs/reject-history-page/spec.md | 8 + .../specs/resource-history-page/spec.md | 16 + .../slow-query-concurrency-control/spec.md | 53 ++ .../historical-query-slow-connection/tasks.md | 39 + .../.openspec.yaml | 2 + .../design.md | 107 +++ .../proposal.md | 76 ++ .../specs/msd-analysis-transparency/spec.md | 48 ++ .../specs/msd-multifactor-attribution/spec.md | 93 +++ .../specs/msd-suspect-context/spec.md | 77 ++ openspec/specs/hold-dataset-cache/spec.md | 64 ++ openspec/specs/hold-history-api/spec.md | 184 +---- openspec/specs/hold-history-page/spec.md | 190 +---- openspec/specs/resource-dataset-cache/spec.md | 71 ++ openspec/specs/resource-history-page/spec.md | 161 +--- src/mes_dashboard/config/settings.py | 8 + src/mes_dashboard/core/database.py | 68 +- .../routes/hold_history_routes.py | 236 +++--- .../routes/resource_history_routes.py | 195 +++-- .../services/excel_query_service.py | 56 +- .../services/hold_dataset_cache.py | 553 ++++++++++++++ .../services/hold_history_service.py | 2 +- .../services/job_query_service.py | 560 +++++++------- .../services/query_tool_service.py | 2 +- .../services/reject_dataset_cache.py | 2 +- .../services/reject_history_service.py | 2 +- .../services/resource_dataset_cache.py | 685 ++++++++++++++++++ .../services/resource_history_service.py | 2 +- .../sql/hold_history/base_facts.sql | 101 +++ .../sql/resource_history/base_facts.sql | 31 + tests/test_hold_history_routes.py | 313 ++++---- tests/test_resource_history_routes.py | 186 +++-- 64 files changed, 3806 insertions(+), 1442 deletions(-) create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/.openspec.yaml create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/design.md create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/proposal.md create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/specs/hold-dataset-cache/spec.md create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/specs/hold-history-api/spec.md create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/specs/hold-history-page/spec.md create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/specs/resource-dataset-cache/spec.md create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/specs/resource-history-page/spec.md create mode 100644 openspec/changes/archive/2026-02-25-hold-resource-history-dataset-cache/tasks.md create mode 100644 openspec/changes/historical-query-slow-connection/.openspec.yaml create mode 100644 openspec/changes/historical-query-slow-connection/design.md create mode 100644 openspec/changes/historical-query-slow-connection/proposal.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/hold-history-api/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/hold-history-page/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/progressive-trace-ux/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/query-tool-equipment/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/query-tool-lot-trace/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/reject-history-api/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/reject-history-page/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/resource-history-page/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/specs/slow-query-concurrency-control/spec.md create mode 100644 openspec/changes/historical-query-slow-connection/tasks.md create mode 100644 openspec/changes/msd-multifactor-backward-tracing/.openspec.yaml create mode 100644 openspec/changes/msd-multifactor-backward-tracing/design.md create mode 100644 openspec/changes/msd-multifactor-backward-tracing/proposal.md create mode 100644 openspec/changes/msd-multifactor-backward-tracing/specs/msd-analysis-transparency/spec.md create mode 100644 openspec/changes/msd-multifactor-backward-tracing/specs/msd-multifactor-attribution/spec.md create mode 100644 openspec/changes/msd-multifactor-backward-tracing/specs/msd-suspect-context/spec.md create mode 100644 openspec/specs/hold-dataset-cache/spec.md create mode 100644 openspec/specs/resource-dataset-cache/spec.md create mode 100644 src/mes_dashboard/services/hold_dataset_cache.py create mode 100644 src/mes_dashboard/services/resource_dataset_cache.py create mode 100644 src/mes_dashboard/sql/hold_history/base_facts.sql create mode 100644 src/mes_dashboard/sql/resource_history/base_facts.sql diff --git a/frontend/src/excel-query/composables/useExcelQueryData.js b/frontend/src/excel-query/composables/useExcelQueryData.js index 2542fb9..c7cd752 100644 --- a/frontend/src/excel-query/composables/useExcelQueryData.js +++ b/frontend/src/excel-query/composables/useExcelQueryData.js @@ -132,7 +132,7 @@ export function useExcelQueryData() { try { const formData = new FormData(); formData.append('file', file); - const payload = await apiUpload('/api/excel-query/upload', formData, { timeout: 120000, silent: true }); + const payload = await apiUpload('/api/excel-query/upload', formData, { timeout: 360000, silent: true }); excelColumns.value = Array.isArray(payload?.columns) ? payload.columns : []; excelPreview.value = Array.isArray(payload?.preview) ? payload.preview : []; uploadState.fileName = String(file.name || ''); @@ -252,7 +252,7 @@ export function useExcelQueryData() { date_from: filters.dateFrom || undefined, date_to: filters.dateTo || undefined, }, - { timeout: 120000, silent: true }, + { timeout: 360000, silent: true }, ); queryResult.rows = Array.isArray(payload?.data) ? payload.data : []; queryResult.columns = Array.isArray(payload?.columns) ? payload.columns : filters.returnColumns; diff --git a/frontend/src/hold-history/App.vue b/frontend/src/hold-history/App.vue index 2fb6ef9..6b14458 100644 --- a/frontend/src/hold-history/App.vue +++ b/frontend/src/hold-history/App.vue @@ -1,7 +1,7 @@ diff --git a/frontend/src/job-query/composables/useJobQueryData.js b/frontend/src/job-query/composables/useJobQueryData.js index 8aa45b5..0acea9f 100644 --- a/frontend/src/job-query/composables/useJobQueryData.js +++ b/frontend/src/job-query/composables/useJobQueryData.js @@ -174,7 +174,7 @@ export function useJobQueryData() { loadingResources.value = true; errorMessage.value = ''; try { - const payload = await apiGet('/api/job-query/resources', { timeout: 60000, silent: true }); + const payload = await apiGet('/api/job-query/resources', { timeout: 360000, silent: true }); resources.value = Array.isArray(payload?.data) ? payload.data : []; } catch (error) { errorMessage.value = error?.message || '載入設備清單失敗'; @@ -206,7 +206,7 @@ export function useJobQueryData() { start_date: filters.startDate, end_date: filters.endDate, }, - { timeout: 60000, silent: true }, + { timeout: 360000, silent: true }, ); jobs.value = Array.isArray(payload?.data) ? payload.data : []; return true; @@ -229,7 +229,7 @@ export function useJobQueryData() { errorMessage.value = ''; try { const payload = await apiGet(`/api/job-query/txn/${encodeURIComponent(id)}`, { - timeout: 60000, + timeout: 360000, silent: true, }); txnRows.value = Array.isArray(payload?.data) ? payload.data : []; diff --git a/frontend/src/mid-section-defect/App.vue b/frontend/src/mid-section-defect/App.vue index 1c0eeb7..1d7592a 100644 --- a/frontend/src/mid-section-defect/App.vue +++ b/frontend/src/mid-section-defect/App.vue @@ -16,7 +16,7 @@ import SuspectContextPanel from './components/SuspectContextPanel.vue'; ensureMesApiAvailable(); -const API_TIMEOUT = 120000; +const API_TIMEOUT = 360000; const PAGE_SIZE = 200; const SESSION_CACHE_KEY = 'msd:cache'; const SESSION_CACHE_TTL = 5 * 60 * 1000; // 5 min, matches backend Redis TTL diff --git a/frontend/src/query-tool/components/LotJobsTable.vue b/frontend/src/query-tool/components/LotJobsTable.vue index 134b0d1..a55a893 100644 --- a/frontend/src/query-tool/components/LotJobsTable.vue +++ b/frontend/src/query-tool/components/LotJobsTable.vue @@ -135,7 +135,7 @@ async function loadTxn(jobId) { try { const payload = await apiGet(`/api/job-query/txn/${encodeURIComponent(id)}`, { - timeout: 60000, + timeout: 360000, silent: true, }); txnRows.value = Array.isArray(payload?.data) ? payload.data : []; diff --git a/frontend/src/query-tool/composables/useEquipmentQuery.js b/frontend/src/query-tool/composables/useEquipmentQuery.js index a6d50a9..5338093 100644 --- a/frontend/src/query-tool/composables/useEquipmentQuery.js +++ b/frontend/src/query-tool/composables/useEquipmentQuery.js @@ -120,7 +120,7 @@ export function useEquipmentQuery(initial = {}) { const payload = await apiPost( '/api/query-tool/equipment-period', buildQueryPayload(queryType), - { timeout: 120000, silent: true }, + { timeout: 360000, silent: true }, ); return Array.isArray(payload?.data) ? payload.data : []; @@ -132,7 +132,7 @@ export function useEquipmentQuery(initial = {}) { try { const payload = await apiGet('/api/query-tool/equipment-list', { - timeout: 60000, + timeout: 360000, silent: true, }); equipmentOptions.value = Array.isArray(payload?.data) ? payload.data : []; diff --git a/frontend/src/query-tool/composables/useLotDetail.js b/frontend/src/query-tool/composables/useLotDetail.js index 734d10f..8ad3a63 100644 --- a/frontend/src/query-tool/composables/useLotDetail.js +++ b/frontend/src/query-tool/composables/useLotDetail.js @@ -164,7 +164,7 @@ export function useLotDetail(initial = {}) { try { const payload = await apiGet('/api/query-tool/workcenter-groups', { - timeout: 60000, + timeout: 360000, silent: true, }); @@ -205,7 +205,7 @@ export function useLotDetail(initial = {}) { } const payload = await apiGet(`/api/query-tool/lot-history?${params.toString()}`, { - timeout: 120000, + timeout: 360000, silent: true, }); @@ -263,7 +263,7 @@ export function useLotDetail(initial = {}) { params.set('time_end', timeRange.time_end); const payload = await apiGet(`/api/query-tool/lot-associations?${params.toString()}`, { - timeout: 120000, + timeout: 360000, silent: true, }); @@ -279,7 +279,7 @@ export function useLotDetail(initial = {}) { params.set('type', associationType); const payload = await apiGet(`/api/query-tool/lot-associations?${params.toString()}`, { - timeout: 120000, + timeout: 360000, silent: true, }); diff --git a/frontend/src/query-tool/composables/useLotLineage.js b/frontend/src/query-tool/composables/useLotLineage.js index 276bcae..859ec59 100644 --- a/frontend/src/query-tool/composables/useLotLineage.js +++ b/frontend/src/query-tool/composables/useLotLineage.js @@ -205,7 +205,7 @@ export function useLotLineage(initial = {}) { profile: 'query_tool', container_ids: containerIds, }, - { timeout: 60000, silent: true }, + { timeout: 360000, silent: true }, ); } catch (error) { const status = Number(error?.status || 0); diff --git a/frontend/src/query-tool/composables/useLotResolve.js b/frontend/src/query-tool/composables/useLotResolve.js index 279336a..0cc46ae 100644 --- a/frontend/src/query-tool/composables/useLotResolve.js +++ b/frontend/src/query-tool/composables/useLotResolve.js @@ -134,7 +134,7 @@ export function useLotResolve(initial = {}) { input_type: inputType.value, values, }, - { timeout: 60000, silent: true }, + { timeout: 360000, silent: true }, ); resolvedLots.value = Array.isArray(payload?.data) ? payload.data : []; diff --git a/frontend/src/query-tool/composables/useReverseLineage.js b/frontend/src/query-tool/composables/useReverseLineage.js index 588db38..44ae418 100644 --- a/frontend/src/query-tool/composables/useReverseLineage.js +++ b/frontend/src/query-tool/composables/useReverseLineage.js @@ -216,7 +216,7 @@ export function useReverseLineage(initial = {}) { profile: 'query_tool_reverse', container_ids: containerIds, }, - { timeout: 60000, silent: true }, + { timeout: 360000, silent: true }, ); } catch (error) { const status = Number(error?.status || 0); diff --git a/frontend/src/reject-history/App.vue b/frontend/src/reject-history/App.vue index b3bd626..f888155 100644 --- a/frontend/src/reject-history/App.vue +++ b/frontend/src/reject-history/App.vue @@ -15,7 +15,7 @@ import ParetoSection from './components/ParetoSection.vue'; import SummaryCards from './components/SummaryCards.vue'; import TrendChart from './components/TrendChart.vue'; -const API_TIMEOUT = 60000; +const API_TIMEOUT = 360000; const DEFAULT_PER_PAGE = 50; // ---- Primary query form state ---- @@ -214,7 +214,11 @@ async function executePrimaryQuery() { updateUrlState(); } catch (error) { if (isStaleRequest(requestId)) return; - errorMessage.value = error?.message || '主查詢執行失敗'; + if (error?.name === 'AbortError') { + errorMessage.value = '查詢逾時,請縮短日期範圍後重試'; + } else { + errorMessage.value = error?.message || '主查詢執行失敗'; + } } finally { if (isStaleRequest(requestId)) return; loading.querying = false; @@ -264,7 +268,11 @@ async function refreshView() { updateUrlState(); } catch (error) { if (isStaleRequest(requestId)) return; - errorMessage.value = error?.message || '視圖查詢失敗'; + if (error?.name === 'AbortError') { + errorMessage.value = '查詢逾時,請縮短日期範圍後重試'; + } else { + errorMessage.value = error?.message || '視圖查詢失敗'; + } } finally { if (isStaleRequest(requestId)) return; loading.list = false; @@ -360,7 +368,9 @@ async function fetchDimensionPareto(dim) { } catch (err) { if (myId !== activeDimRequestId) return; dimensionParetoItems.value = []; - errorMessage.value = err.message || '查詢維度 Pareto 失敗'; + if (err?.name !== 'AbortError') { + errorMessage.value = err.message || '查詢維度 Pareto 失敗'; + } } finally { if (myId === activeDimRequestId) { dimensionParetoLoading.value = false; diff --git a/frontend/src/resource-history/App.vue b/frontend/src/resource-history/App.vue index 52f10da..ff27090 100644 --- a/frontend/src/resource-history/App.vue +++ b/frontend/src/resource-history/App.vue @@ -1,7 +1,7 @@