diff --git a/frontend/src/mid-section-defect/App.vue b/frontend/src/mid-section-defect/App.vue index c9932cd..640e4b9 100644 --- a/frontend/src/mid-section-defect/App.vue +++ b/frontend/src/mid-section-defect/App.vue @@ -1,20 +1,22 @@ - - - + +void initPage(); + + + diff --git a/frontend/src/mid-section-defect/style.css b/frontend/src/mid-section-defect/style.css index 1bfec10..97127f0 100644 --- a/frontend/src/mid-section-defect/style.css +++ b/frontend/src/mid-section-defect/style.css @@ -456,43 +456,79 @@ body { } /* ====== Empty State ====== */ -.empty-state { - display: flex; - align-items: center; - justify-content: center; - padding: 80px 20px; - color: var(--msd-muted); - font-size: 15px; -} - -/* ====== Loading Overlay ====== */ -.loading-overlay { - position: fixed; - inset: 0; - display: flex; - align-items: center; - justify-content: center; - background: rgba(255, 255, 255, 0.7); - z-index: 999; - transition: opacity 0.2s; -} - -.loading-overlay.hidden { - opacity: 0; - pointer-events: none; -} - -.loading-spinner { - width: 40px; - height: 40px; - border: 3px solid #e2e8f0; - border-top-color: var(--msd-primary); - border-radius: 50%; - animation: spin 0.8s linear infinite; -} - -@keyframes spin { - to { - transform: rotate(360deg); - } -} +.empty-state { + display: flex; + align-items: center; + justify-content: center; + padding: 80px 20px; + color: var(--msd-muted); + font-size: 15px; +} + +/* ====== Trace Skeleton ====== */ +.trace-skeleton-section { + margin-bottom: 16px; +} + +.trace-skeleton-kpi-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 12px; + margin-bottom: 12px; +} + +.trace-skeleton-card { + min-height: 100px; + border-radius: 10px; + background: #e5eaf2; + box-shadow: var(--msd-shadow); +} + +.trace-skeleton-chart-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 12px; +} + +.trace-skeleton-chart { + min-height: 300px; + border-radius: 10px; + background: #e5eaf2; + box-shadow: var(--msd-shadow); +} + +.trace-skeleton-trend { + grid-column: 1 / -1; +} + +.trace-skeleton-pulse { + animation: trace-skeleton-pulse 1.2s ease-in-out infinite; +} + +.trace-fade-enter-active, +.trace-fade-leave-active { + transition: opacity 0.3s ease-in; +} + +.trace-fade-enter-from, +.trace-fade-leave-to { + opacity: 0; +} + +@keyframes trace-skeleton-pulse { + 0% { + opacity: 0.5; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0.5; + } +} + +@media (max-width: 1200px) { + .trace-skeleton-chart-grid { + grid-template-columns: 1fr; + } +} diff --git a/frontend/src/query-tool/App.vue b/frontend/src/query-tool/App.vue index 6b7e7a0..8784ba3 100644 --- a/frontend/src/query-tool/App.vue +++ b/frontend/src/query-tool/App.vue @@ -1,5 +1,5 @@ + + + + diff --git a/frontend/src/shared-composables/index.js b/frontend/src/shared-composables/index.js index 63d91fc..50c98de 100644 --- a/frontend/src/shared-composables/index.js +++ b/frontend/src/shared-composables/index.js @@ -2,3 +2,4 @@ export { useAutoRefresh } from './useAutoRefresh.js'; export { useAutocomplete } from './useAutocomplete.js'; export { usePaginationState } from './usePaginationState.js'; export { readQueryState, writeQueryState } from './useQueryState.js'; +export { useTraceProgress } from './useTraceProgress.js'; diff --git a/frontend/src/shared-composables/useTraceProgress.js b/frontend/src/shared-composables/useTraceProgress.js new file mode 100644 index 0000000..16925fa --- /dev/null +++ b/frontend/src/shared-composables/useTraceProgress.js @@ -0,0 +1,181 @@ +import { reactive, ref } from 'vue'; + +import { apiPost, ensureMesApiAvailable } from '../core/api.js'; + +ensureMesApiAvailable(); + +const DEFAULT_STAGE_TIMEOUT_MS = 60000; +const PROFILE_DOMAINS = Object.freeze({ + query_tool: ['history', 'materials', 'rejects', 'holds', 'jobs'], + mid_section_defect: ['upstream_history'], +}); + +function stageKey(stageName) { + if (stageName === 'seed-resolve') return 'seed'; + if (stageName === 'lineage') return 'lineage'; + return 'events'; +} + +function normalizeSeedContainerIds(seedPayload) { + const rows = Array.isArray(seedPayload?.seeds) ? seedPayload.seeds : []; + const seen = new Set(); + const containerIds = []; + rows.forEach((row) => { + const id = String(row?.container_id || '').trim(); + if (!id || seen.has(id)) { + return; + } + seen.add(id); + containerIds.push(id); + }); + return containerIds; +} + +function collectAllContainerIds(seedContainerIds, lineagePayload) { + const seen = new Set(seedContainerIds); + const merged = [...seedContainerIds]; + const ancestors = lineagePayload?.ancestors || {}; + Object.values(ancestors).forEach((values) => { + if (!Array.isArray(values)) { + return; + } + values.forEach((value) => { + const id = String(value || '').trim(); + if (!id || seen.has(id)) { + return; + } + seen.add(id); + merged.push(id); + }); + }); + return merged; +} + +export function useTraceProgress({ profile } = {}) { + const current_stage = ref(null); + const completed_stages = ref([]); + const is_running = ref(false); + + const stage_results = reactive({ + seed: null, + lineage: null, + events: null, + }); + + const stage_errors = reactive({ + seed: null, + lineage: null, + events: null, + }); + + let activeController = null; + + function reset() { + completed_stages.value = []; + current_stage.value = null; + stage_results.seed = null; + stage_results.lineage = null; + stage_results.events = null; + stage_errors.seed = null; + stage_errors.lineage = null; + stage_errors.events = null; + } + + function abort() { + if (activeController) { + activeController.abort(); + activeController = null; + } + } + + async function execute(params = {}) { + const domains = PROFILE_DOMAINS[profile]; + if (!domains) { + throw new Error(`Unsupported trace profile: ${profile}`); + } + + abort(); + reset(); + is_running.value = true; + + const controller = new AbortController(); + activeController = controller; + + try { + current_stage.value = 'seed-resolve'; + const seedPayload = await apiPost( + '/api/trace/seed-resolve', + { profile, params }, + { timeout: DEFAULT_STAGE_TIMEOUT_MS, signal: controller.signal }, + ); + stage_results.seed = seedPayload; + completed_stages.value = [...completed_stages.value, 'seed-resolve']; + + const seedContainerIds = normalizeSeedContainerIds(seedPayload); + if (seedContainerIds.length === 0) { + return stage_results; + } + + current_stage.value = 'lineage'; + const lineagePayload = await apiPost( + '/api/trace/lineage', + { + profile, + container_ids: seedContainerIds, + cache_key: seedPayload?.cache_key || null, + }, + { timeout: DEFAULT_STAGE_TIMEOUT_MS, signal: controller.signal }, + ); + stage_results.lineage = lineagePayload; + completed_stages.value = [...completed_stages.value, 'lineage']; + + const allContainerIds = collectAllContainerIds(seedContainerIds, lineagePayload); + current_stage.value = 'events'; + const eventsPayload = await apiPost( + '/api/trace/events', + { + profile, + container_ids: allContainerIds, + domains, + cache_key: seedPayload?.cache_key || null, + params, + seed_container_ids: seedContainerIds, + lineage: { + ancestors: lineagePayload?.ancestors || {}, + }, + }, + { timeout: DEFAULT_STAGE_TIMEOUT_MS, signal: controller.signal }, + ); + stage_results.events = eventsPayload; + completed_stages.value = [...completed_stages.value, 'events']; + return stage_results; + } catch (error) { + if (error?.name === 'AbortError') { + return stage_results; + } + const key = stageKey(current_stage.value); + stage_errors[key] = { + code: error?.errorCode || null, + message: error?.message || '追溯查詢失敗', + }; + return stage_results; + } finally { + if (activeController === controller) { + activeController = null; + } + current_stage.value = null; + is_running.value = false; + } + } + + return { + current_stage, + completed_stages, + stage_results, + stage_errors, + is_running, + execute, + reset, + abort, + }; +} diff --git a/frontend/src/wip-overview/components/MatrixTable.vue b/frontend/src/wip-overview/components/MatrixTable.vue index e68b5c9..eb13be3 100644 --- a/frontend/src/wip-overview/components/MatrixTable.vue +++ b/frontend/src/wip-overview/components/MatrixTable.vue @@ -11,7 +11,7 @@ const props = defineProps({ const emit = defineEmits(['drilldown']); const workcenters = computed(() => props.data?.workcenters || []); -const packages = computed(() => (props.data?.packages || []).slice(0, 15)); +const packages = computed(() => props.data?.packages || []); function formatNumber(value) { if (!value) { diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/.openspec.yaml b/openspec/changes/archive/2026-02-12-unified-lineage-engine/.openspec.yaml new file mode 100644 index 0000000..05ac962 --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +status: proposal diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/design.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/design.md new file mode 100644 index 0000000..8a86138 --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/design.md @@ -0,0 +1,202 @@ +## Context + +兩個高查詢複雜度頁面（`/mid-section-defect` 和 `/query-tool`）各自實作了 LOT 血緣追溯邏輯。mid-section-defect 使用 Python BFS（`_bfs_split_chain()` + `_fetch_merge_sources()`），query-tool 使用 `_build_in_filter()` 字串拼接。兩者共用的底層資料表為 `DWH.DW_MES_CONTAINER`（5.2M rows, CONTAINERID UNIQUE index）和 `DWH.DW_MES_PJ_COMBINEDASSYLOTS`（1.97M rows, FINISHEDNAME indexed）。 + +現行問題： +- BFS 每輪一次 DB round-trip（3-16 輪），加上 `genealogy_records.sql` 全掃描 `HM_LOTMOVEOUT`（48M rows） +- `_build_in_filter()` 字串拼接存在 SQL injection 風險 +- query-tool 無 rate limit / cache，可打爆 DB pool (pool_size=10, max_overflow=20) +- 兩份 service 各 1200-1300 行，血緣邏輯重複 + +既有安全基礎設施： +- `QueryBuilder`（`sql/builder.py`）：`add_in_condition()` 支援 bind params `:p0, :p1, ...` +- `SQLLoader`（`sql/loader.py`）：`load_with_params()` 支援結構參數 `{{ PARAM }}` +- `configured_rate_limit()`（`core/rate_limit.py`）：per-client rate limit with `Retry-After` header +- `LayeredCache`（`core/cache.py`）：L1 MemoryTTL + L2 Redis + +## Goals / Non-Goals + +**Goals:** +- 以 `CONNECT BY NOCYCLE` 取代 Python BFS，將 3-16 次 DB round-trip 縮減為 1 次 +- 建立 `LineageEngine` 統一模組，消除血緣邏輯重複 +- 消除 `_build_in_filter()` SQL injection 風險 +- 為 query-tool 加入 rate limit + cache（對齊 mid-section-defect） +- 為 `lot_split_merge_history` 加入 fast/full 雙模式 + +**Non-Goals:** +- 不新增 API endpoint（由後續 `trace-progressive-ui` 負責） +- 不改動前端 +- 不建立 materialized view / 不使用 PARALLEL hints +- 不改動其他頁面（wip-detail, lot-detail 等） + +## Decisions + +### D1: CONNECT BY NOCYCLE 作為主要遞迴查詢策略 + +**選擇**: Oracle `CONNECT BY NOCYCLE` with `LEVEL <= 20` +**替代方案**: Recursive `WITH` (recursive subquery factoring) +**理由**: +- `CONNECT BY` 是 Oracle 原生遞迴語法，在 Oracle 19c 上執行計劃最佳化最成熟 +- `LEVEL <= 20` 等價於現行 BFS `bfs_round > 20` 防護 +- `NOCYCLE` 處理循環引用（`SPLITFROMID` 可能存在資料錯誤的循環） +- recursive `WITH` 作為 SQL 檔案內的註解替代方案，若 execution plan 不佳可快速切換 + +**SQL 設計**（`sql/lineage/split_ancestors.sql`）: +```sql +SELECT + c.CONTAINERID, + c.SPLITFROMID, + c.CONTAINERNAME, + LEVEL AS SPLIT_DEPTH +FROM DWH.DW_MES_CONTAINER c +START WITH {{ CID_FILTER }} +CONNECT BY NOCYCLE PRIOR c.SPLITFROMID = c.CONTAINERID + AND LEVEL <= 20 +``` +- `{{ CID_FILTER }}` 由 `QueryBuilder.get_conditions_sql()` 生成，bind params 注入 +- Oracle IN clause 上限透過 `ORACLE_IN_BATCH_SIZE=1000` 分批，多批結果合併 + +### D2: LineageEngine 模組結構 + +``` +src/mes_dashboard/services/lineage_engine.py +├── resolve_split_ancestors(container_ids: List[str]) -> Dict +│ └── 回傳 {child_to_parent: {cid: parent_cid}, cid_to_name: {cid: name}} +├── resolve_merge_sources(container_names: List[str]) -> Dict +│ └── 回傳 {finished_name: [{source_cid, source_name}]} +└── resolve_full_genealogy(container_ids: List[str], initial_names: Dict) -> Dict + └── 組合 split + merge，回傳 {cid: Set[ancestor_cids]} + +src/mes_dashboard/sql/lineage/ +├── split_ancestors.sql (CONNECT BY NOCYCLE) +└── merge_sources.sql (from merge_lookup.sql) +``` + +**函數簽名設計**: +- profile-agnostic：接受 `container_ids: List[str]`，不綁定頁面邏輯 +- 回傳原生 Python 資料結構（dict/set），不回傳 DataFrame +- 內部使用 `QueryBuilder` + `SQLLoader.load_with_params()` + `read_sql_df()` +- batch 邏輯封裝在模組內（caller 不需處理 `ORACLE_IN_BATCH_SIZE`） + +### D3: EventFetcher 模組結構 + +``` +src/mes_dashboard/services/event_fetcher.py +├── fetch_events(container_ids: List[str], domain: str) -> List[Dict] +│ └── 支援 domain: history, materials, rejects, holds, jobs, upstream_history +├── _cache_key(domain: str, container_ids: List[str]) -> str +│ └── 格式: evt:{domain}:{sorted_cids_hash} +└── _get_rate_limit_config(domain: str) -> Dict + └── 回傳 {bucket, max_attempts, window_seconds} +``` + +**快取策略**: +- L2 Redis cache（對齊 `core/cache.py` 模式），TTL 依 domain 配置 +- cache key 使用 `hashlib.md5(sorted(cids).encode()).hexdigest()[:12]` 避免超長 key +- mid-section-defect 既有的 `_fetch_upstream_history()` 遷移到 `fetch_events(cids, "upstream_history")` + +### D4: query-tool SQL injection 修復策略 + +**修復範圍**（6 個呼叫點）: +1. `_resolve_by_lot_id()` (line 262): `_build_in_filter(lot_ids, 'CONTAINERNAME')` + `read_sql_df(sql, {})` +2. `_resolve_by_serial_number()` (line ~320): 同上模式 +3. `_resolve_by_work_order()` (line ~380): 同上模式 +4. `get_lot_history()` 內部的 IN 子句 +5. `get_lot_associations()` 內部的 IN 子句 +6. `lot_split_merge_history` 查詢 + +**修復模式**（統一）: +```python +# Before (unsafe) +in_filter = _build_in_filter(lot_ids, 'CONTAINERNAME') +sql = f"SELECT ... WHERE {in_filter}" +df = read_sql_df(sql, {}) + +# After (safe) +builder = QueryBuilder() +builder.add_in_condition("CONTAINERNAME", lot_ids) +sql = SQLLoader.load_with_params( + "query_tool/lot_resolve_id", + CONTAINER_FILTER=builder.get_conditions_sql(), +) +df = read_sql_df(sql, builder.params) +``` + +**`_build_in_filter()` 和 `_build_in_clause()` 完全刪除**（非 deprecated，直接刪除，因為這是安全漏洞）。 + +### D5: query-tool rate limit + cache 配置 + +**Rate limit**（對齊 `configured_rate_limit()` 模式）: +| Endpoint | Bucket | Max/Window | Env Override | +|----------|--------|------------|-------------| +| `/resolve` | `query-tool-resolve` | 10/60s | `QT_RESOLVE_RATE_*` | +| `/lot-history` | `query-tool-history` | 20/60s | `QT_HISTORY_RATE_*` | +| `/lot-associations` | `query-tool-association` | 20/60s | `QT_ASSOC_RATE_*` | +| `/adjacent-lots` | `query-tool-adjacent` | 20/60s | `QT_ADJACENT_RATE_*` | +| `/equipment-period` | `query-tool-equipment` | 5/60s | `QT_EQUIP_RATE_*` | +| `/export-csv` | `query-tool-export` | 3/60s | `QT_EXPORT_RATE_*` | + +**Cache**: +- resolve result: L2 Redis, TTL=60s, key=`qt:resolve:{input_type}:{values_hash}` +- 其他 GET endpoints: 暫不加 cache（結果依賴動態 CONTAINERID 參數，cache 命中率低） + +### D6: lot_split_merge_history fast/full 雙模式 + +**Fast mode**（預設）: +```sql +-- lot_split_merge_history.sql 加入條件 +AND h.TXNDATE >= ADD_MONTHS(SYSDATE, -6) +... +FETCH FIRST 500 ROWS ONLY +``` + +**Full mode**（`full_history=true`）: +- SQL variant 不含時間窗和 row limit +- 使用 `read_sql_df_slow()` (120s timeout) 取代 `read_sql_df()` (55s timeout) +- Route 層透過 `request.args.get('full_history', 'false').lower() == 'true'` 判斷 + +### D7: 重構順序與 regression 防護 + +**Phase 1**: mid-section-defect（較安全，有 cache + distributed lock 保護） +1. 建立 `lineage_engine.py` + SQL files +2. 在 `mid_section_defect_service.py` 中以 `LineageEngine` 取代 BFS 三函數 +3. golden test 驗證 BFS vs CONNECT BY 結果一致 +4. 廢棄 `genealogy_records.sql` + `split_chain.sql`（標記 deprecated） + +**Phase 2**: query-tool（風險較高，無既有保護） +1. 修復所有 `_build_in_filter()` → `QueryBuilder` +2. 刪除 `_build_in_filter()` + `_build_in_clause()` +3. 加入 route-level rate limit +4. 加入 resolve cache +5. 加入 `lot_split_merge_history` fast/full mode + +**Phase 3**: EventFetcher +1. 建立 `event_fetcher.py` +2. 遷移 `_fetch_upstream_history()` → `EventFetcher` +3. 遷移 query-tool event fetch paths → `EventFetcher` + +## Risks / Trade-offs + +| Risk | Mitigation | +|------|-----------| +| CONNECT BY 對超大血緣樹 (>10000 nodes) 可能產生不預期的 execution plan | `LEVEL <= 20` 硬上限 + SQL 檔案內含 recursive `WITH` 替代方案可快速切換 | +| golden test 覆蓋率不足導致 regression 漏網 | 選取 ≥5 個已知血緣結構的 LOT（含多層 split + merge 交叉），CI gate 強制通過 | +| `_build_in_filter()` 刪除後漏改呼叫點 | Phase 2 完成後 `grep -r "_build_in_filter\|_build_in_clause" src/` 必須 0 結果 | +| fast mode 6 個月時間窗可能截斷需要完整歷史的追溯 | 提供 `full_history=true` 切換完整模式，前端預設不加此參數 = fast mode | +| QueryBuilder `add_in_condition()` 對 >1000 值不自動分批 | LineageEngine 內部封裝分批邏輯（`for i in range(0, len(ids), 1000)`），呼叫者無感 | + +## Migration Plan + +1. **建立新模組**：`lineage_engine.py`, `event_fetcher.py`, `sql/lineage/*.sql` — 無副作用，可安全部署 +2. **Phase 1 切換**：mid-section-defect 內部呼叫改用 `LineageEngine` — 有 cache/lock 保護，regression 可透過 golden test + 手動比對驗證 +3. **Phase 2 切換**：query-tool 修復 + rate limit + cache — 需重新跑 query-tool 路由測試 +4. **Phase 3 切換**：EventFetcher 遷移 — 最後執行，影響範圍最小 +5. **清理**：確認 deprecated SQL files 無引用後刪除 + +**Rollback**: 每個 Phase 獨立，可單獨 revert。`LineageEngine` 和 `EventFetcher` 為新模組，不影響既有程式碼直到各 Phase 的切換 commit。 + +## Open Questions + +- `DW_MES_CONTAINER.SPLITFROMID` 欄位是否有 index？若無，`CONNECT BY` 的 `START WITH` 性能可能依賴全表掃描而非 CONTAINERID index。需確認 Oracle execution plan。 +- `ORACLE_IN_BATCH_SIZE=1000` 對 `CONNECT BY START WITH ... IN (...)` 的行為是否與普通 `WHERE ... IN (...)` 一致？需在開發環境驗證。 +- EventFetcher 的 cache TTL 各 domain 是否需要差異化（如 `upstream_history` 較長、`holds` 較短）？暫統一 300s，後續視使用模式調整。 diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/proposal.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/proposal.md new file mode 100644 index 0000000..0ce41e1 --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/proposal.md @@ -0,0 +1,110 @@ +## Why + +批次追蹤工具 (`/query-tool`) 與中段製程不良追溯分析 (`/mid-section-defect`) 是本專案中查詢複雜度最高的兩個頁面。兩者都需要解析 LOT 血緣關係（拆批 split + 併批 merge），但各自實作了獨立的追溯邏輯，導致： + +1. **效能瓶頸**：mid-section-defect 使用 Python 多輪 BFS 追溯 split chain（`_bfs_split_chain()`，每次 3-16 次 DB round-trip），加上 `genealogy_records.sql` 對 48M 行的 `HM_LOTMOVEOUT` 全表掃描（30-120 秒）。 +2. **安全風險**：query-tool 的 `_build_in_filter()` 使用字串拼接建構 IN 子句（`query_tool_service.py:156-174`），`_resolve_by_lot_id()` / `_resolve_by_serial_number()` / `_resolve_by_work_order()` 系列函數傳入空 params `read_sql_df(sql, {})`——值直接嵌入 SQL 字串中，存在 SQL 注入風險。 +3. **缺乏防護**：query-tool 無 rate limit、無 cache，高併發時可打爆 DB connection pool（Production pool_size=10, max_overflow=20）。 +4. **重複程式碼**：兩個 service 各自維護 split chain 追溯、merge lookup、batch IN 分段等相同邏輯。 + +Oracle 19c 的 `CONNECT BY NOCYCLE` 可以用一條 SQL 取代整套 Python BFS，將 3-16 次 DB round-trip 縮減為 1 次。備選方案為 Oracle 19c 支援的 recursive `WITH` (recursive subquery factoring)，功能等價但可讀性更好。split/merge 的資料來源 (`DW_MES_CONTAINER.SPLITFROMID` + `DW_MES_PJ_COMBINEDASSYLOTS`) 完全不需碰 `HM_LOTMOVEOUT`，可消除 48M 行全表掃描。 + +**邊界聲明**：本變更為純後端內部重構，不新增任何 API endpoint，不改動前端。既有 API contract 向下相容（URL、request/response 格式不變），僅新增可選的 `full_history` query param 作為向下相容擴展。後續的前端分段載入和新增 API endpoints 列入獨立的 `trace-progressive-ui` 變更。 + +## What Changes + +- 建立統一的 `LineageEngine` 模組（`src/mes_dashboard/services/lineage_engine.py`），提供 LOT 血緣解析共用核心： + - `resolve_split_ancestors()` — 使用 `CONNECT BY NOCYCLE` 單次 SQL 查詢取代 Python BFS（備選: recursive `WITH`，於 SQL 檔案中以註解標註替代寫法） + - `resolve_merge_sources()` — 從 `DW_MES_PJ_COMBINEDASSYLOTS` 查詢併批來源 + - `resolve_full_genealogy()` — 組合 split + merge 為完整血緣圖 + - 設計為 profile-agnostic 的公用函數，未來其他頁面（wip-detail、lot-detail）可直接呼叫，但本變更僅接入 mid-section-defect 和 query-tool +- 建立統一的 `EventFetcher` 模組，提供帶 cache + rate limit 的批次事件查詢，封裝既有的 domain 查詢（history、materials、rejects、holds、jobs、upstream_history）。 +- 重構 `mid_section_defect_service.py`：以 `LineageEngine` 取代 `_bfs_split_chain()` + `_fetch_merge_sources()` + `_resolve_full_genealogy()`；以 `EventFetcher` 取代 `_fetch_upstream_history()`。 +- 重構 `query_tool_service.py`：以 `QueryBuilder` bind params 全面取代 `_build_in_filter()` 字串拼接；加入 route-level rate limit 和 cache 對齊 mid-section-defect 既有模式。 +- 新增 SQL 檔案： + - `sql/lineage/split_ancestors.sql`（CONNECT BY NOCYCLE 實作，檔案內包含 recursive WITH 替代寫法作為 Oracle 版本兼容備註） + - `sql/lineage/merge_sources.sql`（從 `sql/mid_section_defect/merge_lookup.sql` 遷移） +- 廢棄 SQL 檔案（標記 deprecated，保留一個版本後刪除）： + - `sql/mid_section_defect/genealogy_records.sql`（48M row HM_LOTMOVEOUT 全掃描不再需要） + - `sql/mid_section_defect/split_chain.sql`（由 lineage CONNECT BY 取代） +- 為 query-tool 的 `lot_split_merge_history.sql` 加入雙模式查詢： + - **fast mode**（預設）：`TXNDATE >= ADD_MONTHS(SYSDATE, -6)` + `FETCH FIRST 500 ROWS ONLY`——涵蓋近半年追溯，回應 <5s + - **full mode**：前端傳入 `full_history=true` 時不加時間窗，保留完整歷史追溯能力，走 `read_sql_df_slow` (120s timeout) + - query-tool route 新增 `full_history` boolean query param，service 依此選擇 SQL variant + +## Capabilities + +### New Capabilities + +- `lineage-engine-core`: 統一 LOT 血緣解析引擎。提供 `resolve_split_ancestors()`（CONNECT BY NOCYCLE，`LEVEL <= 20` 上限）、`resolve_merge_sources()`、`resolve_full_genealogy()` 三個公用函數。全部使用 `QueryBuilder` bind params，支援批次 IN 分段（`ORACLE_IN_BATCH_SIZE=1000`）。函數簽名設計為 profile-agnostic，接受 `container_ids: List[str]` 並回傳字典結構，不綁定特定頁面邏輯。 +- `event-fetcher-unified`: 統一事件查詢層，封裝 cache key 生成（格式: `evt:{domain}:{sorted_cids_hash}`）、L1/L2 layered cache（對齊 `core/cache.py` LayeredCache 模式）、rate limit bucket 配置（對齊 `configured_rate_limit()` 模式）。domain 包含 `history`、`materials`、`rejects`、`holds`、`jobs`、`upstream_history`。 +- `query-tool-safety-hardening`: 修復 query-tool SQL 注入風險——`_build_in_filter()` 和 `_build_in_clause()` 全面改用 `QueryBuilder.add_in_condition()`，消除 `read_sql_df(sql, {})` 空 params 模式；加入 route-level rate limit（對齊 `configured_rate_limit()` 模式：resolve 10/min, history 20/min, association 20/min）和 response cache（L2 Redis, 60s TTL）。 + +### Modified Capabilities + +- `cache-indexed-query-acceleration`: mid-section-defect 的 genealogy 查詢從 Python BFS 多輪 + HM_LOTMOVEOUT 全掃描改為 CONNECT BY 單輪 + 索引查詢。 +- `oracle-query-fragment-governance`: `_build_in_filter()` / `_build_in_clause()` 廢棄，統一收斂到 `QueryBuilder.add_in_condition()`。新增 `sql/lineage/` 目錄遵循既有 SQLLoader 慣例。 + +## Impact + +- **Affected code**: + - 新建: `src/mes_dashboard/services/lineage_engine.py`, `src/mes_dashboard/sql/lineage/split_ancestors.sql`, `src/mes_dashboard/sql/lineage/merge_sources.sql` + - 重構: `src/mes_dashboard/services/mid_section_defect_service.py` (1194L), `src/mes_dashboard/services/query_tool_service.py` (1329L), `src/mes_dashboard/routes/query_tool_routes.py` + - 廢棄: `src/mes_dashboard/sql/mid_section_defect/genealogy_records.sql`, `src/mes_dashboard/sql/mid_section_defect/split_chain.sql` (由 lineage 模組取代，標記 deprecated 保留一版) + - 修改: `src/mes_dashboard/sql/query_tool/lot_split_merge_history.sql` (加時間窗 + row limit) +- **Runtime/deploy**: 無新依賴，仍為 Flask/Gunicorn + Oracle + Redis。DB query pattern 改變但 connection pool 設定不變。 +- **APIs/pages**: `/query-tool` 和 `/mid-section-defect` 既有 API contract 向下相容——URL、輸入輸出格式、HTTP status code 均不變，純內部實作替換。向下相容的擴展：query-tool API 新增 rate limit header（`Retry-After`，對齊 `rate_limit.py` 既有實作）；query-tool split-merge history 新增可選 `full_history` query param（預設 false = fast mode，不傳時行為與舊版等價）。 +- **Performance**: 見下方 Verification 章節的量化驗收基準。 +- **Security**: query-tool IN clause SQL injection 風險消除。所有 `_build_in_filter()` / `_build_in_clause()` 呼叫點改為 `QueryBuilder.add_in_condition()`。 +- **Testing**: 需新增 LineageEngine 單元測試，並建立 golden test 比對 BFS vs CONNECT BY 結果一致性。既有 mid-section-defect 和 query-tool 測試需更新 mock 路徑。 + +## Verification + +效能驗收基準——所有指標須在以下條件下量測： + +**測試資料規模**: +- LOT 血緣樹: 目標 seed lot 具備 ≥3 層 split depth、≥50 ancestor nodes、至少 1 條 merge path +- mid-section-defect: 使用 TMTT detection 產出 ≥10 seed lots 的日期範圍查詢 +- query-tool: resolve 結果 ≥20 lots 的 work order 查詢 + +**驗收指標**（冷查詢 = cache miss，熱查詢 = L2 Redis hit）: + +| 指標 | 現況 (P95) | 目標 (P95) | 條件 | +|------|-----------|-----------|------| +| mid-section-defect genealogy（冷） | 30-120s | ≤8s | CONNECT BY 單輪，≥50 ancestor nodes | +| mid-section-defect genealogy（熱） | 3-5s (L2 hit) | ≤1s | Redis cache hit | +| query-tool lot_split_merge_history fast mode（冷） | 無上限（可 >120s timeout） | ≤5s | 時間窗 6 個月 + FETCH FIRST 500 ROWS | +| query-tool lot_split_merge_history full mode（冷） | 同上 | ≤60s | 無時間窗，走 `read_sql_df_slow` 120s timeout | +| LineageEngine.resolve_split_ancestors（冷） | N/A (新模組) | ≤3s | ≥50 ancestor nodes, CONNECT BY | +| DB connection 佔用時間 | 3-16 round-trips × 0.5-2s each | 單次 ≤3s | 單一 CONNECT BY 查詢 | + +**安全驗收**: +- `_build_in_filter()` 和 `_build_in_clause()` 零引用（grep 確認） +- 所有含使用者輸入的查詢（resolve_by_lot_id, resolve_by_serial_number, resolve_by_work_order 等）必須使用 `QueryBuilder` bind params，不可字串拼接。純靜態 SQL（無使用者輸入）允許空 params + +**結果一致性驗收**: +- Golden test: 選取 ≥5 個已知血緣結構的 LOT，比對 BFS vs CONNECT BY 輸出的 `child_to_parent` 和 `cid_to_name` 結果集合完全一致 + +## Non-Goals + +- 前端 UI 改動不在此變更範圍內（前端分段載入和漸進式 UX 列入後續 `trace-progressive-ui` 變更）。 +- 不新增任何 API endpoint——既有 API contract 向下相容（僅新增可選 query param `full_history` 作為擴展）。新增 endpoint 由後續 `trace-progressive-ui` 負責。 +- 不改動 DB schema、不建立 materialized view、不使用 PARALLEL hints——所有最佳化在應用層（SQL 改寫 + Python 重構 + Redis cache）完成。 +- 不改動其他頁面（wip-detail、lot-detail 等）的查詢邏輯——`LineageEngine` 設計為可擴展，但本變更僅接入兩個目標頁面。 +- 不使用 Oracle PARALLEL hints（在 connection pool 環境下行為不可預測，不做為最佳化手段）。 + +## Dependencies + +- 無前置依賴。本變更可獨立實施。 +- 後續 `trace-progressive-ui` 依賴本變更完成後的 `LineageEngine` 和 `EventFetcher` 模組。 + +## Risks + +| 風險 | 緩解 | +|------|------| +| CONNECT BY 遇超大血緣樹（>10000 ancestors）效能退化 | `LEVEL <= 20` 上限 + `NOCYCLE` 防循環；與目前 BFS `bfs_round > 20` 等效。若 Oracle 19c 執行計劃不佳，SQL 檔案內含 recursive `WITH` 替代寫法可快速切換 | +| 血緣結果與 BFS 版本不一致（regression） | 建立 golden test：用 ≥5 個已知 LOT 比對 BFS vs CONNECT BY 輸出，CI gate 確保結果集合完全一致 | +| 重構範圍橫跨兩個大 service（2500+ 行） | 分階段：先重構 mid-section-defect（有 cache+lock 保護，regression 風險較低），再做 query-tool | +| `genealogy_records.sql` 廢棄後遺漏引用 | grep 全域搜索確認無其他引用點；SQL file 標記 deprecated 保留一個版本後刪除 | +| query-tool 新增 rate limit 影響使用者體驗 | 預設值寬鬆（resolve 10/min, history 20/min），與 mid-section-defect 既有 rate limit 對齊，回應包含 `Retry-After` header | +| `QueryBuilder` 取代 `_build_in_filter()` 時漏改呼叫點 | grep 搜索 `_build_in_filter` 和 `_build_in_clause` 所有引用，逐一替換並確認 0 殘留引用 | diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/cache-indexed-query-acceleration/spec.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/cache-indexed-query-acceleration/spec.md new file mode 100644 index 0000000..5ece204 --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/cache-indexed-query-acceleration/spec.md @@ -0,0 +1,18 @@ +## ADDED Requirements + +### Requirement: Mid-section defect genealogy SHALL use CONNECT BY instead of Python BFS +The mid-section-defect genealogy resolution SHALL use `LineageEngine.resolve_full_genealogy()` (CONNECT BY NOCYCLE) instead of the existing `_bfs_split_chain()` Python BFS implementation. + +#### Scenario: Genealogy cold query performance +- **WHEN** mid-section-defect analysis executes genealogy resolution with cache miss +- **THEN** `LineageEngine.resolve_split_ancestors()` SHALL be called (single CONNECT BY query) +- **THEN** response time SHALL be ≤8s (P95) for ≥50 ancestor nodes +- **THEN** Python BFS `_bfs_split_chain()` SHALL NOT be called + +#### Scenario: Genealogy hot query performance +- **WHEN** mid-section-defect analysis executes genealogy resolution with L2 Redis cache hit +- **THEN** response time SHALL be ≤1s (P95) + +#### Scenario: Golden test result equivalence +- **WHEN** golden test runs with ≥5 known LOTs +- **THEN** CONNECT BY output (`child_to_parent`, `cid_to_name`) SHALL be identical to BFS output for the same inputs diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/event-fetcher-unified/spec.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/event-fetcher-unified/spec.md new file mode 100644 index 0000000..bf5a0d3 --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/event-fetcher-unified/spec.md @@ -0,0 +1,20 @@ +## ADDED Requirements + +### Requirement: EventFetcher SHALL provide unified cached event querying across domains +`EventFetcher` SHALL encapsulate batch event queries with L1/L2 layered cache and rate limit bucket configuration, supporting domains: `history`, `materials`, `rejects`, `holds`, `jobs`, `upstream_history`. + +#### Scenario: Cache miss for event domain query +- **WHEN** `EventFetcher` is called for a domain with container IDs and no cache exists +- **THEN** the domain query SHALL execute against Oracle via `read_sql_df()` +- **THEN** the result SHALL be stored in L2 Redis cache with key format `evt:{domain}:{sorted_cids_hash}` +- **THEN** L1 memory cache SHALL also be populated (aligned with `core/cache.py` LayeredCache pattern) + +#### Scenario: Cache hit for event domain query +- **WHEN** `EventFetcher` is called for a domain and L2 Redis cache contains a valid entry +- **THEN** the cached result SHALL be returned without executing Oracle query +- **THEN** DB connection pool SHALL NOT be consumed + +#### Scenario: Rate limit bucket per domain +- **WHEN** `EventFetcher` is used from a route handler +- **THEN** each domain SHALL have a configurable rate limit bucket aligned with `configured_rate_limit()` pattern +- **THEN** rate limit configuration SHALL be overridable via environment variables diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/lineage-engine-core/spec.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/lineage-engine-core/spec.md new file mode 100644 index 0000000..5ddafdf --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/lineage-engine-core/spec.md @@ -0,0 +1,57 @@ +## ADDED Requirements + +### Requirement: LineageEngine SHALL provide unified split ancestor resolution via CONNECT BY NOCYCLE +`LineageEngine.resolve_split_ancestors()` SHALL accept a list of container IDs and return the complete split ancestry graph using a single Oracle `CONNECT BY NOCYCLE` query on `DW_MES_CONTAINER.SPLITFROMID`. + +#### Scenario: Normal split chain resolution +- **WHEN** `resolve_split_ancestors()` is called with a list of container IDs +- **THEN** a single SQL query using `CONNECT BY NOCYCLE` SHALL be executed against `DW_MES_CONTAINER` +- **THEN** the result SHALL include a `child_to_parent` mapping and a `cid_to_name` mapping for all discovered ancestor nodes +- **THEN** the traversal depth SHALL be limited to `LEVEL <= 20` (equivalent to existing BFS `bfs_round > 20` guard) + +#### Scenario: Large input batch exceeding Oracle IN clause limit +- **WHEN** the input `container_ids` list exceeds `ORACLE_IN_BATCH_SIZE` (1000) +- **THEN** `QueryBuilder.add_in_condition()` SHALL batch the IDs and combine results +- **THEN** all bind parameters SHALL use `QueryBuilder.params` (no string concatenation) + +#### Scenario: Cyclic split references in data +- **WHEN** `DW_MES_CONTAINER.SPLITFROMID` contains cyclic references +- **THEN** `NOCYCLE` SHALL prevent infinite traversal +- **THEN** the query SHALL return all non-cyclic ancestors up to `LEVEL <= 20` + +#### Scenario: CONNECT BY performance regression +- **WHEN** Oracle 19c execution plan for `CONNECT BY NOCYCLE` performs worse than expected +- **THEN** the SQL file SHALL contain a commented-out recursive `WITH` (recursive subquery factoring) alternative that can be swapped in without code changes + +### Requirement: LineageEngine SHALL provide unified merge source resolution +`LineageEngine.resolve_merge_sources()` SHALL accept a list of container IDs and return merge source mappings from `DW_MES_PJ_COMBINEDASSYLOTS`. + +#### Scenario: Merge source lookup +- **WHEN** `resolve_merge_sources()` is called with container IDs +- **THEN** the result SHALL include `{cid: [merge_source_cid, ...]}` for all containers that have merge sources +- **THEN** all queries SHALL use `QueryBuilder` bind params + +### Requirement: LineageEngine SHALL provide combined genealogy resolution +`LineageEngine.resolve_full_genealogy()` SHALL combine split ancestors and merge sources into a complete genealogy graph. + +#### Scenario: Full genealogy for a set of seed lots +- **WHEN** `resolve_full_genealogy()` is called with seed container IDs +- **THEN** split ancestors SHALL be resolved first via `resolve_split_ancestors()` +- **THEN** merge sources SHALL be resolved for all discovered ancestor nodes +- **THEN** the combined result SHALL be equivalent to the existing `_resolve_full_genealogy()` output in `mid_section_defect_service.py` + +### Requirement: LineageEngine functions SHALL be profile-agnostic +All `LineageEngine` public functions SHALL accept `container_ids: List[str]` and return dictionary structures without binding to any specific page logic. + +#### Scenario: Reuse from different pages +- **WHEN** a new page (e.g., wip-detail) needs lineage resolution +- **THEN** it SHALL be able to call `LineageEngine` functions directly without modification +- **THEN** no page-specific logic (profile, TMTT detection, etc.) SHALL exist in `LineageEngine` + +### Requirement: LineageEngine SQL files SHALL reside in `sql/lineage/` directory +New SQL files SHALL follow the existing `SQLLoader` convention under `src/mes_dashboard/sql/lineage/`. + +#### Scenario: SQL file organization +- **WHEN** `LineageEngine` executes queries +- **THEN** `split_ancestors.sql` and `merge_sources.sql` SHALL be loaded via `SQLLoader.load_with_params("lineage/split_ancestors", ...)` +- **THEN** the SQL files SHALL NOT reference `HM_LOTMOVEOUT` (48M row table no longer needed for genealogy) diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/oracle-query-fragment-governance/spec.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/oracle-query-fragment-governance/spec.md new file mode 100644 index 0000000..0400f5e --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/oracle-query-fragment-governance/spec.md @@ -0,0 +1,23 @@ +## ADDED Requirements + +### Requirement: Lineage SQL fragments SHALL be centralized in `sql/lineage/` directory +Split ancestor and merge source SQL queries SHALL be defined in `sql/lineage/` and shared across services via `SQLLoader`. + +#### Scenario: Mid-section-defect lineage query +- **WHEN** `mid_section_defect_service.py` needs split ancestry or merge source data +- **THEN** it SHALL call `LineageEngine` which loads SQL from `sql/lineage/split_ancestors.sql` and `sql/lineage/merge_sources.sql` +- **THEN** it SHALL NOT use `sql/mid_section_defect/split_chain.sql` or `sql/mid_section_defect/genealogy_records.sql` + +#### Scenario: Deprecated SQL file handling +- **WHEN** `sql/mid_section_defect/genealogy_records.sql` and `sql/mid_section_defect/split_chain.sql` are deprecated +- **THEN** the files SHALL be marked with a deprecated comment at the top +- **THEN** grep SHALL confirm zero `SQLLoader.load` references to these files +- **THEN** the files SHALL be retained for one version before deletion + +### Requirement: All user-input SQL queries SHALL use QueryBuilder bind params +`_build_in_filter()` and `_build_in_clause()` in `query_tool_service.py` SHALL be fully replaced by `QueryBuilder.add_in_condition()`. + +#### Scenario: Complete migration to QueryBuilder +- **WHEN** the refactoring is complete +- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results +- **THEN** all queries involving user-supplied values SHALL use `QueryBuilder.params` diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/query-tool-safety-hardening/spec.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/query-tool-safety-hardening/spec.md new file mode 100644 index 0000000..0e7d3e8 --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/specs/query-tool-safety-hardening/spec.md @@ -0,0 +1,57 @@ +## ADDED Requirements + +### Requirement: query-tool resolve functions SHALL use QueryBuilder bind params for all user input +All `resolve_lots()` family functions (`_resolve_by_lot_id`, `_resolve_by_serial_number`, `_resolve_by_work_order`) SHALL use `QueryBuilder.add_in_condition()` with bind parameters instead of `_build_in_filter()` string concatenation. + +#### Scenario: Lot resolve with user-supplied values +- **WHEN** a resolve function receives user-supplied lot IDs, serial numbers, or work order names +- **THEN** the SQL query SHALL use `:p0, :p1, ...` bind parameters via `QueryBuilder` +- **THEN** `read_sql_df()` SHALL receive `builder.params` (never an empty `{}` dict for queries with user input) +- **THEN** `_build_in_filter()` and `_build_in_clause()` SHALL NOT be called + +#### Scenario: Pure static SQL without user input +- **WHEN** a query contains no user-supplied values (e.g., static lookups) +- **THEN** empty params `{}` is acceptable +- **THEN** no `_build_in_filter()` SHALL be used + +#### Scenario: Zero residual references to deprecated functions +- **WHEN** the refactoring is complete +- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results across the entire codebase + +### Requirement: query-tool routes SHALL apply rate limiting +All query-tool API endpoints SHALL apply per-client rate limiting using the existing `configured_rate_limit` mechanism. + +#### Scenario: Resolve endpoint rate limit exceeded +- **WHEN** a client sends more than 10 requests to query-tool resolve endpoints within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header +- **THEN** the resolve service function SHALL NOT be called + +#### Scenario: History endpoint rate limit exceeded +- **WHEN** a client sends more than 20 requests to query-tool history endpoints within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header + +#### Scenario: Association endpoint rate limit exceeded +- **WHEN** a client sends more than 20 requests to query-tool association endpoints within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header + +### Requirement: query-tool routes SHALL apply response caching +High-cost query-tool endpoints SHALL cache responses in L2 Redis. + +#### Scenario: Resolve result caching +- **WHEN** a resolve request succeeds +- **THEN** the response SHALL be cached in L2 Redis with TTL = 60s +- **THEN** subsequent identical requests within TTL SHALL return cached result without Oracle query + +### Requirement: lot_split_merge_history SHALL support fast and full query modes +The `lot_split_merge_history.sql` query SHALL support two modes to balance traceability completeness vs performance. + +#### Scenario: Fast mode (default) +- **WHEN** `full_history` query parameter is absent or `false` +- **THEN** the SQL SHALL include `TXNDATE >= ADD_MONTHS(SYSDATE, -6)` time window and `FETCH FIRST 500 ROWS ONLY` +- **THEN** query response time SHALL be ≤5s (P95) + +#### Scenario: Full mode +- **WHEN** `full_history=true` query parameter is provided +- **THEN** the SQL SHALL NOT include time window restriction +- **THEN** the query SHALL use `read_sql_df_slow` (120s timeout) +- **THEN** query response time SHALL be ≤60s (P95) diff --git a/openspec/changes/archive/2026-02-12-unified-lineage-engine/tasks.md b/openspec/changes/archive/2026-02-12-unified-lineage-engine/tasks.md new file mode 100644 index 0000000..9534628 --- /dev/null +++ b/openspec/changes/archive/2026-02-12-unified-lineage-engine/tasks.md @@ -0,0 +1,57 @@ +## Phase 1: LineageEngine 模組建立 + +- [x] 1.1 建立 `src/mes_dashboard/sql/lineage/split_ancestors.sql`（CONNECT BY NOCYCLE，含 recursive WITH 註解替代方案） +- [x] 1.2 建立 `src/mes_dashboard/sql/lineage/merge_sources.sql`（從 `mid_section_defect/merge_lookup.sql` 遷移，改用 `{{ FINISHED_NAME_FILTER }}` 結構參數） +- [x] 1.3 建立 `src/mes_dashboard/services/lineage_engine.py`：`resolve_split_ancestors()`、`resolve_merge_sources()`、`resolve_full_genealogy()` 三個公用函數，使用 `QueryBuilder` bind params + `ORACLE_IN_BATCH_SIZE=1000` 分批 +- [x] 1.4 LineageEngine 單元測試：mock `read_sql_df` 驗證 batch 分割、dict 回傳結構、LEVEL <= 20 防護 + +## Phase 2: mid-section-defect 切換到 LineageEngine + +- [x] 2.1 在 `mid_section_defect_service.py` 中以 `LineageEngine.resolve_split_ancestors()` 取代 `_bfs_split_chain()` +- [x] 2.2 以 `LineageEngine.resolve_merge_sources()` 取代 `_fetch_merge_sources()` +- [x] 2.3 以 `LineageEngine.resolve_full_genealogy()` 取代 `_resolve_full_genealogy()` +- [x] 2.4 Golden test：選取 ≥5 個已知血緣結構 LOT，比對 BFS vs CONNECT BY 輸出的 `child_to_parent` 和 `cid_to_name` 結果集合完全一致 +- [x] 2.5 標記 `sql/mid_section_defect/genealogy_records.sql` 和 `sql/mid_section_defect/split_chain.sql` 為 deprecated（檔案頂部加 `-- DEPRECATED: replaced by sql/lineage/split_ancestors.sql`） + +## Phase 3: query-tool SQL injection 修復 + +- [x] 3.1 建立 `sql/query_tool/lot_resolve_id.sql`、`lot_resolve_serial.sql`、`lot_resolve_work_order.sql` SQL 檔案（從 inline SQL 遷移到 SQLLoader 管理） +- [x] 3.2 修復 `_resolve_by_lot_id()`：`_build_in_filter()` → `QueryBuilder.add_in_condition()` + `SQLLoader.load_with_params()` + `read_sql_df(sql, builder.params)` +- [x] 3.3 修復 `_resolve_by_serial_number()`：同上模式 +- [x] 3.4 修復 `_resolve_by_work_order()`：同上模式 +- [x] 3.5 修復 `get_lot_history()` 內部 IN 子句：改用 `QueryBuilder` +- [x] 3.6 修復 lot-associations 查詢路徑（`get_lot_materials()` / `get_lot_rejects()` / `get_lot_holds()` / `get_lot_splits()` / `get_lot_jobs()`）中涉及使用者輸入的 IN 子句：改用 `QueryBuilder` +- [x] 3.7 修復 `lot_split_merge_history` 查詢：改用 `QueryBuilder` +- [x] 3.8 刪除 `_build_in_filter()` 和 `_build_in_clause()` 函數 +- [x] 3.9 驗證：`grep -r "_build_in_filter\|_build_in_clause" src/` 回傳 0 結果 +- [x] 3.10 更新既有 query-tool 路由測試的 mock 路徑 + +## Phase 4: query-tool rate limit + cache + +- [x] 4.1 在 `query_tool_routes.py` 為 `/resolve` 加入 `configured_rate_limit(bucket='query-tool-resolve', default_max_attempts=10, default_window_seconds=60)` +- [x] 4.2 為 `/lot-history` 加入 `configured_rate_limit(bucket='query-tool-history', default_max_attempts=20, default_window_seconds=60)` +- [x] 4.3 為 `/lot-associations` 加入 `configured_rate_limit(bucket='query-tool-association', default_max_attempts=20, default_window_seconds=60)` +- [x] 4.4 為 `/adjacent-lots` 加入 `configured_rate_limit(bucket='query-tool-adjacent', default_max_attempts=20, default_window_seconds=60)` +- [x] 4.5 為 `/equipment-period` 加入 `configured_rate_limit(bucket='query-tool-equipment', default_max_attempts=5, default_window_seconds=60)` +- [x] 4.6 為 `/export-csv` 加入 `configured_rate_limit(bucket='query-tool-export', default_max_attempts=3, default_window_seconds=60)` +- [x] 4.7 為 resolve 結果加入 L2 Redis cache（key=`qt:resolve:{input_type}:{values_hash}`, TTL=60s） + +## Phase 5: lot_split_merge_history fast/full 雙模式 + +- [x] 5.1 修改 `sql/query_tool/lot_split_merge_history.sql`：加入 `{{ TIME_WINDOW }}` 和 `{{ ROW_LIMIT }}` 結構參數 +- [x] 5.2 在 `query_tool_service.py` 中根據 `full_history` 參數選擇 SQL variant（fast: `AND h.TXNDATE >= ADD_MONTHS(SYSDATE, -6)` + `FETCH FIRST 500 ROWS ONLY`，full: 無限制 + `read_sql_df_slow`） +- [x] 5.3 在 `query_tool_routes.py` 的 `/api/query-tool/lot-associations?type=splits` 路徑加入 `full_history` query param 解析，並傳遞到 split-merge-history 查詢 +- [x] 5.4 路由測試：驗證 fast mode（預設）和 full mode（`full_history=true`）的行為差異 + +## Phase 6: EventFetcher 模組建立 + +- [x] 6.1 建立 `src/mes_dashboard/services/event_fetcher.py`：`fetch_events(container_ids, domain)` + cache key 生成 + rate limit config +- [x] 6.2 遷移 `mid_section_defect_service.py` 的 `_fetch_upstream_history()` 到 `EventFetcher.fetch_events(cids, "upstream_history")` +- [x] 6.3 遷移 query-tool event fetch paths 到 `EventFetcher`（`get_lot_history`、`get_lot_associations` 的 DB 查詢部分） +- [x] 6.4 EventFetcher 單元測試：mock DB 驗證 cache key 格式、rate limit config、domain 分支 + +## Phase 7: 清理與驗證 + +- [x] 7.1 確認 `genealogy_records.sql` 和 `split_chain.sql` 無活躍引用（`grep -r` 確認），保留 deprecated 標記 +- [x] 7.2 確認所有含使用者輸入的查詢使用 `QueryBuilder` bind params（grep `read_sql_df` 呼叫點逐一確認） +- [x] 7.3 執行完整 query-tool 和 mid-section-defect 路由測試，確認無 regression diff --git a/openspec/changes/trace-progressive-ui/.openspec.yaml b/openspec/changes/trace-progressive-ui/.openspec.yaml new file mode 100644 index 0000000..05ac962 --- /dev/null +++ b/openspec/changes/trace-progressive-ui/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +status: proposal diff --git a/openspec/changes/trace-progressive-ui/design.md b/openspec/changes/trace-progressive-ui/design.md new file mode 100644 index 0000000..b5f9b42 --- /dev/null +++ b/openspec/changes/trace-progressive-ui/design.md @@ -0,0 +1,446 @@ +## Context + +`unified-lineage-engine` 完成後，後端追溯管線從 30-120 秒降至 3-8 秒。但目前的 UX 模式仍是黑盒等待——mid-section-defect 的 `/analysis` GET 一次回傳全部結果（KPI + charts + trend + genealogy_status），query-tool 雖有手動順序（resolve → history → association）但 lineage 查詢仍在批次載入。 + +既有前端架構： +- mid-section-defect: `App.vue` 用 `Promise.all([apiGet('/analysis'), loadDetail(1)])` 並行呼叫，`loading.querying` 單一布林控制整頁 loading state +- query-tool: `useQueryToolData.js` composable 管理 `loading.resolving / .history / .association / .equipment`，各自獨立但無分段進度 +- 共用: `useAutoRefresh` (jittered interval + abort signal), `usePaginationState`, `apiGet/apiPost` (timeout + abort), `useQueryState` (URL sync) +- API 模式: `apiGet/apiPost` 支援 `signal: AbortSignal` + `timeout`，錯誤物件含 `error.retryAfterSeconds` + +## Goals / Non-Goals + +**Goals:** +- 新增 `/api/trace/*` 三段式 API（seed-resolve → lineage → events），通過 `profile` 參數區分頁面行為 +- 建立 `useTraceProgress` composable 封裝三段式 sequential fetch + reactive state +- mid-section-defect 漸進渲染: seed lots 先出 → 血緣 → KPI/charts fade-in +- query-tool lineage tab 改為 on-demand（點擊單一 lot 後才查血緣） +- 保持 `/api/mid-section-defect/analysis` GET endpoint 向下相容 +- 刪除 pre-Vite dead code `static/js/query-tool.js` + +**Non-Goals:** +- 不實作 SSE / WebSocket（gunicorn sync workers 限制） +- 不新增 Celery/RQ 任務隊列 +- 不改動追溯計算邏輯（由 `unified-lineage-engine` 負責） +- 不改動 defect attribution 演算法 +- 不改動 equipment-period 查詢 + +## Decisions + +### D1: trace_routes.py Blueprint 架構 + +**選擇**: 單一 Blueprint `trace_bp`，三個 route handler + profile dispatch +**替代方案**: 每個 profile 獨立 Blueprint（`trace_msd_bp`, `trace_qt_bp`） +**理由**: +- 三個 endpoint 的 request/response 結構統一，僅內部呼叫邏輯依 profile 分支 +- 獨立 Blueprint 會重複 rate limit / cache / error handling boilerplate +- profile 驗證集中在一處（`_validate_profile()`），新增 profile 只需加 if 分支 + +**路由設計**: +```python +trace_bp = Blueprint('trace', __name__, url_prefix='/api/trace') + +@trace_bp.route('/seed-resolve', methods=['POST']) +@configured_rate_limit(bucket='trace-seed', default_max_attempts=10, default_window_seconds=60) +def seed_resolve(): + body = request.get_json() + profile = body.get('profile') + params = body.get('params', {}) + # profile dispatch → _seed_resolve_query_tool(params) or _seed_resolve_msd(params) + # return jsonify({ "stage": "seed-resolve", "seeds": [...], "seed_count": N, "cache_key": "trace:{hash}" }) + +@trace_bp.route('/lineage', methods=['POST']) +@configured_rate_limit(bucket='trace-lineage', default_max_attempts=10, default_window_seconds=60) +def lineage(): + body = request.get_json() + container_ids = body.get('container_ids', []) + # call LineageEngine.resolve_full_genealogy(container_ids) + # return jsonify({ "stage": "lineage", "ancestors": {...}, "merges": {...}, "total_nodes": N }) + +@trace_bp.route('/events', methods=['POST']) +@configured_rate_limit(bucket='trace-events', default_max_attempts=15, default_window_seconds=60) +def events(): + body = request.get_json() + container_ids = body.get('container_ids', []) + domains = body.get('domains', []) + profile = body.get('profile') + # call EventFetcher for each domain + # if profile == 'mid_section_defect': run aggregation + # return jsonify({ "stage": "events", "results": {...}, "aggregation": {...} | null }) +``` + +**Profile dispatch 內部函數**: +``` +_seed_resolve_query_tool(params) → 呼叫 query_tool_service 既有 resolve 邏輯 +_seed_resolve_msd(params) → 呼叫 mid_section_defect_service TMTT 偵測邏輯 +_aggregate_msd(events_data) → mid-section-defect 專屬 aggregation (KPI, charts, trend) +``` + +**Cache 策略**: +- seed-resolve: `trace:seed:{profile}:{params_hash}`, TTL=300s +- lineage: `trace:lineage:{sorted_cids_hash}`, TTL=300s（profile-agnostic，因為 lineage 不依賴 profile） +- events: `trace:evt:{profile}:{domains_hash}:{sorted_cids_hash}`, TTL=300s +- 使用 `LayeredCache` L2 Redis（對齊 `core/cache.py` 既有模式） +- cache key hash: `hashlib.md5(sorted(values).encode()).hexdigest()[:12]` + +**錯誤處理統一模式**: +```python +def _make_stage_error(stage, code, message, status=400): + return jsonify({"error": message, "code": code}), status + +# Timeout 處理: 每個 stage 內部用 read_sql_df() 的 55s call_timeout +# 若超時: return _make_stage_error(stage, f"{STAGE}_TIMEOUT", "...", 504) +``` + +### D2: useTraceProgress composable 設計 + +**選擇**: 新建 `frontend/src/shared-composables/useTraceProgress.js`，封裝 sequential fetch + reactive stage state +**替代方案**: 直接在各頁面 App.vue 內實作分段 fetch +**理由**: +- 兩個頁面共用相同的三段式 fetch 邏輯 +- 將 stage 狀態管理抽離，頁面只需關注渲染邏輯 +- 對齊既有 `shared-composables/` 目錄結構 + +**Composable 簽名**: +```javascript +export function useTraceProgress({ profile, buildParams }) { + // --- Reactive State --- + const current_stage = ref(null) // 'seed-resolve' | 'lineage' | 'events' | null + const completed_stages = ref([]) // ['seed-resolve', 'lineage'] + const stage_results = reactive({ + seed: null, // { seeds: [], seed_count: N, cache_key: '...' } + lineage: null, // { ancestors: {...}, merges: {...}, total_nodes: N } + events: null, // { results: {...}, aggregation: {...} } + }) + const stage_errors = reactive({ + seed: null, // { code: '...', message: '...' } + lineage: null, + events: null, + }) + const is_running = ref(false) + + // --- Methods --- + async function execute(params) // 執行三段式 fetch + function reset() // 清空所有 state + function abort() // 中止進行中的 fetch + + return { + current_stage, + completed_stages, + stage_results, + stage_errors, + is_running, + execute, + reset, + abort, + } +} +``` + +**Sequential fetch 邏輯**: +```javascript +async function execute(params) { + reset() + is_running.value = true + const abortCtrl = new AbortController() + + try { + // Stage 1: seed-resolve + current_stage.value = 'seed-resolve' + const seedResult = await apiPost('/api/trace/seed-resolve', { + profile, + params, + }, { timeout: 60000, signal: abortCtrl.signal }) + stage_results.seed = seedResult.data + completed_stages.value.push('seed-resolve') + + if (!seedResult.data?.seeds?.length) return // 無 seed，不繼續 + + // Stage 2: lineage + current_stage.value = 'lineage' + const cids = seedResult.data.seeds.map(s => s.container_id) + const lineageResult = await apiPost('/api/trace/lineage', { + profile, + container_ids: cids, + cache_key: seedResult.data.cache_key, + }, { timeout: 60000, signal: abortCtrl.signal }) + stage_results.lineage = lineageResult.data + completed_stages.value.push('lineage') + + // Stage 3: events + current_stage.value = 'events' + const allCids = _collectAllCids(cids, lineageResult.data) + const eventsResult = await apiPost('/api/trace/events', { + profile, + container_ids: allCids, + domains: _getDomainsForProfile(profile), + cache_key: seedResult.data.cache_key, + }, { timeout: 60000, signal: abortCtrl.signal }) + stage_results.events = eventsResult.data + completed_stages.value.push('events') + + } catch (err) { + if (err?.name === 'AbortError') return + // 記錄到當前 stage 的 error state + const stage = current_stage.value + if (stage) stage_errors[_stageKey(stage)] = { code: err.errorCode, message: err.message } + } finally { + current_stage.value = null + is_running.value = false + } +} +``` + +**設計重點**: +- `stage_results` 為 reactive object，每個 stage 完成後立即賦值，觸發依賴該 stage 的 UI 更新 +- 錯誤不拋出到頁面——記錄在 `stage_errors` 中，已完成的 stage 結果保留 +- `abort()` 方法供 `useAutoRefresh` 在新一輪 refresh 前中止上一輪 +- `profile` 為建構時注入（不可變），`params` 為執行時傳入（每次查詢可變） +- `cache_key` 在 stage 間傳遞，用於 logging correlation + +### D3: mid-section-defect 漸進渲染策略 + +**選擇**: 分段渲染 + skeleton placeholders + CSS fade-in transition +**替代方案**: 保持一次性渲染（等全部 stage 完成） +**理由**: +- seed stage ≤3s 可先顯示 seed lots 數量和基本資訊 +- lineage + events 完成後再填入 KPI/charts，使用者感受到進度 +- skeleton placeholders 避免 layout shift（chart container 預留固定高度） + +**App.vue 查詢流程改造**: +```javascript +// Before (current) +async function loadAnalysis() { + loading.querying = true + const [summaryResult] = await Promise.all([ + apiGet('/api/mid-section-defect/analysis', { params, timeout: 120000, signal }), + loadDetail(1, signal), + ]) + analysisData.value = summaryResult.data // 一次全部更新 + loading.querying = false +} + +// After (progressive) +const trace = useTraceProgress({ profile: 'mid_section_defect' }) + +async function loadAnalysis() { + const params = buildFilterParams() + // 分段 fetch（seed → lineage → events+aggregation） + await trace.execute(params) + // Detail 仍用舊 endpoint 分頁（不走分段 API） + await loadDetail(1) +} +``` + +**渲染層對應**: +``` +trace.completed_stages 包含 'seed-resolve' + → 顯示 seed lots 數量 badge + 基本 filter feedback + → KPI cards / charts / trend 顯示 skeleton + +trace.completed_stages 包含 'lineage' + → 顯示 genealogy_status（ancestor 數量） + → KPI/charts 仍為 skeleton + +trace.completed_stages 包含 'events' + → trace.stage_results.events.aggregation 不為 null + → KPI cards 以 fade-in 填入數值 + → Pareto charts 以 fade-in 渲染 + → Trend chart 以 fade-in 渲染 +``` + +**Skeleton Placeholder 規格**: +- KpiCards: 6 個固定高度 card frame（`min-height: 100px`），灰色脈動動畫 +- ParetoChart: 6 個固定高度 chart frame（`min-height: 300px`），灰色脈動動畫 +- TrendChart: 1 個固定高度 frame（`min-height: 300px`） +- fade-in: CSS transition `opacity 0→1, 300ms ease-in` + +**Auto-refresh 整合**: +- `useAutoRefresh.onRefresh` → `trace.abort()` + `trace.execute(committedFilters)` +- 保持現行 5 分鐘 jittered interval + +**Detail 分頁不變**: +- `/api/mid-section-defect/analysis/detail` GET endpoint 保持不變 +- 不走分段 API（detail 是分頁查詢，與 trace pipeline 獨立） + +### D4: query-tool on-demand lineage 策略 + +**選擇**: per-lot on-demand fetch，使用者點擊 lot card 才查血緣 +**替代方案**: batch-load all lots lineage at resolve time +**理由**: +- resolve 結果可能有 20+ lots，批次查全部 lineage 增加不必要的 DB 負擔 +- 大部分使用者只關注特定幾個 lot 的 lineage +- per-lot fetch 控制在 ≤3s，使用者體驗可接受 + +**useQueryToolData.js 改造**: +```javascript +// 新增 lineage state +const lineageCache = reactive({}) // { [containerId]: { ancestors, merges, loading, error } } + +async function loadLotLineage(containerId) { + if (lineageCache[containerId]?.ancestors) return // 已快取 + + lineageCache[containerId] = { ancestors: null, merges: null, loading: true, error: null } + try { + const result = await apiPost('/api/trace/lineage', { + profile: 'query_tool', + container_ids: [containerId], + }, { timeout: 60000 }) + lineageCache[containerId] = { + ancestors: result.data.ancestors, + merges: result.data.merges, + loading: false, + error: null, + } + } catch (err) { + lineageCache[containerId] = { + ancestors: null, + merges: null, + loading: false, + error: err.message, + } + } +} +``` + +**UI 行為**: +- lot 列表中每個 lot 有展開按鈕（或 accordion） +- 點擊展開 → 呼叫 `loadLotLineage(containerId)` → 顯示 loading → 顯示 lineage tree +- 已展開的 lot 再次點擊收合（不重新 fetch） +- `lineageCache` 在新一輪 `resolveLots()` 時清空 + +**query-tool 主流程保持不變**: +- resolve → lot-history → lot-associations 的既有流程不改 +- lineage 是新增的 on-demand 功能，不取代既有功能 +- query-tool 暫不使用 `useTraceProgress`（因為它的流程是使用者驅動的互動式，非自動 sequential） + +### D5: 進度指示器元件設計 + +**選擇**: 共用 `TraceProgressBar.vue` 元件，props 驅動 +**替代方案**: 各頁面各自實作進度顯示 +**理由**: +- 兩個頁面顯示相同的 stage 進度（seed → lineage → events） +- 統一視覺語言 + +**元件設計**: +```javascript +// frontend/src/shared-composables/TraceProgressBar.vue +// (放在 shared-composables 目錄，雖然是 .vue 但與 composable 搭配使用) + +props: { + current_stage: String | null, // 'seed-resolve' | 'lineage' | 'events' + completed_stages: Array, // ['seed-resolve', 'lineage'] + stage_errors: Object, // { seed: null, lineage: { code, message } } +} + +// 三個 step indicator: +// [●] Seed → [●] Lineage → [○] Events +// ↑ 完成(green) ↑ 進行中(blue pulse) ↑ 待處理(gray) +// ↑ 錯誤(red) +``` + +**Stage 顯示名稱**: +| Stage ID | 中文顯示 | 英文顯示 | +|----------|---------|---------| +| seed-resolve | 批次解析 | Resolving | +| lineage | 血緣追溯 | Lineage | +| events | 事件查詢 | Events | + +**取代 loading spinner**: +- mid-section-defect: `loading.querying` 原本控制單一 spinner → 改為顯示 `TraceProgressBar` +- 進度指示器放在 filter bar 下方、結果區域上方 + +### D6: `/analysis` GET endpoint 向下相容橋接 + +**選擇**: 保留原 handler，內部改為呼叫分段管線後合併結果 +**替代方案**: 直接改原 handler 不經過分段管線 +**理由**: +- 分段管線（LineageEngine + EventFetcher）在 `unified-lineage-engine` 完成後已是標準路徑 +- 保留原 handler 確保非 portal-shell 路由 fallback 仍可用 +- golden test 比對確保結果等價 + +**橋接邏輯**: +```python +# mid_section_defect_routes.py — /analysis handler 內部改造 + +@mid_section_defect_bp.route('/analysis', methods=['GET']) +@configured_rate_limit(bucket='msd-analysis', ...) +def api_analysis(): + # 現行: result = mid_section_defect_service.query_analysis(start_date, end_date, loss_reasons) + # 改為: 呼叫 service 層的管線函數（service 內部已使用 LineageEngine + EventFetcher） + # response format 完全不變 + result = mid_section_defect_service.query_analysis(start_date, end_date, loss_reasons) + return jsonify({"success": True, "data": result}) +``` + +**實際上 `/analysis` handler 不需要改**——`unified-lineage-engine` Phase 1 已將 service 內部改為使用 `LineageEngine`。本變更只需確認 `/analysis` 回傳結果與重構前完全一致（golden test 驗證），不需額外的橋接程式碼。 + +**Golden test 策略**: +- 選取 ≥3 組已知查詢參數（不同日期範圍、不同 loss_reasons 組合） +- 比對重構前後 `/analysis` JSON response 結構和數值 +- 允許浮點數 tolerance（defect_rate 等百分比欄位 ±0.01%） + +### D7: Legacy static JS 清理 + +**選擇**: 直接刪除 `src/mes_dashboard/static/js/query-tool.js` +**理由**: +- 此檔案 3056L / 126KB，是 pre-Vite 時代的靜態 JS +- `query_tool.html` template 使用 `frontend_asset('query-tool.js')` 載入 Vite 建置產物，非此靜態檔案 +- Vite config 確認 entry point: `'query-tool': resolve(__dirname, 'src/query-tool/main.js')` +- `frontend_asset()` 解析 Vite manifest，不會指向 `static/js/` +- grep 確認無其他引用 + +**驗證步驟**: +1. `grep -r "static/js/query-tool.js" src/ frontend/ templates/` → 0 結果 +2. 確認 `frontend_asset('query-tool.js')` 正確解析到 Vite manifest 中的 hashed filename +3. 確認 `frontend/src/query-tool/main.js` 為 active entry（Vite config `input` 對應） + +### D8: 實作順序 + +**Phase 1**: 後端 trace_routes.py（無前端改動） +1. 建立 `trace_routes.py` + 三個 route handler +2. 在 `app.py` 註冊 `trace_bp` Blueprint +3. Profile dispatch functions（呼叫既有 service 邏輯） +4. Rate limit + cache 配置 +5. 錯誤碼 + HTTP status 對齊 spec +6. API contract 測試（request/response schema 驗證） + +**Phase 2**: 前端共用元件 +1. 建立 `useTraceProgress.js` composable +2. 建立 `TraceProgressBar.vue` 進度指示器 +3. 單元測試（mock API calls，驗證 stage 狀態轉換） + +**Phase 3**: mid-section-defect 漸進渲染 +1. `App.vue` 查詢流程改為 `useTraceProgress` +2. 加入 skeleton placeholders + fade-in transitions +3. 用 `TraceProgressBar` 取代 loading spinner +4. 驗證 auto-refresh 整合 +5. Golden test: `/analysis` 回傳結果不變 + +**Phase 4**: query-tool on-demand lineage +1. `useQueryToolData.js` 新增 `lineageCache` + `loadLotLineage()` +2. lot 列表加入 lineage 展開 UI +3. 驗證既有 resolve → history → association 流程不受影響 + +**Phase 5**: Legacy cleanup +1. 刪除 `src/mes_dashboard/static/js/query-tool.js` +2. grep 確認零引用 +3. 確認 `frontend_asset()` 解析正常 + +## Risks / Trade-offs + +| Risk | Mitigation | +|------|-----------| +| 分段 API 增加前端複雜度（3 次 fetch + 狀態管理） | 封裝在 `useTraceProgress` composable，頁面只需 `execute(params)` + watch `stage_results` | +| `/analysis` golden test 因浮點精度失敗 | 允許 defect_rate 等百分比 ±0.01% tolerance，整數欄位嚴格比對 | +| mid-section-defect skeleton → chart 渲染閃爍 | 固定高度 placeholder + fade-in 300ms transition，chart container 不允許 height auto | +| `useTraceProgress` abort 與 `useAutoRefresh` 衝突 | auto-refresh 觸發前先呼叫 `trace.abort()`，確保上一輪 fetch 完整中止 | +| query-tool lineage per-lot fetch 對高頻展開造成 DB 壓力 | lineageCache 防止重複 fetch + trace-lineage rate limit (10/60s) 保護 | +| `static/js/query-tool.js` 刪除影響未知路徑 | grep 全域確認 0 引用 + `frontend_asset()` 確認 Vite manifest 解析正確 | +| cache_key 傳遞中斷（前端忘記傳 cache_key） | cache_key 為選填，僅用於 logging correlation，缺少不影響功能 | + +## Open Questions + +- `useTraceProgress` 是否需要支援 retry（某段失敗後重試該段而非整體重新執行）？暫不支援——失敗後使用者重新按查詢按鈕即可。 +- mid-section-defect 的 aggregation 邏輯（KPI、charts、trend 計算）是放在 `/api/trace/events` 的 mid_section_defect profile 分支內，還是由前端從 raw events 自行計算？**決定: 放在後端 `/api/trace/events` 的 aggregation field**——前端不應承擔 defect attribution 計算責任，且計算邏輯已在 service 層成熟。 +- `TraceProgressBar.vue` 放在 `shared-composables/` 還是獨立的 `shared-components/` 目錄？暫放 `shared-composables/`（與 composable 搭配使用），若未來 shared 元件增多再考慮拆分。 diff --git a/openspec/changes/trace-progressive-ui/proposal.md b/openspec/changes/trace-progressive-ui/proposal.md new file mode 100644 index 0000000..7c4b28f --- /dev/null +++ b/openspec/changes/trace-progressive-ui/proposal.md @@ -0,0 +1,148 @@ +## Why + +`unified-lineage-engine` 完成後，後端追溯管線從 30-120 秒降至 3-8 秒，但對於大範圍查詢（日期跨度長、LOT 數量多）仍可能需要 5-15 秒。目前的 UX 模式是「使用者點擊查詢 → 等待黑盒 → 全部結果一次出現」，即使後端已加速，使用者仍感受不到進度，只有一個 loading spinner。 + +兩個頁面的前端載入模式存在差異： +- **mid-section-defect**: 一次 API call (`/analysis`) 拿全部結果（KPI + charts + detail），後端做完全部 4 個 stage 才回傳。 +- **query-tool**: Vue 3 版本（`frontend/src/query-tool/`）已有手動順序（resolve → history → association），但部分流程仍可改善漸進載入體驗。 + +需要統一兩個頁面的前端查詢體驗，實現「分段載入 + 進度可見」的 UX 模式，讓使用者看到追溯的漸進結果而非等待黑盒。 + +**邊界聲明**：本變更負責新增分段 API endpoints（`/api/trace/*`）和前端漸進式載入 UX。後端追溯核心邏輯（`LineageEngine`、`EventFetcher`）由前置的 `unified-lineage-engine` 變更提供，本變更僅作為 API routing layer 呼叫這些模組。 + +## What Changes + +### 後端：新增分段 API endpoints + +新增 `trace_routes.py` Blueprint（`/api/trace/`），將追溯管線的每個 stage 獨立暴露為 endpoint。通過 `profile` 參數區分頁面行為： + +**POST `/api/trace/seed-resolve`** +- Request: `{ "profile": "query_tool" | "mid_section_defect", "params": { ... } }` + - `query_tool` params: `{ "resolve_type": "lot_id" | "serial_number" | "work_order", "values": [...] }` + - `mid_section_defect` params: `{ "date_range": [...], "workcenter": "...", ... }` (TMTT detection 參數) +- Response: `{ "stage": "seed-resolve", "seeds": [{ "container_id": "...", "container_name": "...", "lot_id": "..." }], "seed_count": N, "cache_key": "trace:{hash}" }` +- Error: `{ "error": "...", "code": "SEED_RESOLVE_EMPTY" | "SEED_RESOLVE_TIMEOUT" | "INVALID_PROFILE" }` +- Rate limit: `configured_rate_limit(bucket="trace-seed", default_max_attempts=10, default_window_seconds=60)` +- Cache: L2 Redis, key = `trace:seed:{profile}:{params_hash}`, TTL = 300s + +**POST `/api/trace/lineage`** +- Request: `{ "profile": "query_tool" | "mid_section_defect", "container_ids": [...], "cache_key": "trace:{hash}" }` +- Response: `{ "stage": "lineage", "ancestors": { "{cid}": ["{ancestor_cid}", ...] }, "merges": { "{cid}": ["{merge_source_cid}", ...] }, "total_nodes": N }` +- Error: `{ "error": "...", "code": "LINEAGE_TIMEOUT" | "LINEAGE_TOO_LARGE" }` +- Rate limit: `configured_rate_limit(bucket="trace-lineage", default_max_attempts=10, default_window_seconds=60)` +- Cache: L2 Redis, key = `trace:lineage:{sorted_cids_hash}`, TTL = 300s +- 冪等性: 相同 `container_ids` 集合（排序後 hash）回傳 cache 結果 + +**POST `/api/trace/events`** +- Request: `{ "profile": "query_tool" | "mid_section_defect", "container_ids": [...], "domains": ["history", "materials", ...], "cache_key": "trace:{hash}" }` + - `mid_section_defect` 額外支援 `"domains": ["upstream_history"]` 和自動串接 aggregation +- Response: `{ "stage": "events", "results": { "{domain}": { "data": [...], "count": N } }, "aggregation": { ... } | null }` +- Error: `{ "error": "...", "code": "EVENTS_TIMEOUT" | "EVENTS_PARTIAL_FAILURE" }` + - `EVENTS_PARTIAL_FAILURE`: 部分 domain 查詢失敗時仍回傳已成功的結果，`failed_domains` 列出失敗項 +- Rate limit: `configured_rate_limit(bucket="trace-events", default_max_attempts=15, default_window_seconds=60)` +- Cache: L2 Redis, key = `trace:evt:{profile}:{domains_hash}:{sorted_cids_hash}`, TTL = 300s + +**所有 endpoints 共通規則**: +- HTTP status: 200 (success), 400 (invalid params/profile), 429 (rate limited), 504 (stage timeout >10s) +- Rate limit headers: `Retry-After`（對齊 `rate_limit.py` 既有實作，回應 body 含 `retry_after_seconds` 欄位） +- `cache_key` 為選填欄位，前端可傳入前一 stage 回傳的 cache_key 作為追溯鏈標識（用於 logging correlation），不影響 cache 命中邏輯 +- 每個 stage 獨立可呼叫——前端可按需組合，不要求嚴格順序（但 lineage 需要 seed 結果的 container_ids，events 需要 lineage 結果的 container_ids） + +### 舊 endpoint 兼容 + +- `/api/mid-section-defect/analysis` 保留，內部改為呼叫分段管線（seed-resolve → lineage → events+aggregation）後合併結果回傳。行為等價，API contract 不變。 +- `/api/query-tool/*` 保留不變，前端可視進度逐步遷移到新 API。 + +### 前端：漸進式載入 + +- 新增 `frontend/src/shared-composables/useTraceProgress.js` composable，封裝： + - 三段式 sequential fetch（seed → lineage → events） + - 每段完成後更新 reactive state（`current_stage`, `completed_stages`, `stage_results`） + - 錯誤處理: 每段獨立，某段失敗不阻斷已完成的結果顯示 + - profile 參數注入 +- **mid-section-defect** (`App.vue`): 查詢流程改為分段 fetch + 漸進渲染： + - 查詢後先顯示 seed lots 列表（skeleton UI → 填入 seed 結果） + - 血緣樹結構逐步展開 + - KPI/圖表以 skeleton placeholders + fade-in 動畫漸進填入，避免 layout shift + - 明細表格仍使用 detail endpoint 分頁 +- **query-tool** (`useQueryToolData.js`): lineage tab 改為 on-demand 展開（使用者點擊 lot 後才查血緣），主要強化漸進載入體驗。 +- 兩個頁面新增進度指示器元件，顯示目前正在執行的 stage（seed → lineage → events → aggregation）和已完成的 stage。 + +### Legacy 檔案處理 + +- **廢棄**: `src/mes_dashboard/static/js/query-tool.js`（3056L, 126KB）——這是 pre-Vite 時代的靜態 JS 檔案，目前已無任何 template 載入（`query_tool.html` 使用 `frontend_asset('query-tool.js')` 載入 Vite 建置產物，非此靜態檔案）。此檔案為 dead code，可安全刪除。 +- **保留**: `frontend/src/query-tool/main.js`（3139L）——這是 Vue 3 版本的 Vite entry point，Vite config 確認為 `'query-tool': resolve(__dirname, 'src/query-tool/main.js')`。此檔案持續維護。 +- **保留**: `src/mes_dashboard/templates/query_tool.html`——Jinja2 模板，line 1264 `{% set query_tool_js = frontend_asset('query-tool.js') %}` 載入 Vite 建置產物。目前 portal-shell route 已生效（`/portal-shell/query-tool` 走 Vue 3），此模板為 non-portal-shell 路由的 fallback，暫不刪除。 + +## Capabilities + +### New Capabilities + +- `trace-staged-api`: 統一的分段追溯 API 層（`/api/trace/seed-resolve`、`/api/trace/lineage`、`/api/trace/events`）。通過 `profile` 參數配置頁面行為。每段獨立可 cache（L2 Redis）、可 rate limit（`configured_rate_limit()`），前端可按需組合。API contract 定義於本提案 What Changes 章節。 +- `progressive-trace-ux`: 兩個頁面的漸進式載入 UX。`useTraceProgress` composable 封裝三段式 sequential fetch + reactive state。包含： + - 進度指示器元件（顯示 seed → lineage → events → aggregation 各階段狀態） + - mid-section-defect: seed lots 先出 → 血緣結構 → KPI/圖表漸進填入（skeleton + fade-in） + - query-tool: lineage tab 改為 on-demand 展開（使用者點擊 lot 後才查血緣） + +### Modified Capabilities + +- `trace-staged-api` 取代 mid-section-defect 現有的單一 `/analysis` endpoint 邏輯（保留舊 endpoint 作為兼容，內部改為呼叫分段管線 + 合併結果，行為等價）。 +- query-tool 現有的 `useQueryToolData.js` composable 改為使用分段 API。 + +## Impact + +- **Affected code**: + - 新建: `src/mes_dashboard/routes/trace_routes.py`, `frontend/src/shared-composables/useTraceProgress.js` + - 重構: `frontend/src/mid-section-defect/App.vue`（查詢流程改為分段 fetch） + - 重構: `frontend/src/query-tool/composables/useQueryToolData.js`（lineage 改分段） + - 修改: `src/mes_dashboard/routes/mid_section_defect_routes.py`（`/analysis` 內部改用分段管線） + - 刪除: `src/mes_dashboard/static/js/query-tool.js`（pre-Vite dead code, 3056L, 126KB, 無任何引用） +- **Runtime/deploy**: 無新依賴。新增 3 個 API endpoints（`/api/trace/*`），原有 endpoints 保持兼容。 +- **APIs/pages**: 新增 `/api/trace/seed-resolve`、`/api/trace/lineage`、`/api/trace/events` 三個 endpoints（contract 定義見 What Changes 章節）。原有 `/api/mid-section-defect/analysis` 和 `/api/query-tool/*` 保持兼容但 `/analysis` 內部實作改為呼叫分段管線。 +- **UX**: 查詢體驗從「黑盒等待」變為「漸進可見」。mid-section-defect 使用者可在血緣解析階段就看到 seed lots 和初步資料。 + +## Verification + +**前端漸進載入驗收**: + +| 指標 | 現況 | 目標 | 條件 | +|------|------|------|------| +| mid-section-defect 首次可見內容 (seed lots) | 全部完成後一次顯示（30-120s, unified-lineage-engine 後 3-8s） | seed stage 完成即顯示（≤3s） | ≥10 seed lots 查詢 | +| mid-section-defect KPI/chart 完整顯示 | 同上 | lineage + events 完成後顯示（≤8s） | skeleton → fade-in, 無 layout shift | +| query-tool lineage tab | 一次載入全部 lot 的 lineage | 點擊單一 lot 後載入該 lot lineage（≤3s） | on-demand, ≥20 lots resolved | +| 進度指示器 | 無（loading spinner） | 每個 stage 切換時更新進度文字 | seed → lineage → events 三階段可見 | + +**API contract 驗收**: +- 每個 `/api/trace/*` endpoint 回傳 JSON 結構符合 What Changes 章節定義的 schema +- 400 (invalid params) / 429 (rate limited) / 504 (timeout) status code 正確回傳 +- Rate limit header `Retry-After` 存在（對齊既有 `rate_limit.py` 實作） +- `/api/mid-section-defect/analysis` 兼容性: 回傳結果與重構前完全一致（golden test 比對） + +**Legacy cleanup 驗收**: +- `src/mes_dashboard/static/js/query-tool.js` 已刪除 +- grep 確認無任何程式碼引用 `static/js/query-tool.js` +- `query_tool.html` 中 `frontend_asset('query-tool.js')` 仍正常解析到 Vite 建置產物 + +## Dependencies + +- **前置條件**: `unified-lineage-engine` 變更必須先完成。本變更依賴 `LineageEngine` 和 `EventFetcher` 作為分段 API 的後端實作。 + +## Non-Goals + +- 不實作 SSE (Server-Sent Events) 或 WebSocket 即時推送——考慮到 gunicorn sync workers 的限制，使用分段 API + 前端 sequential fetch 模式。 +- 不改動後端追溯邏輯——分段 API 純粹是將 `LineageEngine` / `EventFetcher` 各 stage 獨立暴露為 HTTP endpoint，不改變計算邏輯。 +- 不新增任務隊列（Celery/RQ）——維持同步 request-response 模式，各 stage 控制在 <10s 回應時間內。 +- 不改動 mid-section-defect 的 defect attribution 演算法。 +- 不改動 query-tool 的 equipment-period 查詢（已有 `read_sql_df_slow` 120s timeout 處理）。 +- 不改動 DB schema、不建立 materialized view——所有最佳化在應用層完成。 + +## Risks + +| 風險 | 緩解 | +|------|------| +| 分段 API 增加前端複雜度（多次 fetch + 狀態管理） | 封裝為 `useTraceProgress` composable，頁面只需提供 profile + params，內部處理 sequential fetch + error + state | +| 前後端分段 contract 不匹配 | API contract 完整定義於本提案 What Changes 章節，含 request/response schema、error codes、cache key 格式。CI 契約測試驗證 | +| 舊 `/analysis` endpoint 需保持兼容 | 保留舊 endpoint，內部改為呼叫分段管線 + 合併結果。golden test 比對重構前後輸出一致 | +| 刪除 `static/js/query-tool.js` 影響功能 | 已確認此檔案為 pre-Vite dead code：`query_tool.html` 使用 `frontend_asset('query-tool.js')` 載入 Vite 建置產物，非此靜態檔案。grep 確認無其他引用 | +| mid-section-defect 分段渲染導致 chart 閃爍 | 使用 skeleton placeholders + fade-in 動畫，避免 layout shift。chart container 預留固定高度 | +| `cache_key` 被濫用於跨 stage 繞過 rate limit | cache_key 僅用於 logging correlation，不影響 cache 命中或 rate limit 邏輯。每個 stage 獨立計算 cache key | diff --git a/openspec/changes/trace-progressive-ui/specs/api-safety-hygiene/spec.md b/openspec/changes/trace-progressive-ui/specs/api-safety-hygiene/spec.md new file mode 100644 index 0000000..f285561 --- /dev/null +++ b/openspec/changes/trace-progressive-ui/specs/api-safety-hygiene/spec.md @@ -0,0 +1,25 @@ +## MODIFIED Requirements + +### Requirement: Staged trace API endpoints SHALL apply rate limiting +The `/api/trace/seed-resolve`, `/api/trace/lineage`, and `/api/trace/events` endpoints SHALL apply per-client rate limiting using the existing `configured_rate_limit` mechanism. + +#### Scenario: Seed-resolve rate limit exceeded +- **WHEN** a client sends more than 10 requests to `/api/trace/seed-resolve` within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header + +#### Scenario: Lineage rate limit exceeded +- **WHEN** a client sends more than 10 requests to `/api/trace/lineage` within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header + +#### Scenario: Events rate limit exceeded +- **WHEN** a client sends more than 15 requests to `/api/trace/events` within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header + +### Requirement: Mid-section defect analysis endpoint SHALL internally use staged pipeline +The existing `/api/mid-section-defect/analysis` endpoint SHALL internally delegate to the staged trace pipeline while maintaining full backward compatibility. + +#### Scenario: Analysis endpoint backward compatibility +- **WHEN** a client calls `GET /api/mid-section-defect/analysis` with existing query parameters +- **THEN** the response JSON structure SHALL be identical to pre-refactoring output +- **THEN** existing rate limiting (6/min analysis, 15/min detail, 3/min export) SHALL remain unchanged +- **THEN** existing distributed lock behavior SHALL remain unchanged diff --git a/openspec/changes/trace-progressive-ui/specs/progressive-trace-ux/spec.md b/openspec/changes/trace-progressive-ui/specs/progressive-trace-ux/spec.md new file mode 100644 index 0000000..553e66b --- /dev/null +++ b/openspec/changes/trace-progressive-ui/specs/progressive-trace-ux/spec.md @@ -0,0 +1,64 @@ +## ADDED Requirements + +### Requirement: useTraceProgress composable SHALL orchestrate staged fetching with reactive state +`useTraceProgress` SHALL provide a shared composable for sequential stage fetching with per-stage reactive state updates. + +#### Scenario: Normal three-stage fetch sequence +- **WHEN** `useTraceProgress` is invoked with profile and params +- **THEN** it SHALL execute seed-resolve → lineage → events sequentially +- **THEN** after each stage completes, `current_stage` and `completed_stages` reactive refs SHALL update immediately +- **THEN** `stage_results` SHALL accumulate results from completed stages + +#### Scenario: Stage failure does not block completed results +- **WHEN** the lineage stage fails after seed-resolve has completed +- **THEN** seed-resolve results SHALL remain visible and accessible +- **THEN** the error SHALL be captured in stage-specific error state +- **THEN** subsequent stages (events) SHALL NOT execute + +### Requirement: mid-section-defect SHALL render progressively as stages complete +The mid-section-defect page SHALL display partial results as each trace stage completes. + +#### Scenario: Seed lots visible before lineage completes +- **WHEN** seed-resolve stage completes (≤3s for ≥10 seed lots) +- **THEN** the seed lots list SHALL be rendered immediately +- **THEN** lineage and events sections SHALL show skeleton placeholders + +#### Scenario: KPI/charts visible after events complete +- **WHEN** lineage and events stages complete +- **THEN** KPI cards and charts SHALL render with fade-in animation +- **THEN** no layout shift SHALL occur (skeleton placeholders SHALL have matching dimensions) + +#### Scenario: Detail table pagination unchanged +- **WHEN** the user requests detail data +- **THEN** the existing detail endpoint with pagination SHALL be used (not the staged API) + +### Requirement: query-tool lineage tab SHALL load on-demand +The query-tool lineage tab SHALL load lineage data for individual lots on user interaction, not batch-load all lots. + +#### Scenario: User clicks a lot to view lineage +- **WHEN** the user clicks a lot card to expand lineage information +- **THEN** lineage SHALL be fetched via `/api/trace/lineage` for that single lot's container IDs +- **THEN** response time SHALL be ≤3s for the individual lot + +#### Scenario: Multiple lots expanded +- **WHEN** the user expands lineage for multiple lots +- **THEN** each lot's lineage SHALL be fetched independently (not batch) +- **THEN** already-fetched lineage data SHALL be preserved (not re-fetched) + +### Requirement: Both pages SHALL display a stage progress indicator +Both mid-section-defect and query-tool SHALL display a progress indicator showing the current trace stage. + +#### Scenario: Progress indicator during staged fetch +- **WHEN** a trace query is in progress +- **THEN** a progress indicator SHALL display the current stage (seed → lineage → events) +- **THEN** completed stages SHALL be visually distinct from pending/active stages +- **THEN** the indicator SHALL replace the existing single loading spinner + +### Requirement: Legacy static query-tool.js SHALL be removed +The pre-Vite static file `src/mes_dashboard/static/js/query-tool.js` (3056L, 126KB) SHALL be deleted as dead code. + +#### Scenario: Dead code removal verification +- **WHEN** `static/js/query-tool.js` is deleted +- **THEN** grep for `static/js/query-tool.js` SHALL return zero results across the codebase +- **THEN** `query_tool.html` template SHALL continue to function via `frontend_asset('query-tool.js')` which resolves to the Vite-built bundle +- **THEN** `frontend/src/query-tool/main.js` (Vue 3 Vite entry) SHALL remain unaffected diff --git a/openspec/changes/trace-progressive-ui/specs/trace-staged-api/spec.md b/openspec/changes/trace-progressive-ui/specs/trace-staged-api/spec.md new file mode 100644 index 0000000..48b97c9 --- /dev/null +++ b/openspec/changes/trace-progressive-ui/specs/trace-staged-api/spec.md @@ -0,0 +1,89 @@ +## ADDED Requirements + +### Requirement: Staged trace API SHALL expose seed-resolve endpoint +`POST /api/trace/seed-resolve` SHALL resolve seed lots based on the provided profile and parameters. + +#### Scenario: query_tool profile seed resolve +- **WHEN** request body contains `{ "profile": "query_tool", "params": { "resolve_type": "lot_id", "values": [...] } }` +- **THEN** the endpoint SHALL call existing lot resolve logic and return `{ "stage": "seed-resolve", "seeds": [...], "seed_count": N, "cache_key": "trace:{hash}" }` +- **THEN** each seed object SHALL contain `container_id`, `container_name`, and `lot_id` + +#### Scenario: mid_section_defect profile seed resolve +- **WHEN** request body contains `{ "profile": "mid_section_defect", "params": { "date_range": [...], "workcenter": "..." } }` +- **THEN** the endpoint SHALL call TMTT detection logic and return seed lots in the same response format + +#### Scenario: Empty seed result +- **WHEN** seed resolution finds no matching lots +- **THEN** the endpoint SHALL return HTTP 200 with `{ "stage": "seed-resolve", "seeds": [], "seed_count": 0, "cache_key": "trace:{hash}" }` +- **THEN** the error code `SEED_RESOLVE_EMPTY` SHALL NOT be used for empty results (reserved for resolution failures) + +#### Scenario: Invalid profile +- **WHEN** request body contains an unrecognized `profile` value +- **THEN** the endpoint SHALL return HTTP 400 with `{ "error": "...", "code": "INVALID_PROFILE" }` + +### Requirement: Staged trace API SHALL expose lineage endpoint +`POST /api/trace/lineage` SHALL resolve lineage graph for provided container IDs using `LineageEngine`. + +#### Scenario: Normal lineage resolution +- **WHEN** request body contains `{ "profile": "query_tool", "container_ids": [...] }` +- **THEN** the endpoint SHALL call `LineageEngine.resolve_full_genealogy()` and return `{ "stage": "lineage", "ancestors": {...}, "merges": {...}, "total_nodes": N }` + +#### Scenario: Lineage result caching with idempotency +- **WHEN** two requests with the same `container_ids` set (regardless of order) arrive +- **THEN** the cache key SHALL be computed as `trace:lineage:{sorted_cids_hash}` +- **THEN** the second request SHALL return cached result from L2 Redis (TTL = 300s) + +#### Scenario: Lineage timeout +- **WHEN** lineage resolution exceeds 10 seconds +- **THEN** the endpoint SHALL return HTTP 504 with `{ "error": "...", "code": "LINEAGE_TIMEOUT" }` + +### Requirement: Staged trace API SHALL expose events endpoint +`POST /api/trace/events` SHALL query events for specified domains using `EventFetcher`. + +#### Scenario: Normal events query +- **WHEN** request body contains `{ "profile": "query_tool", "container_ids": [...], "domains": ["history", "materials"] }` +- **THEN** the endpoint SHALL return `{ "stage": "events", "results": { "history": { "data": [...], "count": N }, "materials": { "data": [...], "count": N } }, "aggregation": null }` + +#### Scenario: mid_section_defect profile with aggregation +- **WHEN** request body contains `{ "profile": "mid_section_defect", "container_ids": [...], "domains": ["upstream_history"] }` +- **THEN** the endpoint SHALL automatically run aggregation logic after event fetching +- **THEN** the response `aggregation` field SHALL contain the aggregated results (not null) + +#### Scenario: Partial domain failure +- **WHEN** one domain query fails while others succeed +- **THEN** the endpoint SHALL return HTTP 200 with `{ "error": "...", "code": "EVENTS_PARTIAL_FAILURE" }` +- **THEN** the response SHALL include successfully fetched domains in `results` and list failed domains in `failed_domains` + +### Requirement: All staged trace endpoints SHALL apply rate limiting and caching +Every `/api/trace/*` endpoint SHALL use `configured_rate_limit()` and L2 Redis caching. + +#### Scenario: Rate limit exceeded on any trace endpoint +- **WHEN** a client exceeds the configured request budget for a trace endpoint +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header +- **THEN** the body SHALL contain `{ "error": "...", "meta": { "retry_after_seconds": N } }` + +#### Scenario: Cache hit on trace endpoint +- **WHEN** a request matches a cached result in L2 Redis (TTL = 300s) +- **THEN** the cached result SHALL be returned without executing backend logic +- **THEN** Oracle DB connection pool SHALL NOT be consumed + +### Requirement: cache_key parameter SHALL be used for logging correlation only +The optional `cache_key` field in request bodies SHALL be used solely for logging and tracing correlation. + +#### Scenario: cache_key provided in request +- **WHEN** a request includes `cache_key` from a previous stage response +- **THEN** the value SHALL be logged for correlation purposes +- **THEN** the value SHALL NOT influence cache lookup or rate limiting logic + +#### Scenario: cache_key omitted in request +- **WHEN** a request omits the `cache_key` field +- **THEN** the endpoint SHALL function normally without any degradation + +### Requirement: Existing `GET /api/mid-section-defect/analysis` SHALL remain compatible +The existing analysis endpoint (GET method) SHALL internally delegate to the staged pipeline while maintaining identical external behavior. + +#### Scenario: Legacy analysis endpoint invocation +- **WHEN** a client calls `GET /api/mid-section-defect/analysis` with existing query parameters +- **THEN** the endpoint SHALL internally execute seed-resolve → lineage → events + aggregation +- **THEN** the response format SHALL be identical to the pre-refactoring output +- **THEN** a golden test SHALL verify output equivalence diff --git a/openspec/changes/trace-progressive-ui/tasks.md b/openspec/changes/trace-progressive-ui/tasks.md new file mode 100644 index 0000000..da02182 --- /dev/null +++ b/openspec/changes/trace-progressive-ui/tasks.md @@ -0,0 +1,41 @@ +## Phase 1: 後端 trace_routes.py Blueprint + +- [x] 1.1 建立 `src/mes_dashboard/routes/trace_routes.py`：`trace_bp` Blueprint（`url_prefix='/api/trace'`） +- [x] 1.2 實作 `POST /api/trace/seed-resolve` handler：request body 驗證、profile dispatch（`_seed_resolve_query_tool` / `_seed_resolve_msd`）、response format +- [x] 1.3 實作 `POST /api/trace/lineage` handler：呼叫 `LineageEngine.resolve_full_genealogy()`、response format、504 timeout 處理 +- [x] 1.4 實作 `POST /api/trace/events` handler：呼叫 `EventFetcher.fetch_events()`、mid_section_defect profile 自動 aggregation、`EVENTS_PARTIAL_FAILURE` 處理 +- [x] 1.5 為三個 endpoint 加入 `configured_rate_limit()`（seed: 10/60s, lineage: 10/60s, events: 15/60s） +- [x] 1.6 為三個 endpoint 加入 L2 Redis cache（seed: `trace:seed:{profile}:{params_hash}`, lineage: `trace:lineage:{sorted_cids_hash}`, events: `trace:evt:{profile}:{domains_hash}:{sorted_cids_hash}`，TTL=300s） +- [x] 1.7 在 `src/mes_dashboard/routes/__init__.py` 匯入並註冊 `trace_bp` Blueprint（維持專案統一的 route 註冊入口） +- [x] 1.8 API contract 測試：驗證 200/400/429/504 status code、`Retry-After` header、error code 格式、snake_case field names + +## Phase 2: 前端共用元件 + +- [x] 2.1 建立 `frontend/src/shared-composables/useTraceProgress.js`：reactive state（`current_stage`, `completed_stages`, `stage_results`, `stage_errors`, `is_running`）+ `execute()` / `reset()` / `abort()` methods +- [x] 2.2 實作 sequential fetch 邏輯：seed-resolve → lineage → events，每段完成後立即更新 reactive state，錯誤記錄到 stage_errors 不拋出 +- [x] 2.3 建立 `frontend/src/shared-composables/TraceProgressBar.vue`：三段式進度指示器（props: `current_stage`, `completed_stages`, `stage_errors`），完成=green、進行中=blue pulse、待處理=gray、錯誤=red + +## Phase 3: mid-section-defect 漸進渲染 + +- [x] 3.1 在 `frontend/src/mid-section-defect/App.vue` 中引入 `useTraceProgress({ profile: 'mid_section_defect' })` +- [x] 3.2 改造 `loadAnalysis()` 流程：從 `apiGet('/analysis')` 單次呼叫改為 `trace.execute(params)` 分段 fetch +- [x] 3.3 加入 skeleton placeholders：KpiCards（6 cards, min-height 100px）、ParetoChart（6 charts, min-height 300px）、TrendChart（min-height 300px），灰色脈動動畫 +- [x] 3.4 加入 fade-in transition：stage_results.events 完成後 KPI/charts 以 `opacity 0→1, 300ms ease-in` 填入 +- [x] 3.5 用 `TraceProgressBar` 取代 filter bar 下方的 loading spinner +- [x] 3.6 整合 `useAutoRefresh`：`onRefresh` → `trace.abort()` + `trace.execute(committedFilters)` +- [x] 3.7 驗證 detail 分頁不受影響（仍使用 `/api/mid-section-defect/analysis/detail` GET endpoint） +- [x] 3.8 Golden test：`/api/mid-section-defect/analysis` GET endpoint 回傳結果與重構前完全一致（浮點 tolerance ±0.01%） + +## Phase 4: query-tool on-demand lineage + +- [x] 4.1 在 `useQueryToolData.js` 新增 `lineageCache` reactive object + `loadLotLineage(containerId)` 函數 +- [x] 4.2 `loadLotLineage` 呼叫 `POST /api/trace/lineage`（`profile: 'query_tool'`, `container_ids: [containerId]`），結果存入 `lineageCache` +- [x] 4.3 在 lot 列表 UI 新增 lineage 展開按鈕（accordion pattern），點擊觸發 `loadLotLineage`，已快取的不重新 fetch +- [x] 4.4 `resolveLots()` 時清空 `lineageCache`（新一輪查詢） +- [x] 4.5 驗證既有 resolve → lot-history → lot-associations 流程不受影響 + +## Phase 5: Legacy cleanup + +- [x] 5.1 刪除 `src/mes_dashboard/static/js/query-tool.js`（3056L, 126KB pre-Vite dead code） +- [x] 5.2 `grep -r "static/js/query-tool.js" src/ frontend/ templates/` 確認 0 結果 +- [x] 5.3 確認 `frontend_asset('query-tool.js')` 正確解析到 Vite manifest 中的 hashed filename diff --git a/openspec/specs/cache-indexed-query-acceleration/spec.md b/openspec/specs/cache-indexed-query-acceleration/spec.md index baf8e19..464151c 100644 --- a/openspec/specs/cache-indexed-query-acceleration/spec.md +++ b/openspec/specs/cache-indexed-query-acceleration/spec.md @@ -24,3 +24,20 @@ The system SHALL continue to maintain full-table cache behavior for `resource` a - **WHEN** cache update runs for `resource` or `wip` - **THEN** the updater MUST retain full-table snapshot semantics and MUST NOT switch these domains to partial-only cache mode +### Requirement: Mid-section defect genealogy SHALL use CONNECT BY instead of Python BFS +The mid-section-defect genealogy resolution SHALL use `LineageEngine.resolve_full_genealogy()` (CONNECT BY NOCYCLE) instead of the existing `_bfs_split_chain()` Python BFS implementation. + +#### Scenario: Genealogy cold query performance +- **WHEN** mid-section-defect analysis executes genealogy resolution with cache miss +- **THEN** `LineageEngine.resolve_split_ancestors()` SHALL be called (single CONNECT BY query) +- **THEN** response time SHALL be ≤8s (P95) for ≥50 ancestor nodes +- **THEN** Python BFS `_bfs_split_chain()` SHALL NOT be called + +#### Scenario: Genealogy hot query performance +- **WHEN** mid-section-defect analysis executes genealogy resolution with L2 Redis cache hit +- **THEN** response time SHALL be ≤1s (P95) + +#### Scenario: Golden test result equivalence +- **WHEN** golden test runs with ≥5 known LOTs +- **THEN** CONNECT BY output (`child_to_parent`, `cid_to_name`) SHALL be identical to BFS output for the same inputs + diff --git a/openspec/specs/event-fetcher-unified/spec.md b/openspec/specs/event-fetcher-unified/spec.md new file mode 100644 index 0000000..8aeb2d6 --- /dev/null +++ b/openspec/specs/event-fetcher-unified/spec.md @@ -0,0 +1,24 @@ +# event-fetcher-unified Specification + +## Purpose +TBD - created by archiving change unified-lineage-engine. Update Purpose after archive. +## Requirements +### Requirement: EventFetcher SHALL provide unified cached event querying across domains +`EventFetcher` SHALL encapsulate batch event queries with L1/L2 layered cache and rate limit bucket configuration, supporting domains: `history`, `materials`, `rejects`, `holds`, `jobs`, `upstream_history`. + +#### Scenario: Cache miss for event domain query +- **WHEN** `EventFetcher` is called for a domain with container IDs and no cache exists +- **THEN** the domain query SHALL execute against Oracle via `read_sql_df()` +- **THEN** the result SHALL be stored in L2 Redis cache with key format `evt:{domain}:{sorted_cids_hash}` +- **THEN** L1 memory cache SHALL also be populated (aligned with `core/cache.py` LayeredCache pattern) + +#### Scenario: Cache hit for event domain query +- **WHEN** `EventFetcher` is called for a domain and L2 Redis cache contains a valid entry +- **THEN** the cached result SHALL be returned without executing Oracle query +- **THEN** DB connection pool SHALL NOT be consumed + +#### Scenario: Rate limit bucket per domain +- **WHEN** `EventFetcher` is used from a route handler +- **THEN** each domain SHALL have a configurable rate limit bucket aligned with `configured_rate_limit()` pattern +- **THEN** rate limit configuration SHALL be overridable via environment variables + diff --git a/openspec/specs/lineage-engine-core/spec.md b/openspec/specs/lineage-engine-core/spec.md new file mode 100644 index 0000000..e4f823f --- /dev/null +++ b/openspec/specs/lineage-engine-core/spec.md @@ -0,0 +1,61 @@ +# lineage-engine-core Specification + +## Purpose +TBD - created by archiving change unified-lineage-engine. Update Purpose after archive. +## Requirements +### Requirement: LineageEngine SHALL provide unified split ancestor resolution via CONNECT BY NOCYCLE +`LineageEngine.resolve_split_ancestors()` SHALL accept a list of container IDs and return the complete split ancestry graph using a single Oracle `CONNECT BY NOCYCLE` query on `DW_MES_CONTAINER.SPLITFROMID`. + +#### Scenario: Normal split chain resolution +- **WHEN** `resolve_split_ancestors()` is called with a list of container IDs +- **THEN** a single SQL query using `CONNECT BY NOCYCLE` SHALL be executed against `DW_MES_CONTAINER` +- **THEN** the result SHALL include a `child_to_parent` mapping and a `cid_to_name` mapping for all discovered ancestor nodes +- **THEN** the traversal depth SHALL be limited to `LEVEL <= 20` (equivalent to existing BFS `bfs_round > 20` guard) + +#### Scenario: Large input batch exceeding Oracle IN clause limit +- **WHEN** the input `container_ids` list exceeds `ORACLE_IN_BATCH_SIZE` (1000) +- **THEN** `QueryBuilder.add_in_condition()` SHALL batch the IDs and combine results +- **THEN** all bind parameters SHALL use `QueryBuilder.params` (no string concatenation) + +#### Scenario: Cyclic split references in data +- **WHEN** `DW_MES_CONTAINER.SPLITFROMID` contains cyclic references +- **THEN** `NOCYCLE` SHALL prevent infinite traversal +- **THEN** the query SHALL return all non-cyclic ancestors up to `LEVEL <= 20` + +#### Scenario: CONNECT BY performance regression +- **WHEN** Oracle 19c execution plan for `CONNECT BY NOCYCLE` performs worse than expected +- **THEN** the SQL file SHALL contain a commented-out recursive `WITH` (recursive subquery factoring) alternative that can be swapped in without code changes + +### Requirement: LineageEngine SHALL provide unified merge source resolution +`LineageEngine.resolve_merge_sources()` SHALL accept a list of container IDs and return merge source mappings from `DW_MES_PJ_COMBINEDASSYLOTS`. + +#### Scenario: Merge source lookup +- **WHEN** `resolve_merge_sources()` is called with container IDs +- **THEN** the result SHALL include `{cid: [merge_source_cid, ...]}` for all containers that have merge sources +- **THEN** all queries SHALL use `QueryBuilder` bind params + +### Requirement: LineageEngine SHALL provide combined genealogy resolution +`LineageEngine.resolve_full_genealogy()` SHALL combine split ancestors and merge sources into a complete genealogy graph. + +#### Scenario: Full genealogy for a set of seed lots +- **WHEN** `resolve_full_genealogy()` is called with seed container IDs +- **THEN** split ancestors SHALL be resolved first via `resolve_split_ancestors()` +- **THEN** merge sources SHALL be resolved for all discovered ancestor nodes +- **THEN** the combined result SHALL be equivalent to the existing `_resolve_full_genealogy()` output in `mid_section_defect_service.py` + +### Requirement: LineageEngine functions SHALL be profile-agnostic +All `LineageEngine` public functions SHALL accept `container_ids: List[str]` and return dictionary structures without binding to any specific page logic. + +#### Scenario: Reuse from different pages +- **WHEN** a new page (e.g., wip-detail) needs lineage resolution +- **THEN** it SHALL be able to call `LineageEngine` functions directly without modification +- **THEN** no page-specific logic (profile, TMTT detection, etc.) SHALL exist in `LineageEngine` + +### Requirement: LineageEngine SQL files SHALL reside in `sql/lineage/` directory +New SQL files SHALL follow the existing `SQLLoader` convention under `src/mes_dashboard/sql/lineage/`. + +#### Scenario: SQL file organization +- **WHEN** `LineageEngine` executes queries +- **THEN** `split_ancestors.sql` and `merge_sources.sql` SHALL be loaded via `SQLLoader.load_with_params("lineage/split_ancestors", ...)` +- **THEN** the SQL files SHALL NOT reference `HM_LOTMOVEOUT` (48M row table no longer needed for genealogy) + diff --git a/openspec/specs/oracle-query-fragment-governance/spec.md b/openspec/specs/oracle-query-fragment-governance/spec.md index 001701a..59c0338 100644 --- a/openspec/specs/oracle-query-fragment-governance/spec.md +++ b/openspec/specs/oracle-query-fragment-governance/spec.md @@ -17,3 +17,25 @@ Services consuming shared Oracle query fragments SHALL preserve existing selecte - **WHEN** cache services execute queries via shared fragments - **THEN** resulting payload structure MUST remain compatible with existing aggregation and API contracts +### Requirement: Lineage SQL fragments SHALL be centralized in `sql/lineage/` directory +Split ancestor and merge source SQL queries SHALL be defined in `sql/lineage/` and shared across services via `SQLLoader`. + +#### Scenario: Mid-section-defect lineage query +- **WHEN** `mid_section_defect_service.py` needs split ancestry or merge source data +- **THEN** it SHALL call `LineageEngine` which loads SQL from `sql/lineage/split_ancestors.sql` and `sql/lineage/merge_sources.sql` +- **THEN** it SHALL NOT use `sql/mid_section_defect/split_chain.sql` or `sql/mid_section_defect/genealogy_records.sql` + +#### Scenario: Deprecated SQL file handling +- **WHEN** `sql/mid_section_defect/genealogy_records.sql` and `sql/mid_section_defect/split_chain.sql` are deprecated +- **THEN** the files SHALL be marked with a deprecated comment at the top +- **THEN** grep SHALL confirm zero `SQLLoader.load` references to these files +- **THEN** the files SHALL be retained for one version before deletion + +### Requirement: All user-input SQL queries SHALL use QueryBuilder bind params +`_build_in_filter()` and `_build_in_clause()` in `query_tool_service.py` SHALL be fully replaced by `QueryBuilder.add_in_condition()`. + +#### Scenario: Complete migration to QueryBuilder +- **WHEN** the refactoring is complete +- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results +- **THEN** all queries involving user-supplied values SHALL use `QueryBuilder.params` + diff --git a/openspec/specs/query-tool-safety-hardening/spec.md b/openspec/specs/query-tool-safety-hardening/spec.md new file mode 100644 index 0000000..d1d7541 --- /dev/null +++ b/openspec/specs/query-tool-safety-hardening/spec.md @@ -0,0 +1,61 @@ +# query-tool-safety-hardening Specification + +## Purpose +TBD - created by archiving change unified-lineage-engine. Update Purpose after archive. +## Requirements +### Requirement: query-tool resolve functions SHALL use QueryBuilder bind params for all user input +All `resolve_lots()` family functions (`_resolve_by_lot_id`, `_resolve_by_serial_number`, `_resolve_by_work_order`) SHALL use `QueryBuilder.add_in_condition()` with bind parameters instead of `_build_in_filter()` string concatenation. + +#### Scenario: Lot resolve with user-supplied values +- **WHEN** a resolve function receives user-supplied lot IDs, serial numbers, or work order names +- **THEN** the SQL query SHALL use `:p0, :p1, ...` bind parameters via `QueryBuilder` +- **THEN** `read_sql_df()` SHALL receive `builder.params` (never an empty `{}` dict for queries with user input) +- **THEN** `_build_in_filter()` and `_build_in_clause()` SHALL NOT be called + +#### Scenario: Pure static SQL without user input +- **WHEN** a query contains no user-supplied values (e.g., static lookups) +- **THEN** empty params `{}` is acceptable +- **THEN** no `_build_in_filter()` SHALL be used + +#### Scenario: Zero residual references to deprecated functions +- **WHEN** the refactoring is complete +- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results across the entire codebase + +### Requirement: query-tool routes SHALL apply rate limiting +All query-tool API endpoints SHALL apply per-client rate limiting using the existing `configured_rate_limit` mechanism. + +#### Scenario: Resolve endpoint rate limit exceeded +- **WHEN** a client sends more than 10 requests to query-tool resolve endpoints within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header +- **THEN** the resolve service function SHALL NOT be called + +#### Scenario: History endpoint rate limit exceeded +- **WHEN** a client sends more than 20 requests to query-tool history endpoints within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header + +#### Scenario: Association endpoint rate limit exceeded +- **WHEN** a client sends more than 20 requests to query-tool association endpoints within 60 seconds +- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header + +### Requirement: query-tool routes SHALL apply response caching +High-cost query-tool endpoints SHALL cache responses in L2 Redis. + +#### Scenario: Resolve result caching +- **WHEN** a resolve request succeeds +- **THEN** the response SHALL be cached in L2 Redis with TTL = 60s +- **THEN** subsequent identical requests within TTL SHALL return cached result without Oracle query + +### Requirement: lot_split_merge_history SHALL support fast and full query modes +The `lot_split_merge_history.sql` query SHALL support two modes to balance traceability completeness vs performance. + +#### Scenario: Fast mode (default) +- **WHEN** `full_history` query parameter is absent or `false` +- **THEN** the SQL SHALL include `TXNDATE >= ADD_MONTHS(SYSDATE, -6)` time window and `FETCH FIRST 500 ROWS ONLY` +- **THEN** query response time SHALL be ≤5s (P95) + +#### Scenario: Full mode +- **WHEN** `full_history=true` query parameter is provided +- **THEN** the SQL SHALL NOT include time window restriction +- **THEN** the query SHALL use `read_sql_df_slow` (120s timeout) +- **THEN** query response time SHALL be ≤60s (P95) + diff --git a/src/mes_dashboard/routes/__init__.py b/src/mes_dashboard/routes/__init__.py index faf336c..60446c6 100644 --- a/src/mes_dashboard/routes/__init__.py +++ b/src/mes_dashboard/routes/__init__.py @@ -19,6 +19,7 @@ from .query_tool_routes import query_tool_bp from .tmtt_defect_routes import tmtt_defect_bp from .qc_gate_routes import qc_gate_bp from .mid_section_defect_routes import mid_section_defect_bp +from .trace_routes import trace_bp def register_routes(app) -> None: @@ -36,6 +37,7 @@ def register_routes(app) -> None: app.register_blueprint(tmtt_defect_bp) app.register_blueprint(qc_gate_bp) app.register_blueprint(mid_section_defect_bp) + app.register_blueprint(trace_bp) __all__ = [ 'wip_bp', @@ -53,5 +55,6 @@ __all__ = [ 'tmtt_defect_bp', 'qc_gate_bp', 'mid_section_defect_bp', + 'trace_bp', 'register_routes', ] diff --git a/src/mes_dashboard/routes/query_tool_routes.py b/src/mes_dashboard/routes/query_tool_routes.py index dbea159..8313775 100644 --- a/src/mes_dashboard/routes/query_tool_routes.py +++ b/src/mes_dashboard/routes/query_tool_routes.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Query Tool API routes. +"""Query Tool API routes. Contains Flask Blueprint for batch tracing and equipment period query endpoints: - LOT resolution (LOT ID / Serial Number / Work Order → CONTAINERID) @@ -7,12 +7,16 @@ Contains Flask Blueprint for batch tracing and equipment period query endpoints: - LOT associations (materials, rejects, holds, jobs) - Equipment period queries (status hours, lots, materials, rejects, jobs) - CSV export functionality -""" - -from flask import Blueprint, jsonify, request, Response, render_template - -from mes_dashboard.core.modernization_policy import maybe_redirect_to_canonical_shell -from mes_dashboard.services.query_tool_service import ( +""" + +import hashlib + +from flask import Blueprint, jsonify, request, Response, render_template + +from mes_dashboard.core.cache import cache_get, cache_set +from mes_dashboard.core.modernization_policy import maybe_redirect_to_canonical_shell +from mes_dashboard.core.rate_limit import configured_rate_limit +from mes_dashboard.services.query_tool_service import ( resolve_lots, get_lot_history, get_adjacent_lots, @@ -33,8 +37,51 @@ from mes_dashboard.services.query_tool_service import ( validate_equipment_input, ) -# Create Blueprint -query_tool_bp = Blueprint('query_tool', __name__) +# Create Blueprint +query_tool_bp = Blueprint('query_tool', __name__) + +_QUERY_TOOL_RESOLVE_RATE_LIMIT = configured_rate_limit( + bucket="query-tool-resolve", + max_attempts_env="QT_RESOLVE_RATE_MAX_REQUESTS", + window_seconds_env="QT_RESOLVE_RATE_WINDOW_SECONDS", + default_max_attempts=10, + default_window_seconds=60, +) +_QUERY_TOOL_HISTORY_RATE_LIMIT = configured_rate_limit( + bucket="query-tool-history", + max_attempts_env="QT_HISTORY_RATE_MAX_REQUESTS", + window_seconds_env="QT_HISTORY_RATE_WINDOW_SECONDS", + default_max_attempts=20, + default_window_seconds=60, +) +_QUERY_TOOL_ASSOC_RATE_LIMIT = configured_rate_limit( + bucket="query-tool-association", + max_attempts_env="QT_ASSOC_RATE_MAX_REQUESTS", + window_seconds_env="QT_ASSOC_RATE_WINDOW_SECONDS", + default_max_attempts=20, + default_window_seconds=60, +) +_QUERY_TOOL_ADJACENT_RATE_LIMIT = configured_rate_limit( + bucket="query-tool-adjacent", + max_attempts_env="QT_ADJACENT_RATE_MAX_REQUESTS", + window_seconds_env="QT_ADJACENT_RATE_WINDOW_SECONDS", + default_max_attempts=20, + default_window_seconds=60, +) +_QUERY_TOOL_EQUIPMENT_RATE_LIMIT = configured_rate_limit( + bucket="query-tool-equipment", + max_attempts_env="QT_EQUIP_RATE_MAX_REQUESTS", + window_seconds_env="QT_EQUIP_RATE_WINDOW_SECONDS", + default_max_attempts=5, + default_window_seconds=60, +) +_QUERY_TOOL_EXPORT_RATE_LIMIT = configured_rate_limit( + bucket="query-tool-export", + max_attempts_env="QT_EXPORT_RATE_MAX_REQUESTS", + window_seconds_env="QT_EXPORT_RATE_WINDOW_SECONDS", + default_max_attempts=3, + default_window_seconds=60, +) # ============================================================ @@ -54,8 +101,9 @@ def query_tool_page(): # LOT Resolution API # ============================================================ -@query_tool_bp.route('/api/query-tool/resolve', methods=['POST']) -def resolve_lot_input(): +@query_tool_bp.route('/api/query-tool/resolve', methods=['POST']) +@_QUERY_TOOL_RESOLVE_RATE_LIMIT +def resolve_lot_input(): """Resolve user input to CONTAINERID list. Expects JSON body: @@ -86,24 +134,43 @@ def resolve_lot_input(): return jsonify({'error': f'不支援的查詢類型: {input_type}'}), 400 # Validate values - validation_error = validate_lot_input(input_type, values) - if validation_error: - return jsonify({'error': validation_error}), 400 - - result = resolve_lots(input_type, values) - - if 'error' in result: - return jsonify(result), 400 - - return jsonify(result) + validation_error = validate_lot_input(input_type, values) + if validation_error: + return jsonify({'error': validation_error}), 400 + + cache_values = [ + v.strip() + for v in values + if isinstance(v, str) and v.strip() + ] + cache_key = None + if cache_values: + values_hash = hashlib.md5( + "|".join(sorted(cache_values)).encode("utf-8") + ).hexdigest()[:16] + cache_key = f"qt:resolve:{input_type}:{values_hash}" + cached = cache_get(cache_key) + if cached is not None: + return jsonify(cached) + + result = resolve_lots(input_type, values) + + if 'error' in result: + return jsonify(result), 400 + + if cache_key is not None: + cache_set(cache_key, result, ttl=60) + + return jsonify(result) # ============================================================ # LOT History API # ============================================================ -@query_tool_bp.route('/api/query-tool/lot-history', methods=['GET']) -def query_lot_history(): +@query_tool_bp.route('/api/query-tool/lot-history', methods=['GET']) +@_QUERY_TOOL_HISTORY_RATE_LIMIT +def query_lot_history(): """Query production history for a LOT. Query params: @@ -137,8 +204,9 @@ def query_lot_history(): # Adjacent Lots API # ============================================================ -@query_tool_bp.route('/api/query-tool/adjacent-lots', methods=['GET']) -def query_adjacent_lots(): +@query_tool_bp.route('/api/query-tool/adjacent-lots', methods=['GET']) +@_QUERY_TOOL_ADJACENT_RATE_LIMIT +def query_adjacent_lots(): """Query adjacent lots (前後批) for a specific equipment. Finds lots before/after target on same equipment until different PJ_TYPE, @@ -170,16 +238,18 @@ def query_adjacent_lots(): # LOT Associations API # ============================================================ -@query_tool_bp.route('/api/query-tool/lot-associations', methods=['GET']) -def query_lot_associations(): +@query_tool_bp.route('/api/query-tool/lot-associations', methods=['GET']) +@_QUERY_TOOL_ASSOC_RATE_LIMIT +def query_lot_associations(): """Query association data for a LOT. Query params: container_id: CONTAINERID (16-char hex) type: Association type ('materials', 'rejects', 'holds', 'jobs') equipment_id: Equipment ID (required for 'jobs' type) - time_start: Start time (required for 'jobs' type) - time_end: End time (required for 'jobs' type) + time_start: Start time (required for 'jobs' type) + time_end: End time (required for 'jobs' type) + full_history: Optional boolean for 'splits' type (default false) Returns association records based on type. """ @@ -199,8 +269,9 @@ def query_lot_associations(): result = get_lot_rejects(container_id) elif assoc_type == 'holds': result = get_lot_holds(container_id) - elif assoc_type == 'splits': - result = get_lot_splits(container_id) + elif assoc_type == 'splits': + full_history = request.args.get('full_history', 'false').lower() == 'true' + result = get_lot_splits(container_id, full_history=full_history) elif assoc_type == 'jobs': equipment_id = request.args.get('equipment_id') time_start = request.args.get('time_start') @@ -221,8 +292,9 @@ def query_lot_associations(): # Equipment Period Query API # ============================================================ -@query_tool_bp.route('/api/query-tool/equipment-period', methods=['POST']) -def query_equipment_period(): +@query_tool_bp.route('/api/query-tool/equipment-period', methods=['POST']) +@_QUERY_TOOL_EQUIPMENT_RATE_LIMIT +def query_equipment_period(): """Query equipment data for a time period. Expects JSON body: @@ -362,8 +434,9 @@ def get_workcenter_groups_list(): # CSV Export API # ============================================================ -@query_tool_bp.route('/api/query-tool/export-csv', methods=['POST']) -def export_csv(): +@query_tool_bp.route('/api/query-tool/export-csv', methods=['POST']) +@_QUERY_TOOL_EXPORT_RATE_LIMIT +def export_csv(): """Export query results as CSV. Expects JSON body: diff --git a/src/mes_dashboard/routes/trace_routes.py b/src/mes_dashboard/routes/trace_routes.py new file mode 100644 index 0000000..9d7e193 --- /dev/null +++ b/src/mes_dashboard/routes/trace_routes.py @@ -0,0 +1,478 @@ +# -*- coding: utf-8 -*- +"""Staged trace API routes. + +Provides three stage endpoints for progressive trace execution: +- seed-resolve +- lineage +- events +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import time +from typing import Any, Dict, List, Optional + +from flask import Blueprint, jsonify, request + +from mes_dashboard.core.cache import cache_get, cache_set +from mes_dashboard.core.rate_limit import configured_rate_limit +from mes_dashboard.core.response import error_response +from mes_dashboard.services.event_fetcher import EventFetcher +from mes_dashboard.services.lineage_engine import LineageEngine +from mes_dashboard.services.mid_section_defect_service import ( + build_trace_aggregation_from_events, + parse_loss_reasons_param, + resolve_trace_seed_lots, +) +from mes_dashboard.services.query_tool_service import resolve_lots + +logger = logging.getLogger("mes_dashboard.trace_routes") + +trace_bp = Blueprint("trace", __name__, url_prefix="/api/trace") + +TRACE_STAGE_TIMEOUT_SECONDS = 10.0 +TRACE_CACHE_TTL_SECONDS = 300 + +PROFILE_QUERY_TOOL = "query_tool" +PROFILE_MID_SECTION_DEFECT = "mid_section_defect" +SUPPORTED_PROFILES = {PROFILE_QUERY_TOOL, PROFILE_MID_SECTION_DEFECT} + +QUERY_TOOL_RESOLVE_TYPES = {"lot_id", "serial_number", "work_order"} +SUPPORTED_EVENT_DOMAINS = { + "history", + "materials", + "rejects", + "holds", + "jobs", + "upstream_history", +} + +_TRACE_SEED_RATE_LIMIT = configured_rate_limit( + bucket="trace-seed", + max_attempts_env="TRACE_SEED_RATE_LIMIT_MAX_REQUESTS", + window_seconds_env="TRACE_SEED_RATE_LIMIT_WINDOW_SECONDS", + default_max_attempts=10, + default_window_seconds=60, +) + +_TRACE_LINEAGE_RATE_LIMIT = configured_rate_limit( + bucket="trace-lineage", + max_attempts_env="TRACE_LINEAGE_RATE_LIMIT_MAX_REQUESTS", + window_seconds_env="TRACE_LINEAGE_RATE_LIMIT_WINDOW_SECONDS", + default_max_attempts=10, + default_window_seconds=60, +) + +_TRACE_EVENTS_RATE_LIMIT = configured_rate_limit( + bucket="trace-events", + max_attempts_env="TRACE_EVENTS_RATE_LIMIT_MAX_REQUESTS", + window_seconds_env="TRACE_EVENTS_RATE_LIMIT_WINDOW_SECONDS", + default_max_attempts=15, + default_window_seconds=60, +) + + +def _json_body() -> Optional[Dict[str, Any]]: + payload = request.get_json(silent=True) + if isinstance(payload, dict): + return payload + return None + + +def _normalize_strings(values: Any) -> List[str]: + if not isinstance(values, list): + return [] + normalized: List[str] = [] + seen = set() + for value in values: + if not isinstance(value, str): + continue + text = value.strip() + if not text or text in seen: + continue + seen.add(text) + normalized.append(text) + return normalized + + +def _short_hash(parts: List[str]) -> str: + digest = hashlib.md5("|".join(parts).encode("utf-8")).hexdigest() + return digest[:12] + + +def _hash_payload(payload: Any) -> str: + dumped = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str, separators=(",", ":")) + return hashlib.md5(dumped.encode("utf-8")).hexdigest()[:12] + + +def _seed_cache_key(profile: str, params: Dict[str, Any]) -> str: + return f"trace:seed:{profile}:{_hash_payload(params)}" + + +def _lineage_cache_key(container_ids: List[str]) -> str: + return f"trace:lineage:{_short_hash(sorted(container_ids))}" + + +def _events_cache_key(profile: str, domains: List[str], container_ids: List[str]) -> str: + domains_hash = _short_hash(sorted(domains)) + cid_hash = _short_hash(sorted(container_ids)) + return f"trace:evt:{profile}:{domains_hash}:{cid_hash}" + + +def _error(code: str, message: str, status_code: int = 400): + return error_response(code, message, status_code=status_code) + + +def _timeout(stage: str): + return _error(f"{stage.upper().replace('-', '_')}_TIMEOUT", f"{stage} stage exceeded timeout budget", 504) + + +def _is_timeout_exception(exc: Exception) -> bool: + text = str(exc).lower() + timeout_fragments = ( + "timeout", + "timed out", + "ora-01013", + "dpi-1067", + "cancelled", + ) + return any(fragment in text for fragment in timeout_fragments) + + +def _validate_profile(profile: Any) -> Optional[str]: + if not isinstance(profile, str): + return None + value = profile.strip() + if value in SUPPORTED_PROFILES: + return value + return None + + +def _extract_date_range(params: Dict[str, Any]) -> tuple[Optional[str], Optional[str]]: + date_range = params.get("date_range") + if isinstance(date_range, list) and len(date_range) == 2: + start_date = str(date_range[0] or "").strip() + end_date = str(date_range[1] or "").strip() + if start_date and end_date: + return start_date, end_date + + start_date = str(params.get("start_date") or "").strip() + end_date = str(params.get("end_date") or "").strip() + if start_date and end_date: + return start_date, end_date + return None, None + + +def _seed_resolve_query_tool(params: Dict[str, Any]) -> tuple[Optional[Dict[str, Any]], Optional[tuple[str, str, int]]]: + resolve_type = str(params.get("resolve_type") or params.get("input_type") or "").strip() + if resolve_type not in QUERY_TOOL_RESOLVE_TYPES: + return None, ("INVALID_PARAMS", "resolve_type must be lot_id/serial_number/work_order", 400) + + values = _normalize_strings(params.get("values", [])) + if not values: + return None, ("INVALID_PARAMS", "values must contain at least one query value", 400) + + resolved = resolve_lots(resolve_type, values) + if not isinstance(resolved, dict): + return None, ("SEED_RESOLVE_FAILED", "seed resolve returned unexpected payload", 500) + if "error" in resolved: + return None, ("SEED_RESOLVE_FAILED", str(resolved.get("error") or "seed resolve failed"), 400) + + seeds = [] + seen = set() + for row in resolved.get("data", []): + if not isinstance(row, dict): + continue + container_id = str(row.get("container_id") or row.get("CONTAINERID") or "").strip() + if not container_id or container_id in seen: + continue + seen.add(container_id) + lot_id = str( + row.get("lot_id") + or row.get("CONTAINERNAME") + or row.get("input_value") + or container_id + ).strip() + seeds.append({ + "container_id": container_id, + "container_name": lot_id, + "lot_id": lot_id, + }) + + return {"seeds": seeds, "seed_count": len(seeds)}, None + + +def _seed_resolve_mid_section_defect( + params: Dict[str, Any], +) -> tuple[Optional[Dict[str, Any]], Optional[tuple[str, str, int]]]: + start_date, end_date = _extract_date_range(params) + if not start_date or not end_date: + return None, ("INVALID_PARAMS", "start_date/end_date (or date_range) is required", 400) + + result = resolve_trace_seed_lots(start_date, end_date) + if result is None: + return None, ("SEED_RESOLVE_FAILED", "seed resolve service unavailable", 503) + if "error" in result: + return None, ("SEED_RESOLVE_FAILED", str(result["error"]), 400) + return result, None + + +def _build_lineage_response(container_ids: List[str], ancestors_raw: Dict[str, Any]) -> Dict[str, Any]: + normalized_ancestors: Dict[str, List[str]] = {} + all_nodes = set(container_ids) + for seed in container_ids: + raw_values = ancestors_raw.get(seed, set()) + values = raw_values if isinstance(raw_values, (set, list, tuple)) else [] + normalized_list = sorted({ + str(item).strip() + for item in values + if isinstance(item, str) and str(item).strip() + }) + normalized_ancestors[seed] = normalized_list + all_nodes.update(normalized_list) + + return { + "stage": "lineage", + "ancestors": normalized_ancestors, + "merges": {}, + "total_nodes": len(all_nodes), + } + + +def _flatten_domain_records(events_by_cid: Dict[str, List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + for records in events_by_cid.values(): + if not isinstance(records, list): + continue + for row in records: + if isinstance(row, dict): + rows.append(row) + return rows + + +def _parse_lineage_payload(payload: Dict[str, Any]) -> Optional[Dict[str, Any]]: + lineage = payload.get("lineage") + if isinstance(lineage, dict): + ancestors = lineage.get("ancestors") + if isinstance(ancestors, dict): + return ancestors + direct_ancestors = payload.get("ancestors") + if isinstance(direct_ancestors, dict): + return direct_ancestors + return None + + +def _build_msd_aggregation( + payload: Dict[str, Any], + domain_results: Dict[str, Dict[str, List[Dict[str, Any]]]], +) -> tuple[Optional[Dict[str, Any]], Optional[tuple[str, str, int]]]: + params = payload.get("params") + if not isinstance(params, dict): + return None, ("INVALID_PARAMS", "params is required for mid_section_defect profile", 400) + + start_date, end_date = _extract_date_range(params) + if not start_date or not end_date: + return None, ("INVALID_PARAMS", "start_date/end_date is required in params", 400) + + raw_loss_reasons = params.get("loss_reasons") + loss_reasons = parse_loss_reasons_param(raw_loss_reasons) + + lineage_ancestors = _parse_lineage_payload(payload) + seed_container_ids = _normalize_strings(payload.get("seed_container_ids", [])) + if not seed_container_ids and isinstance(lineage_ancestors, dict): + seed_container_ids = _normalize_strings(list(lineage_ancestors.keys())) + + upstream_events = domain_results.get("upstream_history", {}) + + aggregation = build_trace_aggregation_from_events( + start_date, + end_date, + loss_reasons=loss_reasons, + seed_container_ids=seed_container_ids, + lineage_ancestors=lineage_ancestors, + upstream_events_by_cid=upstream_events, + ) + if aggregation is None: + return None, ("EVENTS_AGGREGATION_FAILED", "aggregation service unavailable", 503) + if "error" in aggregation: + return None, ("EVENTS_AGGREGATION_FAILED", str(aggregation["error"]), 400) + return aggregation, None + + +@trace_bp.route("/seed-resolve", methods=["POST"]) +@_TRACE_SEED_RATE_LIMIT +def seed_resolve(): + payload = _json_body() + if payload is None: + return _error("INVALID_PARAMS", "request body must be JSON object", 400) + + profile = _validate_profile(payload.get("profile")) + if profile is None: + return _error("INVALID_PROFILE", "unsupported profile", 400) + + params = payload.get("params") + if not isinstance(params, dict): + return _error("INVALID_PARAMS", "params must be an object", 400) + + seed_cache_key = _seed_cache_key(profile, params) + cached = cache_get(seed_cache_key) + if cached is not None: + return jsonify(cached) + + request_cache_key = payload.get("cache_key") + logger.info( + "trace seed-resolve profile=%s correlation_cache_key=%s", + profile, + request_cache_key, + ) + + started = time.monotonic() + if profile == PROFILE_QUERY_TOOL: + resolved, route_error = _seed_resolve_query_tool(params) + else: + resolved, route_error = _seed_resolve_mid_section_defect(params) + + elapsed = time.monotonic() - started + if elapsed > TRACE_STAGE_TIMEOUT_SECONDS: + return _timeout("seed_resolve") + + if route_error is not None: + code, message, status = route_error + return _error(code, message, status) + + response = { + "stage": "seed-resolve", + "seeds": resolved.get("seeds", []), + "seed_count": int(resolved.get("seed_count", 0)), + "cache_key": seed_cache_key, + } + cache_set(seed_cache_key, response, ttl=TRACE_CACHE_TTL_SECONDS) + return jsonify(response) + + +@trace_bp.route("/lineage", methods=["POST"]) +@_TRACE_LINEAGE_RATE_LIMIT +def lineage(): + payload = _json_body() + if payload is None: + return _error("INVALID_PARAMS", "request body must be JSON object", 400) + + profile = _validate_profile(payload.get("profile")) + if profile is None: + return _error("INVALID_PROFILE", "unsupported profile", 400) + + container_ids = _normalize_strings(payload.get("container_ids", [])) + if not container_ids: + return _error("INVALID_PARAMS", "container_ids must contain at least one id", 400) + + lineage_cache_key = _lineage_cache_key(container_ids) + cached = cache_get(lineage_cache_key) + if cached is not None: + return jsonify(cached) + + logger.info( + "trace lineage profile=%s count=%s correlation_cache_key=%s", + profile, + len(container_ids), + payload.get("cache_key"), + ) + + started = time.monotonic() + try: + ancestors_raw = LineageEngine.resolve_full_genealogy(container_ids) + except Exception as exc: + if _is_timeout_exception(exc): + return _error("LINEAGE_TIMEOUT", "lineage query timed out", 504) + logger.error("lineage stage failed: %s", exc, exc_info=True) + return _error("LINEAGE_FAILED", "lineage stage failed", 500) + + elapsed = time.monotonic() - started + if elapsed > TRACE_STAGE_TIMEOUT_SECONDS: + return _error("LINEAGE_TIMEOUT", "lineage query timed out", 504) + + response = _build_lineage_response(container_ids, ancestors_raw) + cache_set(lineage_cache_key, response, ttl=TRACE_CACHE_TTL_SECONDS) + return jsonify(response) + + +@trace_bp.route("/events", methods=["POST"]) +@_TRACE_EVENTS_RATE_LIMIT +def events(): + payload = _json_body() + if payload is None: + return _error("INVALID_PARAMS", "request body must be JSON object", 400) + + profile = _validate_profile(payload.get("profile")) + if profile is None: + return _error("INVALID_PROFILE", "unsupported profile", 400) + + container_ids = _normalize_strings(payload.get("container_ids", [])) + if not container_ids: + return _error("INVALID_PARAMS", "container_ids must contain at least one id", 400) + + domains = _normalize_strings(payload.get("domains", [])) + if not domains: + return _error("INVALID_PARAMS", "domains must contain at least one domain", 400) + invalid_domains = sorted(set(domains) - SUPPORTED_EVENT_DOMAINS) + if invalid_domains: + return _error( + "INVALID_PARAMS", + f"unsupported domains: {','.join(invalid_domains)}", + 400, + ) + + events_cache_key = _events_cache_key(profile, domains, container_ids) + cached = cache_get(events_cache_key) + if cached is not None: + return jsonify(cached) + + logger.info( + "trace events profile=%s domains=%s cid_count=%s correlation_cache_key=%s", + profile, + ",".join(domains), + len(container_ids), + payload.get("cache_key"), + ) + + started = time.monotonic() + results: Dict[str, Dict[str, Any]] = {} + raw_domain_results: Dict[str, Dict[str, List[Dict[str, Any]]]] = {} + failed_domains: List[str] = [] + + for domain in domains: + try: + events_by_cid = EventFetcher.fetch_events(container_ids, domain) + raw_domain_results[domain] = events_by_cid + rows = _flatten_domain_records(events_by_cid) + results[domain] = {"data": rows, "count": len(rows)} + except Exception as exc: + logger.error("events stage domain failed domain=%s: %s", domain, exc, exc_info=True) + failed_domains.append(domain) + + elapsed = time.monotonic() - started + if elapsed > TRACE_STAGE_TIMEOUT_SECONDS: + return _error("EVENTS_TIMEOUT", "events stage timed out", 504) + + aggregation = None + if profile == PROFILE_MID_SECTION_DEFECT: + aggregation, agg_error = _build_msd_aggregation(payload, raw_domain_results) + if agg_error is not None: + code, message, status = agg_error + return _error(code, message, status) + + response: Dict[str, Any] = { + "stage": "events", + "results": results, + "aggregation": aggregation, + } + + if failed_domains: + response["error"] = "one or more domains failed" + response["code"] = "EVENTS_PARTIAL_FAILURE" + response["failed_domains"] = sorted(failed_domains) + + cache_set(events_cache_key, response, ttl=TRACE_CACHE_TTL_SECONDS) + return jsonify(response) diff --git a/src/mes_dashboard/services/event_fetcher.py b/src/mes_dashboard/services/event_fetcher.py new file mode 100644 index 0000000..c56b1a9 --- /dev/null +++ b/src/mes_dashboard/services/event_fetcher.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- +"""Unified event query fetcher with cache and domain-level policy metadata.""" + +from __future__ import annotations + +import hashlib +import logging +import os +from collections import defaultdict +from typing import Any, Dict, List + +from mes_dashboard.core.cache import cache_get, cache_set +from mes_dashboard.core.database import read_sql_df +from mes_dashboard.sql import QueryBuilder, SQLLoader + +logger = logging.getLogger("mes_dashboard.event_fetcher") + +ORACLE_IN_BATCH_SIZE = 1000 + +_DOMAIN_SPECS: Dict[str, Dict[str, Any]] = { + "history": { + "filter_column": "h.CONTAINERID", + "cache_ttl": 300, + "bucket": "event-history", + "max_env": "EVT_HISTORY_RATE_MAX_REQUESTS", + "window_env": "EVT_HISTORY_RATE_WINDOW_SECONDS", + "default_max": 20, + "default_window": 60, + }, + "materials": { + "filter_column": "CONTAINERID", + "cache_ttl": 300, + "bucket": "event-materials", + "max_env": "EVT_MATERIALS_RATE_MAX_REQUESTS", + "window_env": "EVT_MATERIALS_RATE_WINDOW_SECONDS", + "default_max": 20, + "default_window": 60, + }, + "rejects": { + "filter_column": "CONTAINERID", + "cache_ttl": 300, + "bucket": "event-rejects", + "max_env": "EVT_REJECTS_RATE_MAX_REQUESTS", + "window_env": "EVT_REJECTS_RATE_WINDOW_SECONDS", + "default_max": 20, + "default_window": 60, + }, + "holds": { + "filter_column": "CONTAINERID", + "cache_ttl": 180, + "bucket": "event-holds", + "max_env": "EVT_HOLDS_RATE_MAX_REQUESTS", + "window_env": "EVT_HOLDS_RATE_WINDOW_SECONDS", + "default_max": 20, + "default_window": 60, + }, + "jobs": { + "filter_column": "j.CONTAINERIDS", + "match_mode": "contains", + "cache_ttl": 180, + "bucket": "event-jobs", + "max_env": "EVT_JOBS_RATE_MAX_REQUESTS", + "window_env": "EVT_JOBS_RATE_WINDOW_SECONDS", + "default_max": 20, + "default_window": 60, + }, + "upstream_history": { + "filter_column": "h.CONTAINERID", + "cache_ttl": 300, + "bucket": "event-upstream", + "max_env": "EVT_UPSTREAM_RATE_MAX_REQUESTS", + "window_env": "EVT_UPSTREAM_RATE_WINDOW_SECONDS", + "default_max": 20, + "default_window": 60, + }, +} + + +def _env_int(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None: + return int(default) + try: + value = int(raw) + except (TypeError, ValueError): + return int(default) + return max(value, 1) + + +def _normalize_ids(container_ids: List[str]) -> List[str]: + if not container_ids: + return [] + seen = set() + normalized: List[str] = [] + for cid in container_ids: + if not isinstance(cid, str): + continue + value = cid.strip() + if not value or value in seen: + continue + seen.add(value) + normalized.append(value) + return normalized + + +class EventFetcher: + """Fetches container-scoped event records with cache and batching.""" + + @staticmethod + def _cache_key(domain: str, container_ids: List[str]) -> str: + normalized = sorted(_normalize_ids(container_ids)) + digest = hashlib.md5("|".join(normalized).encode("utf-8")).hexdigest()[:12] + return f"evt:{domain}:{digest}" + + @staticmethod + def _get_rate_limit_config(domain: str) -> Dict[str, int | str]: + spec = _DOMAIN_SPECS.get(domain) + if spec is None: + raise ValueError(f"Unsupported event domain: {domain}") + return { + "bucket": spec["bucket"], + "max_attempts": _env_int(spec["max_env"], spec["default_max"]), + "window_seconds": _env_int(spec["window_env"], spec["default_window"]), + } + + @staticmethod + def _build_domain_sql(domain: str, condition_sql: str) -> str: + if domain == "upstream_history": + return SQLLoader.load_with_params( + "mid_section_defect/upstream_history", + ANCESTOR_FILTER=condition_sql, + ) + + if domain == "history": + sql = SQLLoader.load("query_tool/lot_history") + sql = sql.replace("h.CONTAINERID = :container_id", condition_sql) + return sql.replace("{{ WORKCENTER_FILTER }}", "") + + if domain == "materials": + sql = SQLLoader.load("query_tool/lot_materials") + return sql.replace("CONTAINERID = :container_id", condition_sql) + + if domain == "rejects": + sql = SQLLoader.load("query_tool/lot_rejects") + return sql.replace("CONTAINERID = :container_id", condition_sql) + + if domain == "holds": + sql = SQLLoader.load("query_tool/lot_holds") + return sql.replace("CONTAINERID = :container_id", condition_sql) + + if domain == "jobs": + return f""" + SELECT + j.JOBID, + j.RESOURCEID, + j.RESOURCENAME, + j.JOBSTATUS, + j.JOBMODELNAME, + j.JOBORDERNAME, + j.CREATEDATE, + j.COMPLETEDATE, + j.CAUSECODENAME, + j.REPAIRCODENAME, + j.SYMPTOMCODENAME, + j.CONTAINERIDS, + j.CONTAINERNAMES, + NULL AS CONTAINERID + FROM DWH.DW_MES_JOB j + WHERE {condition_sql} + ORDER BY j.CREATEDATE DESC + """ + + raise ValueError(f"Unsupported event domain: {domain}") + + @staticmethod + def fetch_events( + container_ids: List[str], + domain: str, + ) -> Dict[str, List[Dict[str, Any]]]: + """Fetch event records grouped by CONTAINERID.""" + if domain not in _DOMAIN_SPECS: + raise ValueError(f"Unsupported event domain: {domain}") + + normalized_ids = _normalize_ids(container_ids) + if not normalized_ids: + return {} + + cache_key = EventFetcher._cache_key(domain, normalized_ids) + cached = cache_get(cache_key) + if cached is not None: + return cached + + grouped: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + spec = _DOMAIN_SPECS[domain] + filter_column = spec["filter_column"] + match_mode = spec.get("match_mode", "in") + + for i in range(0, len(normalized_ids), ORACLE_IN_BATCH_SIZE): + batch = normalized_ids[i:i + ORACLE_IN_BATCH_SIZE] + builder = QueryBuilder() + if match_mode == "contains": + builder.add_or_like_conditions(filter_column, batch, position="both") + else: + builder.add_in_condition(filter_column, batch) + + sql = EventFetcher._build_domain_sql(domain, builder.get_conditions_sql()) + df = read_sql_df(sql, builder.params) + if df is None or df.empty: + continue + + for _, row in df.iterrows(): + if domain == "jobs": + record = row.to_dict() + containers = record.get("CONTAINERIDS") + if not isinstance(containers, str) or not containers: + continue + for cid in batch: + if cid in containers: + enriched = dict(record) + enriched["CONTAINERID"] = cid + grouped[cid].append(enriched) + continue + + cid = row.get("CONTAINERID") + if not isinstance(cid, str) or not cid: + continue + grouped[cid].append(row.to_dict()) + + result = dict(grouped) + cache_set(cache_key, result, ttl=_DOMAIN_SPECS[domain]["cache_ttl"]) + logger.info( + "EventFetcher fetched domain=%s queried_cids=%s hit_cids=%s", + domain, + len(normalized_ids), + len(result), + ) + return result diff --git a/src/mes_dashboard/services/lineage_engine.py b/src/mes_dashboard/services/lineage_engine.py new file mode 100644 index 0000000..82ec0b1 --- /dev/null +++ b/src/mes_dashboard/services/lineage_engine.py @@ -0,0 +1,221 @@ +# -*- coding: utf-8 -*- +"""Unified LOT lineage resolution helpers.""" + +from __future__ import annotations + +import logging +from collections import defaultdict +from typing import Any, Dict, List, Optional, Set + +from mes_dashboard.core.database import read_sql_df +from mes_dashboard.sql import QueryBuilder, SQLLoader + +logger = logging.getLogger("mes_dashboard.lineage_engine") + +ORACLE_IN_BATCH_SIZE = 1000 +MAX_SPLIT_DEPTH = 20 + + +def _normalize_list(values: List[str]) -> List[str]: + """Normalize string list while preserving input order.""" + if not values: + return [] + seen = set() + normalized: List[str] = [] + for value in values: + if not isinstance(value, str): + continue + text = value.strip() + if not text or text in seen: + continue + seen.add(text) + normalized.append(text) + return normalized + + +def _safe_str(value: Any) -> Optional[str]: + """Convert value to non-empty string if possible.""" + if not isinstance(value, str): + return None + value = value.strip() + return value if value else None + + +class LineageEngine: + """Unified split/merge genealogy resolver.""" + + @staticmethod + def resolve_split_ancestors( + container_ids: List[str], + initial_names: Optional[Dict[str, str]] = None, + ) -> Dict[str, Dict[str, str]]: + """Resolve split lineage with CONNECT BY NOCYCLE. + + Returns: + { + "child_to_parent": {child_cid: parent_cid}, + "cid_to_name": {cid: container_name}, + } + """ + normalized_cids = _normalize_list(container_ids) + child_to_parent: Dict[str, str] = {} + cid_to_name: Dict[str, str] = { + cid: name + for cid, name in (initial_names or {}).items() + if _safe_str(cid) and _safe_str(name) + } + + if not normalized_cids: + return {"child_to_parent": child_to_parent, "cid_to_name": cid_to_name} + + for i in range(0, len(normalized_cids), ORACLE_IN_BATCH_SIZE): + batch = normalized_cids[i:i + ORACLE_IN_BATCH_SIZE] + builder = QueryBuilder() + builder.add_in_condition("c.CONTAINERID", batch) + + sql = SQLLoader.load_with_params( + "lineage/split_ancestors", + CID_FILTER=builder.get_conditions_sql(), + ) + + df = read_sql_df(sql, builder.params) + if df is None or df.empty: + continue + + for _, row in df.iterrows(): + cid = _safe_str(row.get("CONTAINERID")) + if not cid: + continue + + name = _safe_str(row.get("CONTAINERNAME")) + if name: + cid_to_name[cid] = name + + depth_raw = row.get("SPLIT_DEPTH") + depth = int(depth_raw) if depth_raw is not None else 0 + if depth > MAX_SPLIT_DEPTH: + continue + + parent = _safe_str(row.get("SPLITFROMID")) + if parent and parent != cid: + child_to_parent.setdefault(cid, parent) + + logger.info( + "Split ancestor resolution completed: seed=%s, edges=%s, names=%s", + len(normalized_cids), + len(child_to_parent), + len(cid_to_name), + ) + return {"child_to_parent": child_to_parent, "cid_to_name": cid_to_name} + + @staticmethod + def resolve_merge_sources( + container_names: List[str], + ) -> Dict[str, List[str]]: + """Resolve merge source lots from FINISHEDNAME.""" + normalized_names = _normalize_list(container_names) + if not normalized_names: + return {} + + result: Dict[str, Set[str]] = defaultdict(set) + + for i in range(0, len(normalized_names), ORACLE_IN_BATCH_SIZE): + batch = normalized_names[i:i + ORACLE_IN_BATCH_SIZE] + builder = QueryBuilder() + builder.add_in_condition("ca.FINISHEDNAME", batch) + + sql = SQLLoader.load_with_params( + "lineage/merge_sources", + FINISHED_NAME_FILTER=builder.get_conditions_sql(), + ) + + df = read_sql_df(sql, builder.params) + if df is None or df.empty: + continue + + for _, row in df.iterrows(): + finished_name = _safe_str(row.get("FINISHEDNAME")) + source_cid = _safe_str(row.get("SOURCE_CID")) + if not finished_name or not source_cid: + continue + result[finished_name].add(source_cid) + + mapped = {k: sorted(v) for k, v in result.items()} + logger.info( + "Merge source resolution completed: finished_names=%s, mapped=%s", + len(normalized_names), + len(mapped), + ) + return mapped + + @staticmethod + def resolve_full_genealogy( + container_ids: List[str], + initial_names: Optional[Dict[str, str]] = None, + ) -> Dict[str, Set[str]]: + """Resolve combined split + merge genealogy graph. + + Returns: + {seed_cid: set(ancestor_cids)} + """ + seed_cids = _normalize_list(container_ids) + if not seed_cids: + return {} + + split_result = LineageEngine.resolve_split_ancestors(seed_cids, initial_names) + child_to_parent = split_result["child_to_parent"] + cid_to_name = split_result["cid_to_name"] + + ancestors: Dict[str, Set[str]] = {} + for seed in seed_cids: + visited: Set[str] = set() + current = seed + depth = 0 + while current in child_to_parent and depth < MAX_SPLIT_DEPTH: + depth += 1 + parent = child_to_parent[current] + if parent in visited: + break + visited.add(parent) + current = parent + ancestors[seed] = visited + + all_names = [name for name in cid_to_name.values() if _safe_str(name)] + merge_source_map = LineageEngine.resolve_merge_sources(all_names) + if not merge_source_map: + return ancestors + + merge_source_cids_all: Set[str] = set() + for seed in seed_cids: + self_and_ancestors = ancestors[seed] | {seed} + for cid in list(self_and_ancestors): + name = cid_to_name.get(cid) + if not name: + continue + for source_cid in merge_source_map.get(name, []): + if source_cid == cid or source_cid in self_and_ancestors: + continue + ancestors[seed].add(source_cid) + merge_source_cids_all.add(source_cid) + + seen = set(seed_cids) | set(child_to_parent.keys()) | set(child_to_parent.values()) + new_merge_cids = list(merge_source_cids_all - seen) + if not new_merge_cids: + return ancestors + + merge_split_result = LineageEngine.resolve_split_ancestors(new_merge_cids) + merge_child_to_parent = merge_split_result["child_to_parent"] + + for seed in seed_cids: + for merge_cid in list(ancestors[seed] & merge_source_cids_all): + current = merge_cid + depth = 0 + while current in merge_child_to_parent and depth < MAX_SPLIT_DEPTH: + depth += 1 + parent = merge_child_to_parent[current] + if parent in ancestors[seed]: + break + ancestors[seed].add(parent) + current = parent + + return ancestors diff --git a/src/mes_dashboard/services/mid_section_defect_service.py b/src/mes_dashboard/services/mid_section_defect_service.py index f913cfa..66acfe7 100644 --- a/src/mes_dashboard/services/mid_section_defect_service.py +++ b/src/mes_dashboard/services/mid_section_defect_service.py @@ -33,15 +33,16 @@ import pandas as pd from mes_dashboard.core.database import read_sql_df from mes_dashboard.core.cache import cache_get, cache_set, make_cache_key from mes_dashboard.core.redis_client import try_acquire_lock, release_lock -from mes_dashboard.sql import SQLLoader, QueryBuilder +from mes_dashboard.sql import SQLLoader +from mes_dashboard.services.event_fetcher import EventFetcher +from mes_dashboard.services.lineage_engine import LineageEngine logger = logging.getLogger('mes_dashboard.mid_section_defect') # Constants MAX_QUERY_DAYS = 180 CACHE_TTL_TMTT = 300 # 5 min for TMTT detection data -CACHE_TTL_LOSS_REASONS = 86400 # 24h for loss reason list (daily sync) -ORACLE_IN_BATCH_SIZE = 1000 # Oracle IN clause limit +CACHE_TTL_LOSS_REASONS = 86400 # 24h for loss reason list (daily sync) # Distributed lock settings for query_analysis cold-cache path ANALYSIS_LOCK_TTL_SECONDS = 120 @@ -82,11 +83,11 @@ CSV_COLUMNS = [ # Public API # ============================================================ -def query_analysis( - start_date: str, - end_date: str, - loss_reasons: Optional[List[str]] = None, -) -> Optional[Dict[str, Any]]: +def query_analysis( + start_date: str, + end_date: str, + loss_reasons: Optional[List[str]] = None, +) -> Optional[Dict[str, Any]]: """Main entry point for mid-section defect traceability analysis. Args: @@ -217,12 +218,161 @@ def query_analysis( finally: if lock_acquired: release_lock(lock_name) - - -def query_analysis_detail( - start_date: str, - end_date: str, - loss_reasons: Optional[List[str]] = None, + + +def parse_loss_reasons_param(loss_reasons: Any) -> Optional[List[str]]: + """Normalize loss reason input from API payloads. + + Accepts comma-separated strings or list-like inputs. + Returns None when no valid value is provided. + """ + if loss_reasons is None: + return None + + values: List[str] + if isinstance(loss_reasons, str): + values = [item.strip() for item in loss_reasons.split(',') if item.strip()] + elif isinstance(loss_reasons, (list, tuple, set)): + values = [] + for item in loss_reasons: + if not isinstance(item, str): + continue + text = item.strip() + if text: + values.append(text) + else: + return None + + if not values: + return None + + deduped: List[str] = [] + seen = set() + for value in values: + if value in seen: + continue + seen.add(value) + deduped.append(value) + return deduped or None + + +def resolve_trace_seed_lots( + start_date: str, + end_date: str, +) -> Optional[Dict[str, Any]]: + """Resolve seed lots for staged mid-section trace API.""" + error = _validate_date_range(start_date, end_date) + if error: + return {'error': error} + + tmtt_df = _fetch_tmtt_data(start_date, end_date) + if tmtt_df is None: + return None + if tmtt_df.empty: + return {'seeds': [], 'seed_count': 0} + + seeds = [] + unique_rows = tmtt_df.drop_duplicates(subset=['CONTAINERID']) + for _, row in unique_rows.iterrows(): + cid = _safe_str(row.get('CONTAINERID')) + if not cid: + continue + lot_id = _safe_str(row.get('CONTAINERNAME')) or cid + seeds.append({ + 'container_id': cid, + 'container_name': lot_id, + 'lot_id': lot_id, + }) + + seeds.sort(key=lambda item: (item.get('lot_id', ''), item.get('container_id', ''))) + return { + 'seeds': seeds, + 'seed_count': len(seeds), + } + + +def build_trace_aggregation_from_events( + start_date: str, + end_date: str, + *, + loss_reasons: Optional[List[str]] = None, + seed_container_ids: Optional[List[str]] = None, + lineage_ancestors: Optional[Dict[str, Any]] = None, + upstream_events_by_cid: Optional[Dict[str, List[Dict[str, Any]]]] = None, +) -> Optional[Dict[str, Any]]: + """Build mid-section summary payload from staged events data.""" + error = _validate_date_range(start_date, end_date) + if error: + return {'error': error} + + normalized_loss_reasons = parse_loss_reasons_param(loss_reasons) + + tmtt_df = _fetch_tmtt_data(start_date, end_date) + if tmtt_df is None: + return None + if tmtt_df.empty: + empty_result = _empty_result() + return { + 'kpi': empty_result['kpi'], + 'charts': empty_result['charts'], + 'daily_trend': empty_result['daily_trend'], + 'available_loss_reasons': empty_result['available_loss_reasons'], + 'genealogy_status': empty_result['genealogy_status'], + 'detail_total_count': 0, + } + + available_loss_reasons = sorted( + tmtt_df.loc[tmtt_df['REJECTQTY'] > 0, 'LOSSREASONNAME'] + .dropna().unique().tolist() + ) + + if normalized_loss_reasons: + filtered_df = tmtt_df[ + (tmtt_df['LOSSREASONNAME'].isin(normalized_loss_reasons)) + | (tmtt_df['REJECTQTY'] == 0) + | (tmtt_df['LOSSREASONNAME'].isna()) + ].copy() + else: + filtered_df = tmtt_df + + tmtt_data = _build_tmtt_lookup(filtered_df) + normalized_ancestors = _normalize_lineage_ancestors( + lineage_ancestors, + seed_container_ids=seed_container_ids, + fallback_seed_ids=list(tmtt_data.keys()), + ) + normalized_upstream = _normalize_upstream_event_records(upstream_events_by_cid or {}) + + attribution = _attribute_defects( + tmtt_data, + normalized_ancestors, + normalized_upstream, + normalized_loss_reasons, + ) + detail = _build_detail_table(filtered_df, normalized_ancestors, normalized_upstream) + + seed_ids = [ + cid for cid in (seed_container_ids or list(tmtt_data.keys())) + if isinstance(cid, str) and cid.strip() + ] + genealogy_status = 'ready' + if seed_ids and lineage_ancestors is None: + genealogy_status = 'error' + + return { + 'kpi': _build_kpi(filtered_df, attribution, normalized_loss_reasons), + 'charts': _build_all_charts(attribution, tmtt_data), + 'daily_trend': _build_daily_trend(filtered_df, normalized_loss_reasons), + 'available_loss_reasons': available_loss_reasons, + 'genealogy_status': genealogy_status, + 'detail_total_count': len(detail), + } + + +def query_analysis_detail( + start_date: str, + end_date: str, + loss_reasons: Optional[List[str]] = None, page: int = 1, page_size: int = 200, ) -> Optional[Dict[str, Any]]: @@ -428,193 +578,14 @@ def _fetch_tmtt_data(start_date: str, end_date: str) -> Optional[pd.DataFrame]: # Query 2: LOT Genealogy # ============================================================ -def _resolve_full_genealogy( - tmtt_cids: List[str], - tmtt_names: Dict[str, str], -) -> Dict[str, Set[str]]: - """Resolve full genealogy for TMTT lots via SPLITFROMID + COMBINEDASSYLOTS. - - Step 1: BFS upward through DW_MES_CONTAINER.SPLITFROMID - Step 2: Merge expansion via DW_MES_PJ_COMBINEDASSYLOTS - Step 3: BFS on merge source CIDs (one more round) - - Args: - tmtt_cids: TMTT lot CONTAINERIDs - tmtt_names: {cid: containername} from TMTT detection data - - Returns: - {tmtt_cid: set(all ancestor CIDs)} - """ - # ---- Step 1: Split chain BFS upward ---- - child_to_parent, cid_to_name = _bfs_split_chain(tmtt_cids, tmtt_names) - - # Build initial ancestor sets per TMTT lot (walk up split chain) - ancestors: Dict[str, Set[str]] = {} - for tmtt_cid in tmtt_cids: - visited: Set[str] = set() - current = tmtt_cid - while current in child_to_parent: - parent = child_to_parent[current] - if parent in visited: - break # cycle protection - visited.add(parent) - current = parent - ancestors[tmtt_cid] = visited - - # ---- Step 2: Merge expansion via COMBINEDASSYLOTS ---- - all_names = set(cid_to_name.values()) - if not all_names: - _log_genealogy_summary(ancestors, tmtt_cids, 0) - return ancestors - - merge_source_map = _fetch_merge_sources(list(all_names)) - if not merge_source_map: - _log_genealogy_summary(ancestors, tmtt_cids, 0) - return ancestors - - # Reverse map: name → set of CIDs with that name - name_to_cids: Dict[str, Set[str]] = defaultdict(set) - for cid, name in cid_to_name.items(): - name_to_cids[name].add(cid) - - # Expand ancestors with merge sources - merge_source_cids_all: Set[str] = set() - for tmtt_cid in tmtt_cids: - self_and_ancestors = ancestors[tmtt_cid] | {tmtt_cid} - for cid in list(self_and_ancestors): - name = cid_to_name.get(cid) - if name and name in merge_source_map: - for src_cid in merge_source_map[name]: - if src_cid != cid and src_cid not in self_and_ancestors: - ancestors[tmtt_cid].add(src_cid) - merge_source_cids_all.add(src_cid) - - # ---- Step 3: BFS on merge source CIDs ---- - seen = set(tmtt_cids) | set(child_to_parent.values()) | set(child_to_parent.keys()) - new_merge_cids = list(merge_source_cids_all - seen) - if new_merge_cids: - merge_c2p, _ = _bfs_split_chain(new_merge_cids, {}) - child_to_parent.update(merge_c2p) - - # Walk up merge sources' split chains for each TMTT lot - for tmtt_cid in tmtt_cids: - for merge_cid in list(ancestors[tmtt_cid] & merge_source_cids_all): - current = merge_cid - while current in merge_c2p: - parent = merge_c2p[current] - if parent in ancestors[tmtt_cid]: - break - ancestors[tmtt_cid].add(parent) - current = parent - - _log_genealogy_summary(ancestors, tmtt_cids, len(merge_source_cids_all)) - return ancestors - - -def _bfs_split_chain( - start_cids: List[str], - initial_names: Dict[str, str], -) -> Tuple[Dict[str, str], Dict[str, str]]: - """BFS upward through DW_MES_CONTAINER.SPLITFROMID. - - Args: - start_cids: Starting CONTAINERIDs - initial_names: Pre-known {cid: containername} mappings - - Returns: - child_to_parent: {child_cid: parent_cid} for all split edges - cid_to_name: {cid: containername} for all encountered CIDs - """ - child_to_parent: Dict[str, str] = {} - cid_to_name: Dict[str, str] = dict(initial_names) - seen: Set[str] = set(start_cids) - frontier = list(start_cids) - bfs_round = 0 - - while frontier: - bfs_round += 1 - batch_results: List[Dict[str, Any]] = [] - - for i in range(0, len(frontier), ORACLE_IN_BATCH_SIZE): - batch = frontier[i:i + ORACLE_IN_BATCH_SIZE] - builder = QueryBuilder() - builder.add_in_condition("c.CONTAINERID", batch) - sql = SQLLoader.load_with_params( - "mid_section_defect/split_chain", - CID_FILTER=builder.get_conditions_sql(), - ) - try: - df = read_sql_df(sql, builder.params) - if df is not None and not df.empty: - batch_results.extend(df.to_dict('records')) - except Exception as exc: - logger.warning(f"Split chain BFS round {bfs_round} batch failed: {exc}") - - new_parents: Set[str] = set() - for row in batch_results: - cid = row['CONTAINERID'] - split_from = row.get('SPLITFROMID') - name = row.get('CONTAINERNAME') - - if isinstance(name, str) and name: - cid_to_name[cid] = name - if isinstance(split_from, str) and split_from and cid != split_from: - child_to_parent[cid] = split_from - if split_from not in seen: - new_parents.add(split_from) - seen.add(split_from) - - frontier = list(new_parents) - if bfs_round > 20: - logger.warning("Split chain BFS exceeded 20 rounds, stopping") - break - - logger.info( - f"Split chain BFS: {bfs_round} rounds, " - f"{len(child_to_parent)} split edges, " - f"{len(cid_to_name)} names collected" - ) - return child_to_parent, cid_to_name - - -def _fetch_merge_sources( - finished_names: List[str], -) -> Dict[str, List[str]]: - """Find source lots merged into finished lots via COMBINEDASSYLOTS. - - Args: - finished_names: CONTAINERNAMEs to look up as FINISHEDNAME - - Returns: - {finished_name: [source_cid, ...]} - """ - result: Dict[str, List[str]] = {} - - for i in range(0, len(finished_names), ORACLE_IN_BATCH_SIZE): - batch = finished_names[i:i + ORACLE_IN_BATCH_SIZE] - builder = QueryBuilder() - builder.add_in_condition("ca.FINISHEDNAME", batch) - sql = SQLLoader.load_with_params( - "mid_section_defect/merge_lookup", - FINISHED_NAME_FILTER=builder.get_conditions_sql(), - ) - try: - df = read_sql_df(sql, builder.params) - if df is not None and not df.empty: - for _, row in df.iterrows(): - fn = row['FINISHEDNAME'] - src = row['SOURCE_CID'] - if isinstance(fn, str) and fn and isinstance(src, str) and src: - result.setdefault(fn, []).append(src) - except Exception as exc: - logger.warning(f"Merge lookup batch failed: {exc}") - - if result: - total_sources = sum(len(v) for v in result.values()) - logger.info( - f"Merge lookup: {len(result)} finished names → {total_sources} source CIDs" - ) - return result +def _resolve_full_genealogy( + tmtt_cids: List[str], + tmtt_names: Dict[str, str], +) -> Dict[str, Set[str]]: + """Resolve full genealogy for TMTT lots via shared LineageEngine.""" + ancestors = LineageEngine.resolve_full_genealogy(tmtt_cids, tmtt_names) + _log_genealogy_summary(ancestors, tmtt_cids, 0) + return ancestors def _log_genealogy_summary( @@ -646,60 +617,81 @@ def _fetch_upstream_history( Returns: {containerid: [{'workcenter_group': ..., 'equipment_name': ..., ...}, ...]} """ - if not all_cids: - return {} - - unique_cids = list(set(all_cids)) - all_rows = [] - - # Batch query in chunks of ORACLE_IN_BATCH_SIZE - for i in range(0, len(unique_cids), ORACLE_IN_BATCH_SIZE): - batch = unique_cids[i:i + ORACLE_IN_BATCH_SIZE] - - builder = QueryBuilder() - builder.add_in_condition("h.CONTAINERID", batch) - conditions_sql = builder.get_conditions_sql() - params = builder.params - - sql = SQLLoader.load_with_params( - "mid_section_defect/upstream_history", - ANCESTOR_FILTER=conditions_sql, - ) - - try: - df = read_sql_df(sql, params) - if df is not None and not df.empty: - all_rows.append(df) - except Exception as exc: - logger.error( - f"Upstream history batch {i//ORACLE_IN_BATCH_SIZE + 1} failed: {exc}", - exc_info=True, - ) - - if not all_rows: - return {} - - combined = pd.concat(all_rows, ignore_index=True) - - result: Dict[str, List[Dict[str, Any]]] = defaultdict(list) - for _, row in combined.iterrows(): - cid = row['CONTAINERID'] - group_name = _safe_str(row.get('WORKCENTER_GROUP')) - if not group_name: - group_name = '(未知)' - result[cid].append({ - 'workcenter_group': group_name, - 'equipment_id': _safe_str(row.get('EQUIPMENTID')), - 'equipment_name': _safe_str(row.get('EQUIPMENTNAME')), - 'spec_name': _safe_str(row.get('SPECNAME')), - 'track_in_time': _safe_str(row.get('TRACKINTIMESTAMP')), - }) + if not all_cids: + return {} + + unique_cids = list(set(all_cids)) + events_by_cid = EventFetcher.fetch_events(unique_cids, "upstream_history") + result = _normalize_upstream_event_records(events_by_cid) logger.info( f"Upstream history: {len(result)} lots with classified records, " f"from {len(unique_cids)} queried CIDs" ) - return dict(result) + return dict(result) + + +def _normalize_lineage_ancestors( + lineage_ancestors: Optional[Dict[str, Any]], + *, + seed_container_ids: Optional[List[str]] = None, + fallback_seed_ids: Optional[List[str]] = None, +) -> Dict[str, Set[str]]: + """Normalize lineage payload to {seed_cid: set(ancestor_cid)}.""" + ancestors: Dict[str, Set[str]] = {} + + if isinstance(lineage_ancestors, dict): + for seed, raw_values in lineage_ancestors.items(): + seed_cid = _safe_str(seed) + if not seed_cid: + continue + + values = raw_values if isinstance(raw_values, (list, tuple, set)) else [] + normalized_values: Set[str] = set() + for value in values: + ancestor_cid = _safe_str(value) + if ancestor_cid and ancestor_cid != seed_cid: + normalized_values.add(ancestor_cid) + ancestors[seed_cid] = normalized_values + + candidate_seeds = [] + for seed in (seed_container_ids or []): + seed_cid = _safe_str(seed) + if seed_cid: + candidate_seeds.append(seed_cid) + if not candidate_seeds: + for seed in (fallback_seed_ids or []): + seed_cid = _safe_str(seed) + if seed_cid: + candidate_seeds.append(seed_cid) + + for seed_cid in candidate_seeds: + ancestors.setdefault(seed_cid, set()) + + return ancestors + + +def _normalize_upstream_event_records( + events_by_cid: Dict[str, List[Dict[str, Any]]], +) -> Dict[str, List[Dict[str, Any]]]: + """Normalize EventFetcher upstream payload into attribution-ready records.""" + result: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for cid, events in events_by_cid.items(): + cid_value = _safe_str(cid) + if not cid_value: + continue + for event in events: + group_name = _safe_str(event.get('WORKCENTER_GROUP')) + if not group_name: + group_name = '(未知)' + result[cid_value].append({ + 'workcenter_group': group_name, + 'equipment_id': _safe_str(event.get('EQUIPMENTID')), + 'equipment_name': _safe_str(event.get('EQUIPMENTNAME')), + 'spec_name': _safe_str(event.get('SPECNAME')), + 'track_in_time': _safe_str(event.get('TRACKINTIMESTAMP')), + }) + return dict(result) # ============================================================ diff --git a/src/mes_dashboard/services/query_tool_service.py b/src/mes_dashboard/services/query_tool_service.py index 915aac0..27554b3 100644 --- a/src/mes_dashboard/services/query_tool_service.py +++ b/src/mes_dashboard/services/query_tool_service.py @@ -18,14 +18,15 @@ Architecture: import csv import io import logging -from datetime import datetime, timedelta -from decimal import Decimal -from typing import Any, Dict, List, Optional, Generator - +from datetime import datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Generator + import pandas as pd from mes_dashboard.core.database import read_sql_df -from mes_dashboard.sql import SQLLoader +from mes_dashboard.sql import QueryBuilder, SQLLoader +from mes_dashboard.services.event_fetcher import EventFetcher try: from mes_dashboard.core.database import read_sql_df_slow @@ -122,59 +123,7 @@ def validate_equipment_input(equipment_ids: List[str]) -> Optional[str]: return None -# ============================================================ -# Helper Functions -# ============================================================ - -def _build_in_clause(values: List[str], max_chunk_size: int = BATCH_SIZE) -> List[str]: - """Build SQL IN clause lists for values. - - Oracle has a limit of ~1000 items per IN clause, so we chunk if needed. - - Args: - values: List of values. - max_chunk_size: Maximum items per IN clause. - - Returns: - List of SQL IN clause strings (e.g., "'val1', 'val2', 'val3'"). - """ - if not values: - return [] - - # Escape single quotes - escaped = [v.replace("'", "''") for v in values] - - # Chunk into groups - chunks = [] - for i in range(0, len(escaped), max_chunk_size): - chunk = escaped[i:i + max_chunk_size] - chunks.append("'" + "', '".join(chunk) + "'") - - return chunks - - -def _build_in_filter(values: List[str], column: str) -> str: - """Build SQL IN filter clause. - - Args: - values: List of values. - column: Column name. - - Returns: - SQL condition string. - """ - chunks = _build_in_clause(values) - if not chunks: - return "1=0" - - if len(chunks) == 1: - return f"{column} IN ({chunks[0]})" - - conditions = [f"{column} IN ({chunk})" for chunk in chunks] - return "(" + " OR ".join(conditions) + ")" - - -def _df_to_records(df: pd.DataFrame) -> List[Dict[str, Any]]: +def _df_to_records(df: pd.DataFrame) -> List[Dict[str, Any]]: """Convert DataFrame to list of records with proper type handling. Args: @@ -250,7 +199,7 @@ def resolve_lots(input_type: str, values: List[str]) -> Dict[str, Any]: return {'error': f'解析失敗: {str(exc)}'} -def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]: +def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]: """Resolve LOT IDs (CONTAINERNAME) to CONTAINERID. Args: @@ -259,23 +208,14 @@ def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]: Returns: Resolution result dict. """ - in_filter = _build_in_filter(lot_ids, 'CONTAINERNAME') - sql = SQLLoader.load("query_tool/lot_resolve_id") - sql = sql.replace("{{ CONTAINER_NAMES }}", in_filter.replace("CONTAINERNAME IN (", "").rstrip(")")) - - # Direct IN clause construction - sql = f""" - SELECT - CONTAINERID, - CONTAINERNAME, - MFGORDERNAME, - SPECNAME, - QTY - FROM DWH.DW_MES_CONTAINER - WHERE {in_filter} - """ - - df = read_sql_df(sql, {}) + builder = QueryBuilder() + builder.add_in_condition("CONTAINERNAME", lot_ids) + sql = SQLLoader.load_with_params( + "query_tool/lot_resolve_id", + CONTAINER_FILTER=builder.get_conditions_sql(), + ) + + df = read_sql_df(sql, builder.params) data = _df_to_records(df) # Map results @@ -305,7 +245,7 @@ def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]: } -def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]: +def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]: """Resolve serial numbers (FINISHEDNAME) to CONTAINERID. Note: One serial number may map to multiple CONTAINERIDs. @@ -316,21 +256,14 @@ def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]: Returns: Resolution result dict. """ - in_filter = _build_in_filter(serial_numbers, 'p.FINISHEDNAME') - - # JOIN with CONTAINER to get LOT ID (CONTAINERNAME) - sql = f""" - SELECT DISTINCT - p.CONTAINERID, - p.FINISHEDNAME, - c.CONTAINERNAME, - c.SPECNAME - FROM DWH.DW_MES_PJ_COMBINEDASSYLOTS p - LEFT JOIN DWH.DW_MES_CONTAINER c ON p.CONTAINERID = c.CONTAINERID - WHERE {in_filter} - """ - - df = read_sql_df(sql, {}) + builder = QueryBuilder() + builder.add_in_condition("p.FINISHEDNAME", serial_numbers) + sql = SQLLoader.load_with_params( + "query_tool/lot_resolve_serial", + SERIAL_FILTER=builder.get_conditions_sql(), + ) + + df = read_sql_df(sql, builder.params) data = _df_to_records(df) # Group by serial number @@ -370,7 +303,7 @@ def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]: } -def _resolve_by_work_order(work_orders: List[str]) -> Dict[str, Any]: +def _resolve_by_work_order(work_orders: List[str]) -> Dict[str, Any]: """Resolve work orders (PJ_WORKORDER) to CONTAINERID. Note: One work order may expand to many CONTAINERIDs (can be 100+). @@ -381,21 +314,14 @@ def _resolve_by_work_order(work_orders: List[str]) -> Dict[str, Any]: Returns: Resolution result dict. """ - in_filter = _build_in_filter(work_orders, 'h.PJ_WORKORDER') - - # JOIN with CONTAINER to get LOT ID (CONTAINERNAME) - sql = f""" - SELECT DISTINCT - h.CONTAINERID, - h.PJ_WORKORDER, - c.CONTAINERNAME, - c.SPECNAME - FROM DWH.DW_MES_LOTWIPHISTORY h - LEFT JOIN DWH.DW_MES_CONTAINER c ON h.CONTAINERID = c.CONTAINERID - WHERE {in_filter} - """ - - df = read_sql_df(sql, {}) + builder = QueryBuilder() + builder.add_in_condition("h.PJ_WORKORDER", work_orders) + sql = SQLLoader.load_with_params( + "query_tool/lot_resolve_work_order", + WORK_ORDER_FILTER=builder.get_conditions_sql(), + ) + + df = read_sql_df(sql, builder.params) data = _df_to_records(df) # Group by work order @@ -455,10 +381,10 @@ def _get_workcenters_for_groups(groups: List[str]) -> List[str]: return get_workcenters_for_groups(groups) -def get_lot_history( - container_id: str, - workcenter_groups: Optional[List[str]] = None -) -> Dict[str, Any]: +def get_lot_history( + container_id: str, + workcenter_groups: Optional[List[str]] = None +) -> Dict[str, Any]: """Get production history for a LOT. Args: @@ -471,28 +397,27 @@ def get_lot_history( if not container_id: return {'error': '請指定 CONTAINERID'} - try: - sql = SQLLoader.load("query_tool/lot_history") - params = {'container_id': container_id} - - # Add workcenter filter if groups specified - workcenter_filter = "" - if workcenter_groups: - workcenters = _get_workcenters_for_groups(workcenter_groups) - if workcenters: - workcenter_filter = f"AND {_build_in_filter(workcenters, 'h.WORKCENTERNAME')}" - logger.debug( - f"Filtering by {len(workcenter_groups)} groups " - f"({len(workcenters)} workcenters)" - ) - - # Replace placeholder in SQL - sql = sql.replace("{{ WORKCENTER_FILTER }}", workcenter_filter) - - df = read_sql_df(sql, params) - data = _df_to_records(df) - - logger.debug(f"LOT history: {len(data)} records for {container_id}") + try: + events_by_cid = EventFetcher.fetch_events([container_id], "history") + rows = list(events_by_cid.get(container_id, [])) + + if workcenter_groups: + workcenters = _get_workcenters_for_groups(workcenter_groups) + if workcenters: + workcenter_set = set(workcenters) + rows = [ + row + for row in rows + if row.get('WORKCENTERNAME') in workcenter_set + ] + logger.debug( + f"Filtering by {len(workcenter_groups)} groups " + f"({len(workcenters)} workcenters)" + ) + + data = _df_to_records(pd.DataFrame(rows)) + + logger.debug(f"LOT history: {len(data)} records for {container_id}") return { 'data': data, @@ -563,7 +488,7 @@ def get_adjacent_lots( # LOT Association Functions # ============================================================ -def get_lot_materials(container_id: str) -> Dict[str, Any]: +def get_lot_materials(container_id: str) -> Dict[str, Any]: """Get material consumption records for a LOT. Args: @@ -575,12 +500,9 @@ def get_lot_materials(container_id: str) -> Dict[str, Any]: if not container_id: return {'error': '請指定 CONTAINERID'} - try: - sql = SQLLoader.load("query_tool/lot_materials") - params = {'container_id': container_id} - - df = read_sql_df(sql, params) - data = _df_to_records(df) + try: + events_by_cid = EventFetcher.fetch_events([container_id], "materials") + data = _df_to_records(pd.DataFrame(events_by_cid.get(container_id, []))) logger.debug(f"LOT materials: {len(data)} records for {container_id}") @@ -595,7 +517,7 @@ def get_lot_materials(container_id: str) -> Dict[str, Any]: return {'error': f'查詢失敗: {str(exc)}'} -def get_lot_rejects(container_id: str) -> Dict[str, Any]: +def get_lot_rejects(container_id: str) -> Dict[str, Any]: """Get reject (defect) records for a LOT. Args: @@ -607,12 +529,9 @@ def get_lot_rejects(container_id: str) -> Dict[str, Any]: if not container_id: return {'error': '請指定 CONTAINERID'} - try: - sql = SQLLoader.load("query_tool/lot_rejects") - params = {'container_id': container_id} - - df = read_sql_df(sql, params) - data = _df_to_records(df) + try: + events_by_cid = EventFetcher.fetch_events([container_id], "rejects") + data = _df_to_records(pd.DataFrame(events_by_cid.get(container_id, []))) logger.debug(f"LOT rejects: {len(data)} records for {container_id}") @@ -627,7 +546,7 @@ def get_lot_rejects(container_id: str) -> Dict[str, Any]: return {'error': f'查詢失敗: {str(exc)}'} -def get_lot_holds(container_id: str) -> Dict[str, Any]: +def get_lot_holds(container_id: str) -> Dict[str, Any]: """Get HOLD/RELEASE records for a LOT. Args: @@ -639,12 +558,9 @@ def get_lot_holds(container_id: str) -> Dict[str, Any]: if not container_id: return {'error': '請指定 CONTAINERID'} - try: - sql = SQLLoader.load("query_tool/lot_holds") - params = {'container_id': container_id} - - df = read_sql_df(sql, params) - data = _df_to_records(df) + try: + events_by_cid = EventFetcher.fetch_events([container_id], "holds") + data = _df_to_records(pd.DataFrame(events_by_cid.get(container_id, []))) logger.debug(f"LOT holds: {len(data)} records for {container_id}") @@ -659,10 +575,11 @@ def get_lot_holds(container_id: str) -> Dict[str, Any]: return {'error': f'查詢失敗: {str(exc)}'} -def get_lot_split_merge_history( - work_order: str, - current_container_id: str = None -) -> Dict[str, Any]: +def get_lot_split_merge_history( + work_order: str, + current_container_id: str = None, + full_history: bool = False, +) -> Dict[str, Any]: """Get complete split/merge history for a work order (完整拆併批歷史). Queries DW_MES_HM_LOTMOVEOUT for SplitLot and CombineLot operations @@ -679,9 +596,11 @@ def get_lot_split_merge_history( - A00-001-01: Split at production station (製程站點拆分) - A00-001-01C: Split at TMTT (TMTT 拆分) - Args: - work_order: MFGORDERNAME value (e.g., GA25120713) - current_container_id: Current LOT's CONTAINERID for highlighting + Args: + work_order: MFGORDERNAME value (e.g., GA25120713) + current_container_id: Current LOT's CONTAINERID for highlighting + full_history: If True, query complete history using slow connection. + If False (default), query only last 6 months with row limit. Returns: Dict with 'data' (split/merge history records) and 'total', or 'error'. @@ -690,16 +609,29 @@ def get_lot_split_merge_history( return {'error': '請指定工單號', 'data': [], 'total': 0} try: - sql = SQLLoader.load("query_tool/lot_split_merge_history") - params = {'work_order': work_order} - - logger.info(f"Starting split/merge history query for MFGORDERNAME={work_order}") - - # Use slow query connection with 120s timeout - # Note: DW_MES_HM_LOTMOVEOUT has 48M rows, no index on CONTAINERID/FROMCONTAINERID - # Query by MFGORDERNAME is faster but still needs extra time - df = read_sql_df_slow(sql, params, timeout_seconds=120) - data = _df_to_records(df) + builder = QueryBuilder() + builder.add_in_condition("MFGORDERNAME", [work_order]) + fast_time_window = "AND h.TXNDATE >= ADD_MONTHS(SYSDATE, -6)" + fast_row_limit = "FETCH FIRST 500 ROWS ONLY" + sql = SQLLoader.load_with_params( + "query_tool/lot_split_merge_history", + WORK_ORDER_FILTER=builder.get_conditions_sql(), + TIME_WINDOW="" if full_history else fast_time_window, + ROW_LIMIT="" if full_history else fast_row_limit, + ) + params = builder.params + + mode = "full" if full_history else "fast" + logger.info( + f"Starting split/merge history query for MFGORDERNAME={work_order} mode={mode}" + ) + + if full_history: + # Full mode uses dedicated slow query timeout path. + df = read_sql_df_slow(sql, params, timeout_seconds=120) + else: + df = read_sql_df(sql, params) + data = _df_to_records(df) # Process records for display processed = [] @@ -734,11 +666,12 @@ def get_lot_split_merge_history( logger.info(f"Split/merge history completed: {len(processed)} records for MFGORDERNAME={work_order}") - return { - 'data': processed, - 'total': len(processed), - 'work_order': work_order, - } + return { + 'data': processed, + 'total': len(processed), + 'work_order': work_order, + 'mode': mode, + } except Exception as exc: error_str = str(exc) @@ -784,10 +717,11 @@ def _get_mfg_order_for_lot(container_id: str) -> Optional[str]: return None -def get_lot_splits( - container_id: str, - include_production_history: bool = True # Uses dedicated slow query connection with 120s timeout -) -> Dict[str, Any]: +def get_lot_splits( + container_id: str, + include_production_history: bool = True, + full_history: bool = False, +) -> Dict[str, Any]: """Get combined split/merge data for a LOT (拆併批紀錄). Data sources: @@ -798,9 +732,10 @@ def get_lot_splits( Production history now queries by MFGORDERNAME (indexed) instead of CONTAINERID for much better performance (~1 second vs 40+ seconds). - Args: - container_id: CONTAINERID (16-char hex) - include_production_history: If True (default), include production history query. + Args: + container_id: CONTAINERID (16-char hex) + include_production_history: If True (default), include production history query. + full_history: If True, query split/merge history without fast-mode limits. Returns: Dict with 'production_history', 'serial_numbers', and totals. @@ -833,10 +768,11 @@ def get_lot_splits( if mfg_order: logger.info(f"Querying production history for MFGORDERNAME={mfg_order} (LOT: {container_id})") - history_result = get_lot_split_merge_history( - work_order=mfg_order, - current_container_id=container_id - ) + history_result = get_lot_split_merge_history( + work_order=mfg_order, + current_container_id=container_id, + full_history=full_history, + ) logger.info(f"[DEBUG] history_result keys: {list(history_result.keys())}") logger.info(f"[DEBUG] history_result total: {history_result.get('total', 0)}") @@ -1005,15 +941,17 @@ def get_equipment_status_hours( if validation_error: return {'error': validation_error} - try: - # Build filter on HISTORYID (which maps to RESOURCEID) - equipment_filter = _build_in_filter(equipment_ids, 'r.RESOURCEID') - - sql = SQLLoader.load("query_tool/equipment_status_hours") - sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter) - - params = {'start_date': start_date, 'end_date': end_date} - df = read_sql_df(sql, params) + try: + builder = QueryBuilder() + builder.add_in_condition("r.RESOURCEID", equipment_ids) + sql = SQLLoader.load_with_params( + "query_tool/equipment_status_hours", + EQUIPMENT_FILTER=builder.get_conditions_sql(), + ) + + params = {'start_date': start_date, 'end_date': end_date} + params.update(builder.params) + df = read_sql_df(sql, params) data = _df_to_records(df) # Calculate totals @@ -1075,14 +1013,17 @@ def get_equipment_lots( if validation_error: return {'error': validation_error} - try: - equipment_filter = _build_in_filter(equipment_ids, 'h.EQUIPMENTID') - - sql = SQLLoader.load("query_tool/equipment_lots") - sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter) - - params = {'start_date': start_date, 'end_date': end_date} - df = read_sql_df(sql, params) + try: + builder = QueryBuilder() + builder.add_in_condition("h.EQUIPMENTID", equipment_ids) + sql = SQLLoader.load_with_params( + "query_tool/equipment_lots", + EQUIPMENT_FILTER=builder.get_conditions_sql(), + ) + + params = {'start_date': start_date, 'end_date': end_date} + params.update(builder.params) + df = read_sql_df(sql, params) data = _df_to_records(df) logger.info(f"Equipment lots: {len(data)} records") @@ -1122,14 +1063,17 @@ def get_equipment_materials( if validation_error: return {'error': validation_error} - try: - equipment_filter = _build_in_filter(equipment_names, 'EQUIPMENTNAME') - - sql = SQLLoader.load("query_tool/equipment_materials") - sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter) - - params = {'start_date': start_date, 'end_date': end_date} - df = read_sql_df(sql, params) + try: + builder = QueryBuilder() + builder.add_in_condition("EQUIPMENTNAME", equipment_names) + sql = SQLLoader.load_with_params( + "query_tool/equipment_materials", + EQUIPMENT_FILTER=builder.get_conditions_sql(), + ) + + params = {'start_date': start_date, 'end_date': end_date} + params.update(builder.params) + df = read_sql_df(sql, params) data = _df_to_records(df) logger.info(f"Equipment materials: {len(data)} records") @@ -1169,14 +1113,17 @@ def get_equipment_rejects( if validation_error: return {'error': validation_error} - try: - equipment_filter = _build_in_filter(equipment_names, 'EQUIPMENTNAME') - - sql = SQLLoader.load("query_tool/equipment_rejects") - sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter) - - params = {'start_date': start_date, 'end_date': end_date} - df = read_sql_df(sql, params) + try: + builder = QueryBuilder() + builder.add_in_condition("EQUIPMENTNAME", equipment_names) + sql = SQLLoader.load_with_params( + "query_tool/equipment_rejects", + EQUIPMENT_FILTER=builder.get_conditions_sql(), + ) + + params = {'start_date': start_date, 'end_date': end_date} + params.update(builder.params) + df = read_sql_df(sql, params) data = _df_to_records(df) logger.info(f"Equipment rejects: {len(data)} records") @@ -1218,14 +1165,17 @@ def get_equipment_jobs( if validation_error: return {'error': validation_error} - try: - equipment_filter = _build_in_filter(equipment_ids, 'RESOURCEID') - - sql = SQLLoader.load("query_tool/equipment_jobs") - sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter) - - params = {'start_date': start_date, 'end_date': end_date} - df = read_sql_df(sql, params) + try: + builder = QueryBuilder() + builder.add_in_condition("RESOURCEID", equipment_ids) + sql = SQLLoader.load_with_params( + "query_tool/equipment_jobs", + EQUIPMENT_FILTER=builder.get_conditions_sql(), + ) + + params = {'start_date': start_date, 'end_date': end_date} + params.update(builder.params) + df = read_sql_df(sql, params) data = _df_to_records(df) logger.info(f"Equipment jobs: {len(data)} records") diff --git a/src/mes_dashboard/sql/lineage/merge_sources.sql b/src/mes_dashboard/sql/lineage/merge_sources.sql new file mode 100644 index 0000000..5662303 --- /dev/null +++ b/src/mes_dashboard/sql/lineage/merge_sources.sql @@ -0,0 +1,13 @@ +-- Unified LineageEngine - Merge Sources +-- Find source lots merged into finished lots from DW_MES_PJ_COMBINEDASSYLOTS. +-- +-- Parameters: +-- FINISHED_NAME_FILTER - QueryBuilder-generated condition on ca.FINISHEDNAME +-- +SELECT + ca.CONTAINERID AS SOURCE_CID, + ca.CONTAINERNAME AS SOURCE_NAME, + ca.FINISHEDNAME, + ca.LOTID AS FINISHED_CID +FROM DWH.DW_MES_PJ_COMBINEDASSYLOTS ca +WHERE {{ FINISHED_NAME_FILTER }} diff --git a/src/mes_dashboard/sql/lineage/split_ancestors.sql b/src/mes_dashboard/sql/lineage/split_ancestors.sql new file mode 100644 index 0000000..4143b25 --- /dev/null +++ b/src/mes_dashboard/sql/lineage/split_ancestors.sql @@ -0,0 +1,23 @@ +-- Unified LineageEngine - Split Ancestors +-- Resolve split genealogy upward via DW_MES_CONTAINER.SPLITFROMID +-- +-- Parameters: +-- CID_FILTER - QueryBuilder-generated condition for START WITH +-- +-- Notes: +-- - CONNECT BY NOCYCLE prevents infinite loops on cyclic data. +-- - LEVEL <= 20 matches previous BFS guard. +-- +-- Recursive WITH fallback (Oracle recursive subquery factoring): +-- If CONNECT BY execution plan regresses, replace this file's content with +-- sql/lineage/split_ancestors_recursive.sql (kept as reference). +-- +SELECT + c.CONTAINERID, + c.SPLITFROMID, + c.CONTAINERNAME, + LEVEL AS SPLIT_DEPTH +FROM DWH.DW_MES_CONTAINER c +START WITH {{ CID_FILTER }} +CONNECT BY NOCYCLE PRIOR c.SPLITFROMID = c.CONTAINERID + AND LEVEL <= 20 diff --git a/src/mes_dashboard/sql/mid_section_defect/genealogy_records.sql b/src/mes_dashboard/sql/mid_section_defect/genealogy_records.sql index bd878f2..99ec0a0 100644 --- a/src/mes_dashboard/sql/mid_section_defect/genealogy_records.sql +++ b/src/mes_dashboard/sql/mid_section_defect/genealogy_records.sql @@ -1,5 +1,6 @@ --- Mid-Section Defect Traceability - LOT Genealogy Records (Query 2) --- Batch query for split/merge records related to work orders +-- DEPRECATED: replaced by sql/lineage/split_ancestors.sql +-- Mid-Section Defect Traceability - LOT Genealogy Records (Query 2) +-- Batch query for split/merge records related to work orders -- -- Parameters: -- MFG_ORDER_FILTER - Dynamic IN clause for MFGORDERNAME (built by QueryBuilder) diff --git a/src/mes_dashboard/sql/mid_section_defect/split_chain.sql b/src/mes_dashboard/sql/mid_section_defect/split_chain.sql index 6aecb20..c1a676a 100644 --- a/src/mes_dashboard/sql/mid_section_defect/split_chain.sql +++ b/src/mes_dashboard/sql/mid_section_defect/split_chain.sql @@ -1,5 +1,6 @@ --- Mid-Section Defect Traceability - Split Chain (Query 2a) --- Resolve split ancestors via DW_MES_CONTAINER.SPLITFROMID +-- DEPRECATED: replaced by sql/lineage/split_ancestors.sql +-- Mid-Section Defect Traceability - Split Chain (Query 2a) +-- Resolve split ancestors via DW_MES_CONTAINER.SPLITFROMID -- -- Parameters: -- Dynamically built IN clause for CONTAINERIDs diff --git a/src/mes_dashboard/sql/query_tool/lot_resolve_id.sql b/src/mes_dashboard/sql/query_tool/lot_resolve_id.sql index 8489320..79f80ee 100644 --- a/src/mes_dashboard/sql/query_tool/lot_resolve_id.sql +++ b/src/mes_dashboard/sql/query_tool/lot_resolve_id.sql @@ -1,11 +1,11 @@ --- LOT ID to CONTAINERID Resolution --- Converts user-input LOT ID (CONTAINERNAME) to internal CONTAINERID --- --- Parameters: --- :container_names - List of CONTAINERNAME values (bind variable list) --- --- Note: CONTAINERID is 16-char hex (e.g., '48810380001cba48') --- CONTAINERNAME is user-visible LOT ID (e.g., 'GA23100020-A00-011') +-- LOT ID to CONTAINERID Resolution +-- Converts user-input LOT ID (CONTAINERNAME) to internal CONTAINERID +-- +-- Parameters: +-- CONTAINER_FILTER - QueryBuilder filter on CONTAINERNAME +-- +-- Note: CONTAINERID is 16-char hex (e.g., '48810380001cba48') +-- CONTAINERNAME is user-visible LOT ID (e.g., 'GA23100020-A00-011') SELECT CONTAINERID, @@ -13,5 +13,5 @@ SELECT MFGORDERNAME, SPECNAME, QTY -FROM DWH.DW_MES_CONTAINER -WHERE CONTAINERNAME IN ({{ CONTAINER_NAMES }}) +FROM DWH.DW_MES_CONTAINER +WHERE {{ CONTAINER_FILTER }} diff --git a/src/mes_dashboard/sql/query_tool/lot_resolve_serial.sql b/src/mes_dashboard/sql/query_tool/lot_resolve_serial.sql new file mode 100644 index 0000000..211ec0a --- /dev/null +++ b/src/mes_dashboard/sql/query_tool/lot_resolve_serial.sql @@ -0,0 +1,14 @@ +-- Serial Number (流水號) to CONTAINERID Resolution +-- Converts finished product serial numbers to CONTAINERID list. +-- +-- Parameters: +-- SERIAL_FILTER - QueryBuilder filter on p.FINISHEDNAME +-- +SELECT DISTINCT + p.CONTAINERID, + p.FINISHEDNAME, + c.CONTAINERNAME, + c.SPECNAME +FROM DWH.DW_MES_PJ_COMBINEDASSYLOTS p +LEFT JOIN DWH.DW_MES_CONTAINER c ON p.CONTAINERID = c.CONTAINERID +WHERE {{ SERIAL_FILTER }} diff --git a/src/mes_dashboard/sql/query_tool/lot_resolve_work_order.sql b/src/mes_dashboard/sql/query_tool/lot_resolve_work_order.sql new file mode 100644 index 0000000..a0f24b6 --- /dev/null +++ b/src/mes_dashboard/sql/query_tool/lot_resolve_work_order.sql @@ -0,0 +1,14 @@ +-- GA Work Order to CONTAINERID Resolution +-- Expands work orders to associated CONTAINERIDs. +-- +-- Parameters: +-- WORK_ORDER_FILTER - QueryBuilder filter on h.PJ_WORKORDER +-- +SELECT DISTINCT + h.CONTAINERID, + h.PJ_WORKORDER, + c.CONTAINERNAME, + c.SPECNAME +FROM DWH.DW_MES_LOTWIPHISTORY h +LEFT JOIN DWH.DW_MES_CONTAINER c ON h.CONTAINERID = c.CONTAINERID +WHERE {{ WORK_ORDER_FILTER }} diff --git a/src/mes_dashboard/sql/query_tool/lot_split_merge_history.sql b/src/mes_dashboard/sql/query_tool/lot_split_merge_history.sql index a6bbdb9..4e401dd 100644 --- a/src/mes_dashboard/sql/query_tool/lot_split_merge_history.sql +++ b/src/mes_dashboard/sql/query_tool/lot_split_merge_history.sql @@ -1,13 +1,18 @@ --- LOT Split/Merge History Query (拆併批歷史紀錄) --- Query by CONTAINERID list from same work order --- Check both TARGET (CONTAINERID) and SOURCE (FROMCONTAINERID) to find all related records - -WITH work_order_lots AS ( - SELECT CONTAINERID - FROM DWH.DW_MES_CONTAINER - WHERE MFGORDERNAME = :work_order -) -SELECT +-- LOT Split/Merge History Query (拆併批歷史紀錄) +-- Query by CONTAINERID list from same work order +-- Check both TARGET (CONTAINERID) and SOURCE (FROMCONTAINERID) to find all related records +-- +-- Parameters: +-- WORK_ORDER_FILTER - QueryBuilder filter on MFGORDERNAME +-- TIME_WINDOW - Optional time-window filter (default fast mode: 6 months) +-- ROW_LIMIT - Optional row limit (default fast mode: 500) + +WITH work_order_lots AS ( + SELECT CONTAINERID + FROM DWH.DW_MES_CONTAINER + WHERE {{ WORK_ORDER_FILTER }} +) +SELECT h.HISTORYMAINLINEID, h.CDONAME AS OPERATION_TYPE, h.CONTAINERID AS TARGET_CONTAINERID, @@ -17,10 +22,11 @@ SELECT h.QTY AS TARGET_QTY, h.TXNDATE FROM DWH.DW_MES_HM_LOTMOVEOUT h -WHERE ( - h.CONTAINERID IN (SELECT CONTAINERID FROM work_order_lots) - OR h.FROMCONTAINERID IN (SELECT CONTAINERID FROM work_order_lots) -) - AND h.FROMCONTAINERID IS NOT NULL -ORDER BY h.TXNDATE -FETCH FIRST 100 ROWS ONLY +WHERE ( + h.CONTAINERID IN (SELECT CONTAINERID FROM work_order_lots) + OR h.FROMCONTAINERID IN (SELECT CONTAINERID FROM work_order_lots) +) + AND h.FROMCONTAINERID IS NOT NULL + {{ TIME_WINDOW }} +ORDER BY h.TXNDATE +{{ ROW_LIMIT }} diff --git a/src/mes_dashboard/static/js/query-tool.js b/src/mes_dashboard/static/js/query-tool.js deleted file mode 100644 index f0afd14..0000000 --- a/src/mes_dashboard/static/js/query-tool.js +++ /dev/null @@ -1,3056 +0,0 @@ -/** - * Query Tool JavaScript - * - * Handles batch tracing and equipment period query functionality. - */ - -// ============================================================ -// State Management -// ============================================================ - -const QueryToolState = { - // LOT query - queryType: 'lot_id', - resolvedLots: [], - selectedLotIndex: 0, - lotHistories: {}, // container_id -> history data - lotAssociations: {}, // container_id -> { materials, rejects, holds, jobs } - - // Timeline - timelineSelectedLots: new Set(), // Set of indices for timeline display - currentLotIndex: 0, // For association highlight - - // Workcenter group filter - workcenterGroups: [], // All available groups [{name, sequence}] - selectedWorkcenterGroups: new Set(), // Selected group names for filtering - - // Equipment query - allEquipments: [], - selectedEquipments: new Set(), - equipmentResults: null, -}; - -// Expose for debugging -window.QueryToolState = QueryToolState; - -// ============================================================ -// State Cleanup (Memory Management) -// ============================================================ - -/** - * Clear all query state to free memory before new query or page unload. - * This prevents browser memory issues with large datasets. - */ -function clearQueryState() { - // Clear LOT query state - QueryToolState.resolvedLots = []; - QueryToolState.selectedLotIndex = 0; - QueryToolState.lotHistories = {}; - QueryToolState.lotAssociations = {}; - QueryToolState.timelineSelectedLots = new Set(); - QueryToolState.currentLotIndex = 0; - - // Clear workcenter group selection (keep workcenterGroups as it's reused) - QueryToolState.selectedWorkcenterGroups = new Set(); - - // Hide selection bar (contains LOT selector and workcenter filter) - const selectionBar = document.getElementById('selectionBar'); - if (selectionBar) selectionBar.style.display = 'none'; - - // Clear equipment query state - QueryToolState.equipmentResults = null; - // Note: Keep allEquipments and selectedEquipments as they are reused - - // Clear global timeline data (can be large) - if (window._timelineData) { - window._timelineData.lotsData = []; - window._timelineData.stationColors = {}; - window._timelineData.allStations = []; - window._timelineData.selectedStations = new Set(); - window._timelineData = null; - } - - // Close any open popups - closeTimelinePopup(); - - // Clear DOM content - const lotResultsContent = document.getElementById('lotResultsContent'); - if (lotResultsContent) { - lotResultsContent.innerHTML = ''; - lotResultsContent.style.display = 'none'; - } - - // Reset empty state visibility - const lotEmptyState = document.getElementById('lotEmptyState'); - if (lotEmptyState) { - lotEmptyState.style.display = 'block'; - } - - // Hide LOT info bar - const lotInfoBar = document.getElementById('lotInfoBar'); - if (lotInfoBar) lotInfoBar.style.display = 'none'; - - console.log('[QueryTool] State cleared for memory management'); -} - -// Clear state before page unload to help garbage collection -window.addEventListener('beforeunload', () => { - clearQueryState(); -}); - -// Expose for manual cleanup if needed -window.clearQueryState = clearQueryState; - -// ============================================================ -// Initialization -// ============================================================ - -document.addEventListener('DOMContentLoaded', () => { - loadEquipments(); - loadWorkcenterGroups(); // Load workcenter groups for filtering - setLast30Days(); - - // Close dropdowns when clicking outside - document.addEventListener('click', (e) => { - // Equipment dropdown - const eqDropdown = document.getElementById('equipmentDropdown'); - const eqSelector = document.querySelector('.equipment-selector'); - if (eqSelector && !eqSelector.contains(e.target)) { - eqDropdown.classList.remove('show'); - } - - // LOT selector dropdown - const lotDropdown = document.getElementById('lotSelectorDropdown'); - const lotSelector = document.getElementById('lotSelectorContainer'); - if (lotSelector && !lotSelector.contains(e.target)) { - lotDropdown.classList.remove('show'); - } - - // Workcenter group dropdown - const wcDropdown = document.getElementById('wcGroupDropdown'); - const wcSelector = document.getElementById('workcenterGroupSelectorContainer'); - if (wcSelector && !wcSelector.contains(e.target)) { - if (wcDropdown) wcDropdown.classList.remove('show'); - } - }); - - // Handle Enter key in search input - const searchInput = document.getElementById('lotInputField'); - if (searchInput) { - searchInput.addEventListener('keypress', (e) => { - if (e.key === 'Enter') { - executeLotQuery(); - } - }); - } -}); - -// ============================================================ -// Query Mode Switching (Batch vs Equipment) -// ============================================================ - -function switchQueryMode(mode) { - // Update tabs - document.querySelectorAll('.query-mode-tab').forEach(tab => { - tab.classList.toggle('active', tab.dataset.mode === mode); - }); - - // Show/hide filter bars - document.getElementById('batchFilterBar').style.display = mode === 'batch' ? 'flex' : 'none'; - document.getElementById('equipmentFilterBar').style.display = mode === 'equipment' ? 'flex' : 'none'; - - // Show/hide results panels - document.getElementById('batchResultsPanel').style.display = mode === 'batch' ? 'block' : 'none'; - document.getElementById('equipmentResultsPanel').style.display = mode === 'equipment' ? 'block' : 'none'; - - // Hide LOT info bar when switching to equipment mode - if (mode === 'equipment') { - document.getElementById('lotInfoBar').style.display = 'none'; - } -} - -// ============================================================ -// Query Type Selection -// ============================================================ - -function setQueryType(type) { - QueryToolState.queryType = type; - - // Update select element if called programmatically - const select = document.getElementById('queryTypeSelect'); - if (select && select.value !== type) { - select.value = type; - } - - // Update input placeholder based on type - const placeholders = { - 'lot_id': '輸入 LOT ID（多筆以逗號分隔）', - 'serial_number': '輸入流水號（多筆以逗號分隔）', - 'work_order': '輸入 GA工單（多筆以逗號分隔）', - }; - - const inputField = document.getElementById('lotInputField'); - if (inputField) { - inputField.placeholder = placeholders[type] || placeholders['lot_id']; - } -} - -// ============================================================ -// LOT Query Functions -// ============================================================ - -function parseInputValues(text) { - // Parse input: split by newlines and commas, trim whitespace, filter empty - return text - .split(/[\n,]/) - .map(s => s.trim()) - .filter(s => s.length > 0); -} - -async function executeLotQuery() { - const input = document.getElementById('lotInputField').value; - const values = parseInputValues(input); - - if (values.length === 0) { - Toast.error('請輸入查詢條件'); - return; - } - - // Validate limits - const limits = { 'lot_id': 50, 'serial_number': 50, 'work_order': 10 }; - const limit = limits[QueryToolState.queryType]; - if (values.length > limit) { - Toast.error(`輸入數量超過上限 (${limit} 筆)`); - return; - } - - // Clear previous query state to free memory - clearQueryState(); - - // Show loading - document.getElementById('lotEmptyState').style.display = 'none'; - document.getElementById('lotResultsContent').style.display = 'block'; - document.getElementById('lotResultsContent').innerHTML = ` -
- -
解析中... -
- `; - - // Hide LOT info bar and selection bar during loading - document.getElementById('lotInfoBar').style.display = 'none'; - const selectionBar = document.getElementById('selectionBar'); - if (selectionBar) selectionBar.style.display = 'none'; - - document.getElementById('lotQueryBtn').disabled = true; - - try { - // Step 1: Resolve to CONTAINERID - const resolveResult = await MesApi.post('/api/query-tool/resolve', { - input_type: QueryToolState.queryType, - values: values - }); - - if (resolveResult.error) { - document.getElementById('lotResultsContent').innerHTML = `
${resolveResult.error}
`; - return; - } - - if (!resolveResult.data || resolveResult.data.length === 0) { - document.getElementById('lotResultsContent').innerHTML = ` -
-

查無符合的批次資料

- ${resolveResult.not_found && resolveResult.not_found.length > 0 - ? `

未找到: ${resolveResult.not_found.join(', ')}

` - : ''} -
- `; - return; - } - - QueryToolState.resolvedLots = resolveResult.data; - QueryToolState.selectedLotIndex = 0; - QueryToolState.lotHistories = {}; - QueryToolState.lotAssociations = {}; - - // Initialize with empty selection - user must confirm - QueryToolState.timelineSelectedLots = new Set(); - - // Clear workcenter group selection for new query - QueryToolState.selectedWorkcenterGroups = new Set(); - - // Hide LOT info bar initially - document.getElementById('lotInfoBar').style.display = 'none'; - - // Show workcenter group selector for filtering - showWorkcenterGroupSelector(); - - if (resolveResult.data.length === 1) { - // Single result - auto-select and show directly - QueryToolState.timelineSelectedLots.add(0); - // Hide LOT selector (not needed for single result), but show workcenter filter - const lotSelector = document.getElementById('lotSelectorContainer'); - if (lotSelector) lotSelector.style.display = 'none'; - // Update hint for single LOT - const hint = document.getElementById('selectionHint'); - if (hint) hint.innerHTML = '選擇站點後點擊「套用篩選」重新載入'; - // Load and show the single lot's data - confirmLotSelection(); - } else { - // Multiple results - show selector for user to choose - const lotSelector = document.getElementById('lotSelectorContainer'); - if (lotSelector) lotSelector.style.display = 'block'; - showLotSelector(resolveResult.data); - // Render empty state - renderLotResults(resolveResult); - // Auto-open the dropdown - document.getElementById('lotSelectorDropdown').classList.add('show'); - } - - } catch (error) { - document.getElementById('lotResultsContent').innerHTML = `
查詢失敗: ${error.message}
`; - } finally { - document.getElementById('lotQueryBtn').disabled = false; - } -} - -// ============================================================ -// LOT Selector Dropdown -// ============================================================ - -function showLotSelector(lots) { - const container = document.getElementById('lotSelectorContainer'); - const dropdown = document.getElementById('lotSelectorDropdown'); - const display = document.getElementById('lotSelectorDisplay'); - const badge = document.getElementById('lotCountBadge'); - - container.style.display = 'block'; - display.textContent = '選擇批次...'; - badge.textContent = lots.length + ' 筆'; - - // Group lots by spec_name and sort within groups - const groupedLots = {}; - lots.forEach((lot, idx) => { - const spec = lot.spec_name || '未分類'; - if (!groupedLots[spec]) { - groupedLots[spec] = []; - } - groupedLots[spec].push({ ...lot, originalIndex: idx }); - }); - - // Sort specs alphabetically, sort lots within each group by lot_id - const sortedSpecs = Object.keys(groupedLots).sort(); - sortedSpecs.forEach(spec => { - groupedLots[spec].sort((a, b) => { - const aId = a.lot_id || a.input_value || ''; - const bId = b.lot_id || b.input_value || ''; - return aId.localeCompare(bId); - }); - }); - - // Populate dropdown with grouped structure and checkboxes for multi-select - let html = ` -
- - 已選 0 筆 -
- `; - - sortedSpecs.forEach(spec => { - html += `
${spec}
`; - - groupedLots[spec].forEach(lot => { - const idx = lot.originalIndex; - const isSelected = QueryToolState.timelineSelectedLots.has(idx); - - html += ` -
- -
-
${lot.lot_id || lot.input_value}
-
${lot.work_order || ''}
-
-
- `; - }); - }); - - // Add confirm button at the bottom - html += ` -
- -
- `; - - dropdown.innerHTML = html; - updateLotSelectorCount(); -} - -// Confirm selection and load data for all selected lots -async function confirmLotSelection() { - const selectedIndices = Array.from(QueryToolState.timelineSelectedLots); - - if (selectedIndices.length === 0) { - Toast.warning('請至少選擇一個批次'); - return; - } - - // Close dropdowns - document.getElementById('lotSelectorDropdown').classList.remove('show'); - const wcDropdown = document.getElementById('wcGroupDropdown'); - if (wcDropdown) wcDropdown.classList.remove('show'); - - // Build workcenter_groups parameter - const wcGroups = Array.from(QueryToolState.selectedWorkcenterGroups); - const wcGroupsParam = wcGroups.length > 0 ? wcGroups.join(',') : null; - - // Update display - const count = selectedIndices.length; - document.getElementById('lotSelectorDisplay').textContent = `已選 ${count} 個批次`; - - // Hide single lot info bar, show loading - document.getElementById('lotInfoBar').style.display = 'none'; - - const panel = document.getElementById('lotResultsContent'); - const filterInfo = wcGroupsParam ? ` (篩選: ${wcGroups.length} 個站點群組)` : ''; - panel.innerHTML = ` -
- -
載入所選批次資料...${filterInfo} -
- `; - - // Clear cached histories when filter changes - QueryToolState.lotHistories = {}; - - // Load history for all selected lots WITH workcenter filter - try { - await Promise.all(selectedIndices.map(async (idx) => { - const lot = QueryToolState.resolvedLots[idx]; - const params = { container_id: lot.container_id }; - if (wcGroupsParam) { - params.workcenter_groups = wcGroupsParam; - } - - const result = await MesApi.get('/api/query-tool/lot-history', { params }); - if (!result.error) { - QueryToolState.lotHistories[lot.container_id] = result.data || []; - } - })); - - // Render combined view - renderCombinedLotView(selectedIndices); - - } catch (error) { - panel.innerHTML = `
載入失敗: ${error.message}
`; - } -} - -function toggleLotInSelector(index, checked) { - if (checked) { - QueryToolState.timelineSelectedLots.add(index); - } else { - QueryToolState.timelineSelectedLots.delete(index); - } - - // Update visual style - const option = document.querySelector(`.lot-option[data-index="${index}"]`); - if (option) { - option.classList.toggle('selected', checked); - } - - updateLotSelectorCount(); - updateTimelineButton(); -} - -function toggleAllLotsInSelector(checked) { - const checkboxes = document.querySelectorAll('#lotSelectorDropdown input[type="checkbox"][data-lot-index]'); - checkboxes.forEach(cb => { - cb.checked = checked; - const idx = parseInt(cb.dataset.lotIndex); - if (checked) { - QueryToolState.timelineSelectedLots.add(idx); - } else { - QueryToolState.timelineSelectedLots.delete(idx); - } - }); - updateLotSelectorCount(); - updateTimelineButton(); -} - -function updateLotSelectorCount() { - const countEl = document.getElementById('lotSelectedCount'); - if (countEl) { - countEl.textContent = `已選 ${QueryToolState.timelineSelectedLots.size} 筆`; - } -} - -function toggleLotSelector() { - const dropdown = document.getElementById('lotSelectorDropdown'); - dropdown.classList.toggle('show'); -} - -function selectLotFromDropdown(index) { - // Toggle checkbox selection - const checkbox = document.querySelector(`#lotSelectorDropdown input[data-lot-index="${index}"]`); - if (checkbox) { - checkbox.checked = !checkbox.checked; - toggleLotInSelector(index, checkbox.checked); - } -} - -function updateLotInfoBar(index) { - const lot = QueryToolState.resolvedLots[index]; - if (!lot) return; - - const infoBar = document.getElementById('lotInfoBar'); - infoBar.style.display = 'flex'; - - document.getElementById('infoLotId').textContent = lot.lot_id || lot.input_value || '-'; - document.getElementById('infoSpec').textContent = lot.spec_name || '-'; - document.getElementById('infoWorkOrder').textContent = lot.work_order || '-'; - - // Step count will be updated after history loads - document.getElementById('infoStepCount').textContent = '-'; -} - -function renderLotResults(resolveResult) { - const lots = QueryToolState.resolvedLots; - const notFound = resolveResult.not_found || []; - - let html = ` -
-
- 共找到 ${lots.length} 個批次 - ${notFound.length > 0 ? `（${notFound.length} 個未找到）` : ''} -
-
- 請從上方選擇批次後點擊「確定選擇」 -
-
- `; - - html += `

請選擇要查看的批次

`; - - document.getElementById('lotResultsContent').innerHTML = html; -} - -// Render combined view for multiple selected lots -function renderCombinedLotView(selectedIndices) { - const lots = QueryToolState.resolvedLots; - const panel = document.getElementById('lotResultsContent'); - - // Collect all history data with LOT ID - const allHistory = []; - selectedIndices.forEach(idx => { - const lot = lots[idx]; - const history = QueryToolState.lotHistories[lot.container_id] || []; - history.forEach(step => { - allHistory.push({ - ...step, - LOT_ID: lot.lot_id || lot.input_value, - LOT_INDEX: idx - }); - }); - }); - - // Sort by track-in time - allHistory.sort((a, b) => { - const timeA = a.TRACKINTIMESTAMP ? new Date(a.TRACKINTIMESTAMP).getTime() : 0; - const timeB = b.TRACKINTIMESTAMP ? new Date(b.TRACKINTIMESTAMP).getTime() : 0; - return timeA - timeB; - }); - - let html = ` -
-
- 已選擇 ${selectedIndices.length} 個批次，共 ${allHistory.length} 筆生產紀錄 -
-
- -
-
- `; - - // Timeline section (auto-displayed) - html += ` -
-
-
- 生產時間線 - (${selectedIndices.length} 個批次) -
- -
-
-
- `; - - // Combined production history table with LOT ID column - html += ` -
生產歷程
-
- - - - - - - - - - - - - - - - - - - `; - - allHistory.forEach((step, idx) => { - html += ` - - - - - - - - - - - - - - - `; - }); - - html += `
LOT ID站點設備規格產品類型BOPWafer Lot上機時間下機時間入數出數操作
${step.LOT_ID}${step.WORKCENTERNAME || ''}${step.EQUIPMENTNAME || ''}${truncateText(step.SPECNAME, 15)}${step.PJ_TYPE || '-'}${step.PJ_BOP || '-'}${step.WAFER_LOT_ID || '-'}${formatDateTime(step.TRACKINTIMESTAMP)}${formatDateTime(step.TRACKOUTTIMESTAMP)}${step.TRACKINQTY || ''}${step.TRACKOUTQTY || ''} - -
`; - - // Association tabs for combined data - html += ` -
-
關聯資料
-
- - - - -
-
-
-
- `; - - panel.innerHTML = html; - - // Store selected indices for association queries - QueryToolState.currentSelectedIndices = selectedIndices; - - // Set timeline selected lots for showTimeline() to work - QueryToolState.timelineSelectedLots = new Set(selectedIndices); - - // Render timeline - renderTimeline(selectedIndices); - - // Load default association - loadCombinedAssociation('materials', document.querySelector('.assoc-tab.active')); -} - -// Load combined association data for all selected lots -async function loadCombinedAssociation(type, tabElement) { - // Update tab states - document.querySelectorAll('.assoc-tab').forEach(t => t.classList.remove('active')); - if (tabElement) tabElement.classList.add('active'); - - const content = document.getElementById('assocContent'); - - // Show custom loading message for splits (slow query) - if (type === 'splits') { - content.innerHTML = ` -
- -
-
查詢生產拆併批紀錄中...
-
此查詢可能需要 30-60 秒，請耐心等候
-
-
`; - } else { - content.innerHTML = `
`; - } - - const selectedIndices = QueryToolState.currentSelectedIndices || []; - const lots = QueryToolState.resolvedLots; - - try { - // Special handling for 'splits' type - different data structure - if (type === 'splits') { - const combinedSplitsData = { - production_history: [], - serial_numbers: [], - production_history_skipped: false, - production_history_skip_reason: '', - production_history_timeout: false, - production_history_timeout_message: '' - }; - - await Promise.all(selectedIndices.map(async (idx) => { - const lot = lots[idx]; - const cacheKey = `${lot.container_id}_${type}`; - - if (!QueryToolState.lotAssociations[cacheKey]) { - const result = await MesApi.get('/api/query-tool/lot-associations', { - params: { container_id: lot.container_id, type: type }, - timeout: 120000 // 2 minute timeout for slow queries - }); - // 'splits' returns {production_history, serial_numbers, ...} directly - // NOT wrapped in {data: ...} - QueryToolState.lotAssociations[cacheKey] = result || {}; - // Debug: log the API response for splits - console.log('[DEBUG] Splits API response for', lot.container_id, ':', result); - console.log('[DEBUG] production_history count:', (result?.production_history || []).length); - console.log('[DEBUG] serial_numbers count:', (result?.serial_numbers || []).length); - } - - const data = QueryToolState.lotAssociations[cacheKey]; - const lotId = lot.lot_id || lot.input_value; - - // Capture skip info from first response - if (data.production_history_skipped && !combinedSplitsData.production_history_skipped) { - combinedSplitsData.production_history_skipped = true; - combinedSplitsData.production_history_skip_reason = data.production_history_skip_reason || ''; - } - - // Capture timeout info - if (data.production_history_timeout && !combinedSplitsData.production_history_timeout) { - combinedSplitsData.production_history_timeout = true; - combinedSplitsData.production_history_timeout_message = data.production_history_timeout_message || '查詢逾時'; - } - - // Merge production_history with LOT_ID - (data.production_history || []).forEach(record => { - combinedSplitsData.production_history.push({ - ...record, - LOT_ID: lotId - }); - }); - - // Merge serial_numbers with LOT_ID - (data.serial_numbers || []).forEach(snGroup => { - // Check if this serial number already exists - const existingSn = combinedSplitsData.serial_numbers.find( - s => s.serial_number === snGroup.serial_number - ); - if (existingSn) { - // Merge lots into existing serial number - snGroup.lots.forEach(lot => { - if (!existingSn.lots.some(l => l.container_id === lot.container_id)) { - existingSn.lots.push(lot); - } - }); - existingSn.total_good_die = existingSn.lots.reduce( - (sum, l) => sum + (l.good_die_qty || 0), 0 - ); - } else { - combinedSplitsData.serial_numbers.push({ - ...snGroup, - LOT_ID: lotId - }); - } - }); - })); - - // Sort production history by date - combinedSplitsData.production_history.sort((a, b) => { - const dateA = a.txn_date ? new Date(a.txn_date).getTime() : 0; - const dateB = b.txn_date ? new Date(b.txn_date).getTime() : 0; - return dateA - dateB; - }); - - // Save to state for export functionality - QueryToolState.combinedSplitsData = combinedSplitsData; - - renderCombinedAssociation(type, combinedSplitsData); - return; - } - - // Standard handling for other types (materials, rejects, holds) - const allData = []; - - await Promise.all(selectedIndices.map(async (idx) => { - const lot = lots[idx]; - const cacheKey = `${lot.container_id}_${type}`; - - if (!QueryToolState.lotAssociations[cacheKey]) { - const result = await MesApi.get('/api/query-tool/lot-associations', { - params: { container_id: lot.container_id, type: type } - }); - QueryToolState.lotAssociations[cacheKey] = result.data || []; - } - - const data = QueryToolState.lotAssociations[cacheKey]; - data.forEach(row => { - allData.push({ - ...row, - LOT_ID: lot.lot_id || lot.input_value - }); - }); - })); - - renderCombinedAssociation(type, allData); - - } catch (error) { - content.innerHTML = `
載入失敗: ${error.message}
`; - } -} - -function renderCombinedAssociation(type, data) { - const content = document.getElementById('assocContent'); - - // Special handling for 'splits' type (different data structure) - if (type === 'splits') { - renderCombinedSplitsAssociation(data); - return; - } - - // Check for empty data (array types) - if (!data || data.length === 0) { - content.innerHTML = `

無${getAssocLabel(type)}資料

`; - return; - } - - // Define columns with LOT_ID first - const columnDefs = { - 'materials': ['LOT_ID', 'MATERIALPARTNAME', 'MATERIALLOTNAME', 'QTYCONSUMED', 'WORKCENTERNAME', 'EQUIPMENTNAME', 'TXNDATE'], - 'rejects': ['LOT_ID', 'REJECTCATEGORYNAME', 'LOSSREASONNAME', 'REJECTQTY', 'WORKCENTERNAME', 'EQUIPMENTNAME', 'TXNDATE'], - 'holds': ['LOT_ID', 'WORKCENTERNAME', 'HOLDREASONNAME', 'HOLDTXNDATE', 'HOLD_STATUS', 'HOLD_HOURS', 'HOLDEMP', 'RELEASETXNDATE'], - }; - - const colLabels = { - 'LOT_ID': 'LOT ID', - 'MATERIALPARTNAME': '物料名稱', - 'MATERIALLOTNAME': '物料批號', - 'QTYCONSUMED': '消耗數量', - 'WORKCENTERNAME': '站點', - 'EQUIPMENTNAME': '設備', - 'TXNDATE': '時間', - 'REJECTCATEGORYNAME': '不良分類', - 'LOSSREASONNAME': '損失原因', - 'REJECTQTY': '不良數量', - 'HOLDREASONNAME': 'HOLD 原因', - 'HOLDTXNDATE': 'HOLD 時間', - 'HOLD_STATUS': '狀態', - 'HOLD_HOURS': 'HOLD 時數', - 'HOLDEMP': 'HOLD 人員', - 'RELEASETXNDATE': 'RELEASE 時間', - }; - - const cols = columnDefs[type] || ['LOT_ID', ...Object.keys(data[0]).filter(k => k !== 'LOT_ID')]; - - let html = `
`; - cols.forEach(col => { - const isLotId = col === 'LOT_ID'; - const style = isLotId ? 'position: sticky; left: 0; background: #f8f9fa; z-index: 2;' : ''; - html += ``; - }); - html += ``; - - data.forEach(row => { - html += ``; - cols.forEach(col => { - let value = row[col]; - const isLotId = col === 'LOT_ID'; - const style = isLotId ? 'position: sticky; left: 0; background: white; z-index: 1; font-weight: 500; font-family: monospace; font-size: 12px;' : ''; - - if (col.includes('DATE') || col.includes('TIMESTAMP')) { - value = formatDateTime(value); - } - if (col === 'HOLD_STATUS') { - value = value === 'HOLD' - ? `HOLD 中` - : `已解除`; - } - html += ``; - }); - html += ``; - }); - - html += `
${colLabels[col] || col}
${value !== null && value !== undefined ? value : ''}
`; - content.innerHTML = html; -} - -function renderCombinedSplitsAssociation(data) { - const content = document.getElementById('assocContent'); - - // New data structure has: production_history, serial_numbers, skip/timeout info - const productionHistory = data.production_history || []; - const serialNumbers = data.serial_numbers || []; - const productionHistorySkipped = data.production_history_skipped || false; - const skipReason = data.production_history_skip_reason || ''; - const productionHistoryTimeout = data.production_history_timeout || false; - const timeoutMessage = data.production_history_timeout_message || ''; - const hasProductionHistory = productionHistory.length > 0; - const hasSerialNumbers = serialNumbers.length > 0; - - if (!hasProductionHistory && !hasSerialNumbers && !productionHistorySkipped && !productionHistoryTimeout) { - content.innerHTML = '

無拆併批紀錄

'; - return; - } - - let html = ''; - - // Show notice if production history query was skipped - if (productionHistorySkipped && skipReason) { - html += ` -
- 注意：${skipReason} -
- `; - } - - // Show warning if production history query timed out - if (productionHistoryTimeout) { - html += ` -
- ⚠ 查詢逾時：${timeoutMessage || '生產拆併批歷史查詢超時。此表格（DW_MES_HM_LOTMOVEOUT）目前無索引，查詢需較長時間。僅顯示 TMTT 成品流水號對應資料。'} -
- `; - } - - // 1. Production Split/Merge History (生產過程拆併批) - if (hasProductionHistory) { - html += ` -
-
-
- 生產過程拆併批紀錄 - (${productionHistory.length} 筆) -
- -
-
- - - - - - - - - - - - `; - - productionHistory.forEach(record => { - const opBadgeClass = record.operation_type === 'SplitLot' ? 'badge-info' : 'badge-warning'; - const isCurrentSource = record.is_current_lot_source; - const isCurrentTarget = record.is_current_lot_target; - const sourceStyle = isCurrentSource ? 'font-weight: 600; color: #4e54c8;' : ''; - const targetStyle = isCurrentTarget ? 'font-weight: 600; color: #4e54c8;' : ''; - - html += ` - - - - - - - - `; - }); - - html += `
操作來源批次目標批次數量時間
${record.operation_type_display}${record.source_lot || '-'}${record.target_lot || '-'}${record.target_qty || '-'}${formatDateTime(record.txn_date)}
`; - } - - // 2. TMTT Serial Number Mapping (成品流水號對應) - if (hasSerialNumbers) { - html += ` -
-
-
- 成品流水號對應 - (${serialNumbers.length} 個流水號) -
- -
- `; - - serialNumbers.forEach(snGroup => { - const sn = snGroup.serial_number || 'Unknown'; - const lots = snGroup.lots || []; - const totalDie = snGroup.total_good_die || 0; - - html += ` -
-
- 流水號: ${sn} - 總 Good Die: ${totalDie} -
-
- `; - - lots.forEach(lot => { - const isCurrentLot = lot.is_current; - const lotStyle = isCurrentLot - ? 'background: #e8f4e8; border: 1px solid #4caf50;' - : 'background: white; border: 1px solid #ddd;'; - - html += ` -
-
${lot.lot_id || '-'}
-
- ${lot.combine_ratio_pct} · ${lot.good_die_qty || 0} die -
-
- `; - }); - - html += `
`; - }); - - html += `
`; - } - - content.innerHTML = html; -} - -async function selectLot(index) { - QueryToolState.selectedLotIndex = index; - - // Update LOT selector dropdown display - const lot = QueryToolState.resolvedLots[index]; - const display = document.getElementById('lotSelectorDisplay'); - if (display && lot) { - display.textContent = lot.lot_id || lot.input_value; - } - - // Update dropdown active state - document.querySelectorAll('.lot-option').forEach((el, idx) => { - el.classList.toggle('active', idx === index); - }); - - // Update info bar - updateLotInfoBar(index); - - // Load history if not cached - loadLotHistory(index); -} - -async function loadLotHistory(index) { - const lot = QueryToolState.resolvedLots[index]; - const containerId = lot.container_id; - - const panel = document.getElementById('lotDetailPanel'); - panel.innerHTML = `

載入生產歷程...
`; - - // Check cache - if (QueryToolState.lotHistories[containerId]) { - renderLotDetail(index); - // Update step count in info bar - const stepCount = QueryToolState.lotHistories[containerId].length; - document.getElementById('infoStepCount').textContent = stepCount + ' 站'; - return; - } - - try { - const result = await MesApi.get('/api/query-tool/lot-history', { - params: { container_id: containerId } - }); - - if (result.error) { - panel.innerHTML = `
${result.error}
`; - return; - } - - QueryToolState.lotHistories[containerId] = result.data || []; - - // Update step count in info bar - const stepCount = (result.data || []).length; - document.getElementById('infoStepCount').textContent = stepCount + ' 站'; - - renderLotDetail(index); - - } catch (error) { - panel.innerHTML = `
載入失敗: ${error.message}
`; - } -} - -function renderLotDetail(index) { - const lot = QueryToolState.resolvedLots[index]; - const containerId = lot.container_id; - const lotId = lot.lot_id || lot.input_value; // LOT ID for display - const history = QueryToolState.lotHistories[containerId] || []; - - const panel = document.getElementById('lotDetailPanel'); - - let html = ''; - - if (history.length === 0) { - html += `

無生產歷程資料

`; - } else { - // Production history table (full width) - html += ` -
生產歷程
-
- - - - - - - - - - - - - - - - - - - `; - - history.forEach((step, idx) => { - html += ` - - - - - - - - - - - - - - - `; - }); - - html += `
#站點設備規格產品類型BOPWafer Lot上機時間下機時間入數出數操作
${idx + 1}${step.WORKCENTERNAME || ''}${step.EQUIPMENTNAME || ''}${truncateText(step.SPECNAME, 12)}${step.PJ_TYPE || '-'}${step.PJ_BOP || '-'}${step.WAFER_LOT_ID || '-'}${formatDateTime(step.TRACKINTIMESTAMP)}${formatDateTime(step.TRACKOUTTIMESTAMP)}${step.TRACKINQTY || ''}${step.TRACKOUTQTY || ''} - -
`; - } - - // Association tabs - html += ` -
-
關聯資料
-
- - - - -
-
-
-
- `; - - panel.innerHTML = html; - - // Load default association (materials) - loadAssociation(containerId, 'materials', document.querySelector('.assoc-tab.active')); -} - -async function loadAssociation(containerId, type, tabElement) { - // Update tab states - document.querySelectorAll('.assoc-tab').forEach(t => t.classList.remove('active')); - if (tabElement) tabElement.classList.add('active'); - - // Save current container ID for export functions - QueryToolState.currentContainerId = containerId; - - const content = document.getElementById('assocContent'); - - // Show custom loading message for splits (slow query) - if (type === 'splits') { - content.innerHTML = ` -
- -
-
查詢生產拆併批紀錄中...
-
此查詢可能需要 30-60 秒，請耐心等候
-
-
`; - } else { - content.innerHTML = `
`; - } - - // Check cache - const cacheKey = `${containerId}_${type}`; - if (QueryToolState.lotAssociations[cacheKey]) { - renderAssociation(type, QueryToolState.lotAssociations[cacheKey]); - return; - } - - try { - const result = await MesApi.get('/api/query-tool/lot-associations', { - params: { container_id: containerId, type: type }, - timeout: type === 'splits' ? 120000 : 60000 // 2 minutes for splits, 1 minute for others - }); - - if (result.error) { - content.innerHTML = `
${result.error}
`; - return; - } - - // 'splits' returns {production_history, serial_numbers, ...} directly - // Other types return {data: [...], ...} - if (type === 'splits') { - QueryToolState.lotAssociations[cacheKey] = result || {}; - renderAssociation(type, result || {}); - } else { - QueryToolState.lotAssociations[cacheKey] = result.data || []; - renderAssociation(type, result.data || []); - } - - } catch (error) { - content.innerHTML = `
載入失敗: ${error.message}
`; - } -} - -function renderAssociation(type, data) { - const content = document.getElementById('assocContent'); - - // Special handling for 'splits' type (object with production_history and serial_numbers) - if (type === 'splits') { - renderSplitsAssociation(data); - return; - } - - // Check empty data for array-based types - if (!data || data.length === 0) { - content.innerHTML = `

無${getAssocLabel(type)}資料

`; - return; - } - - let html = `
`; - - // Define columns based on type - const columns = { - 'materials': ['MATERIALPARTNAME', 'MATERIALLOTNAME', 'QTYCONSUMED', 'WORKCENTERNAME', 'EQUIPMENTNAME', 'TXNDATE'], - 'rejects': ['REJECTCATEGORYNAME', 'LOSSREASONNAME', 'REJECTQTY', 'WORKCENTERNAME', 'EQUIPMENTNAME', 'TXNDATE'], - 'holds': ['WORKCENTERNAME', 'HOLDREASONNAME', 'HOLDTXNDATE', 'HOLD_STATUS', 'HOLD_HOURS', 'HOLDEMP', 'RELEASETXNDATE'], - }; - - const colLabels = { - 'MATERIALPARTNAME': '物料名稱', - 'MATERIALLOTNAME': '物料批號', - 'QTYCONSUMED': '消耗數量', - 'WORKCENTERNAME': '站點', - 'EQUIPMENTNAME': '設備', - 'TXNDATE': '時間', - 'REJECTCATEGORYNAME': '不良分類', - 'LOSSREASONNAME': '損失原因', - 'REJECTQTY': '不良數量', - 'HOLDREASONNAME': 'HOLD 原因', - 'HOLDTXNDATE': 'HOLD 時間', - 'HOLD_STATUS': '狀態', - 'HOLD_HOURS': 'HOLD 時數', - 'HOLDEMP': 'HOLD 人員', - 'RELEASETXNDATE': 'RELEASE 時間', - }; - - const cols = columns[type] || Object.keys(data[0]); - cols.forEach(col => { - html += ``; - }); - html += ``; - - data.forEach(row => { - html += ``; - cols.forEach(col => { - let value = row[col]; - if (col.includes('DATE') || col.includes('TIMESTAMP')) { - value = formatDateTime(value); - } - if (col === 'HOLD_STATUS') { - value = value === 'HOLD' - ? `HOLD 中` - : `已解除`; - } - html += ``; - }); - html += ``; - }); - - html += `
${colLabels[col] || col}
${value !== null && value !== undefined ? value : ''}
`; - content.innerHTML = html; -} - -function renderSplitsAssociation(data) { - const content = document.getElementById('assocContent'); - - // Handle new format: {production_history: [], serial_numbers: [], skip/timeout info, ...} - const productionHistory = data.production_history || []; - const serialNumbers = data.serial_numbers || []; - const productionHistorySkipped = data.production_history_skipped || false; - const skipReason = data.production_history_skip_reason || ''; - const productionHistoryTimeout = data.production_history_timeout || false; - const timeoutMessage = data.production_history_timeout_message || ''; - const hasProductionHistory = productionHistory.length > 0; - const hasSerialNumbers = serialNumbers.length > 0; - - if (!hasProductionHistory && !hasSerialNumbers && !productionHistoryTimeout) { - let emptyHtml = ''; - if (productionHistorySkipped && skipReason) { - emptyHtml += ` -
- 注意：${skipReason} -
- `; - } - emptyHtml += '
無拆併批紀錄
'; - content.innerHTML = emptyHtml; - return; - } - - let html = ''; - - // Show notice if production history query was skipped - if (productionHistorySkipped && skipReason) { - html += ` -
- 注意：${skipReason} -
- `; - } - - // Show warning if production history query timed out - if (productionHistoryTimeout) { - html += ` -
- ⚠ 查詢逾時：${timeoutMessage || '生產拆併批歷史查詢超時。此表格（DW_MES_HM_LOTMOVEOUT）目前無索引，查詢需較長時間。僅顯示 TMTT 成品流水號對應資料。'} -
- `; - } - - // Production history section (if any) - if (hasProductionHistory) { - html += ` -
-
-
- 生產過程拆併批紀錄 (${productionHistory.length} 筆) -
- -
-
- - - - - - - - - - - - `; - - productionHistory.forEach(record => { - const opBadgeClass = record.operation_type === 'SplitLot' ? 'badge-info' : 'badge-warning'; - html += ` - - - - - - - - `; - }); - - html += `
操作來源批次目標批次數量時間
${record.operation_type_display}${record.source_lot || '-'}${record.target_lot || '-'}${record.target_qty || '-'}${formatDateTime(record.txn_date)}
`; - } - - // Serial numbers section - if (!hasSerialNumbers) { - if (hasProductionHistory) { - html += '
此 LOT 尚未產出成品流水號
'; - } - content.innerHTML = html; - return; - } - - html += ` -
-
- 此 LOT 參與產出 ${serialNumbers.length} 個成品流水號，以下顯示各成品的來源批次組成 -
- -
- `; - - serialNumbers.forEach((item, idx) => { - const lots = item.lots || []; - const totalGoodDie = item.total_good_die || 0; - const isCombined = lots.length > 1; - - html += ` -
-
- - 成品流水號: ${item.serial_number || '-'} - - - ${isCombined ? `${lots.length} 批合併` : '單批產出'} | - 良品總數: ${totalGoodDie.toLocaleString()} - -
-
- - - - - - - - - - - - - `; - - lots.forEach((lot, lotIdx) => { - const isCurrent = lot.is_current; - const rowStyle = isCurrent - ? 'background: #fff3cd; border-left: 4px solid #ffc107;' - : 'border-left: 4px solid transparent;'; - const ratioValue = lot.combine_ratio || 0; - const ratioBarWidth = Math.min(ratioValue * 100, 100); - - html += ` - - - - - - - - - `; - }); - - html += ` - -
序LOT ID工單貢獻比例良品數開始時間
${lotIdx + 1} -
${lot.lot_id || '-'}
- ${isCurrent ? '當前查詢批次' : ''} - 		${lot.work_order || '-'} -
-
-
-
- ${lot.combine_ratio_pct || '-'} -
- 		- ${lot.good_die_qty ? lot.good_die_qty.toLocaleString() : '-'} - - ${formatDateTime(lot.original_start_date)} -
-
-
- `; - }); - - content.innerHTML = html; -} - -function getAssocLabel(type) { - const labels = { - 'materials': '物料消耗', - 'rejects': '不良紀錄', - 'holds': 'HOLD 紀錄', - 'splits': '拆併批紀錄', - 'jobs': 'JOB 紀錄' - }; - return labels[type] || type; -} - -// ============================================================ -// Timeline Functions -// ============================================================ - -function toggleLotForTimeline(index, checked) { - // Delegate to the selector function - toggleLotInSelector(index, checked); -} - -function toggleAllLotsForTimeline(checked) { - // This function is now handled by toggleAllLotsInSelector - toggleAllLotsInSelector(checked); -} - -function updateTimelineButton() { - const btn = document.getElementById('timelineBtn'); - const count = QueryToolState.timelineSelectedLots.size; - if (btn) { - btn.disabled = count === 0; - btn.textContent = count > 0 ? `顯示時間線 (${count})` : '顯示時間線'; - } -} - -async function showTimeline() { - const selectedIndices = Array.from(QueryToolState.timelineSelectedLots); - if (selectedIndices.length === 0) { - Toast.warning('請先勾選要顯示時間線的批次'); - return; - } - - const container = document.getElementById('timelineContainer'); - const content = document.getElementById('timelineContent'); - const countSpan = document.getElementById('timelineCount'); - - container.style.display = 'block'; - countSpan.textContent = `(${selectedIndices.length} 個批次)`; - content.innerHTML = '

載入時間線資料...
'; - - // Scroll to timeline - container.scrollIntoView({ behavior: 'smooth', block: 'start' }); - - // Load history for all selected lots - const lotsToLoad = []; - for (const idx of selectedIndices) { - const lot = QueryToolState.resolvedLots[idx]; - if (!QueryToolState.lotHistories[lot.container_id]) { - lotsToLoad.push({ idx, lot }); - } - } - - // Load missing histories - try { - await Promise.all(lotsToLoad.map(async ({ idx, lot }) => { - const result = await MesApi.get('/api/query-tool/lot-history', { - params: { container_id: lot.container_id } - }); - if (!result.error) { - QueryToolState.lotHistories[lot.container_id] = result.data || []; - } - })); - - renderTimeline(selectedIndices); - } catch (error) { - content.innerHTML = `
載入失敗: ${error.message}
`; - } -} - -function hideTimeline() { - document.getElementById('timelineContainer').style.display = 'none'; -} - -function renderTimeline(selectedIndices) { - const content = document.getElementById('timelineContent'); - - // Collect all history data and find time bounds - const lotsData = []; - let minTime = Infinity; - let maxTime = -Infinity; - - // Station color map - const stationColors = {}; - const colorPalette = [ - '#4e79a7', '#f28e2b', '#e15759', '#76b7b2', '#59a14f', - '#edc948', '#b07aa1', '#ff9da7', '#9c755f', '#bab0ac', - '#86bcb6', '#8cd17d', '#499894', '#d37295', '#b6992d' - ]; - let colorIndex = 0; - - for (const idx of selectedIndices) { - const lot = QueryToolState.resolvedLots[idx]; - const history = QueryToolState.lotHistories[lot.container_id] || []; - - const steps = history.map((step, stepIdx) => { - // Parse track-in time with better handling - const trackInRaw = step.TRACKINTIMESTAMP; - let trackIn = 0; - if (trackInRaw) { - // Try parsing - handle both ISO and Oracle formats - const parsed = new Date(trackInRaw); - if (!isNaN(parsed.getTime())) { - trackIn = parsed.getTime(); - } - } - - // Check if TRACKOUTTIMESTAMP is a valid value (not null, not empty string, not "None") - const trackOutRaw = step.TRACKOUTTIMESTAMP; - const hasValidTrackOut = trackOutRaw && - trackOutRaw !== '' && - trackOutRaw !== 'None' && - trackOutRaw !== 'null' && - !isNaN(new Date(trackOutRaw).getTime()); - - // For ongoing steps (no track-out), use track-in + 1 hour as placeholder - // This prevents using Date.now() which can skew the timeline range - const trackOut = hasValidTrackOut - ? new Date(trackOutRaw).getTime() - : (trackIn > 0 ? trackIn + 3600000 : 0); // trackIn + 1 hour for ongoing - const isOngoing = !hasValidTrackOut; - - // Only process steps with valid trackIn (skip pre-scheduled steps) - if (trackIn > 0) { - if (trackIn < minTime) minTime = trackIn; - // For maxTime, use trackOut if valid, otherwise use trackIn - const effectiveMax = hasValidTrackOut ? trackOut : trackIn; - if (effectiveMax > maxTime) maxTime = effectiveMax; - } - - // Assign color to station - const station = step.WORKCENTERNAME || 'Unknown'; - if (!stationColors[station]) { - stationColors[station] = colorPalette[colorIndex % colorPalette.length]; - colorIndex++; - } - - return { - station, - equipment: step.EQUIPMENTNAME || '', - spec: step.SPECNAME || '', - trackIn, - trackOut, - color: stationColors[station], - isOngoing - }; - }); - - lotsData.push({ - lotId: lot.lot_id || lot.input_value, - containerId: lot.container_id, - steps - }); - } - - if (lotsData.length === 0 || minTime === Infinity) { - content.innerHTML = '

無生產歷程資料可顯示

'; - return; - } - - // Add padding to time range - const timeRange = maxTime - minTime; - const padding = timeRange * 0.02; - minTime -= padding; - maxTime += padding; - - // Store timeline data for filtering and popup - const allStations = Object.keys(stationColors); - window._timelineData = { - lotsData, - minTime, - maxTime, - stationColors, - allStations, - selectedStations: new Set(allStations), // All selected by default - pixelsPerHour: 50 - }; - - // Render compact preview with click to open popup - let html = ` -
-
站點篩選 (點擊切換顯示)
-
- ${allStations.map(station => ` - - ${station} - - `).join('')} -
-
- - -
-
- -
-
時間線預覽
- 點擊開啟完整視窗 -
- -
- ${renderTimelinePreview(lotsData, minTime, maxTime, stationColors)} -
- -
- 支援縮放和橫向捲動 -
- `; - - content.innerHTML = html; -} - -function renderTimelinePreview(lotsData, minTime, maxTime, stationColors) { - const MS_PER_HOUR = 3600000; - const PREVIEW_WIDTH = 600; // Fixed preview width - const timeRange = maxTime - minTime; - const selectedStations = window._timelineData?.selectedStations || new Set(Object.keys(stationColors)); - - let html = '
'; - - lotsData.forEach((lotData, lotIdx) => { - html += ` -
-
- ${lotData.lotId} -
-
- `; - - lotData.steps.forEach((step) => { - if (!step.trackIn || step.trackIn <= 0) return; - if (!selectedStations.has(step.station)) return; - - const left = ((step.trackIn - minTime) / timeRange) * 100; - const width = Math.max(((step.trackOut - step.trackIn) / timeRange) * 100, 0.5); - - html += ` -
- `; - }); - - html += ` -
-
- `; - }); - - html += '
'; - return html; -} - -function toggleStationFilter(station) { - const data = window._timelineData; - if (!data) return; - - if (data.selectedStations.has(station)) { - data.selectedStations.delete(station); - } else { - data.selectedStations.add(station); - } - - // Update UI - const item = document.querySelector(`.station-filter-item[data-station="${station}"]`); - if (item) { - item.classList.toggle('active', data.selectedStations.has(station)); - item.style.opacity = data.selectedStations.has(station) ? '1' : '0.4'; - } - - // Re-render preview - const preview = document.getElementById('timelinePreview'); - if (preview) { - preview.innerHTML = renderTimelinePreview(data.lotsData, data.minTime, data.maxTime, data.stationColors); - } -} - -function selectAllStations() { - const data = window._timelineData; - if (!data) return; - - data.selectedStations = new Set(data.allStations); - document.querySelectorAll('.station-filter-item').forEach(item => { - item.classList.add('active'); - item.style.opacity = '1'; - }); - - const preview = document.getElementById('timelinePreview'); - if (preview) { - preview.innerHTML = renderTimelinePreview(data.lotsData, data.minTime, data.maxTime, data.stationColors); - } -} - -function deselectAllStations() { - const data = window._timelineData; - if (!data) return; - - data.selectedStations = new Set(); - document.querySelectorAll('.station-filter-item').forEach(item => { - item.classList.remove('active'); - item.style.opacity = '0.4'; - }); - - const preview = document.getElementById('timelinePreview'); - if (preview) { - preview.innerHTML = renderTimelinePreview(data.lotsData, data.minTime, data.maxTime, data.stationColors); - } -} - -function openTimelinePopup() { - const data = window._timelineData; - if (!data) return; - - // Create popup overlay - const popup = document.createElement('div'); - popup.id = 'timelinePopup'; - popup.style.cssText = 'position: fixed; top: 0; left: 0; right: 0; bottom: 0; background: rgba(0,0,0,0.5); display: flex; align-items: center; justify-content: center; z-index: 10000;'; - - const popupContent = document.createElement('div'); - popupContent.style.cssText = 'background: white; border-radius: 12px; width: 95%; max-width: 1400px; max-height: 90vh; display: flex; flex-direction: column; box-shadow: 0 20px 60px rgba(0,0,0,0.3);'; - - popupContent.innerHTML = ` -
-
- 生產時間線 -
-
-
- - - ${data.pixelsPerHour}px/h -
- -
-
-
- `; - - popup.appendChild(popupContent); - document.body.appendChild(popup); - - // Close on backdrop click - popup.addEventListener('click', (e) => { - if (e.target === popup) closeTimelinePopup(); - }); - - // Close on Escape key - document.addEventListener('keydown', function escHandler(e) { - if (e.key === 'Escape') { - closeTimelinePopup(); - document.removeEventListener('keydown', escHandler); - } - }); - - // Render full timeline - renderFullTimeline(data.pixelsPerHour); -} - -function closeTimelinePopup() { - const popup = document.getElementById('timelinePopup'); - if (popup) { - // Clear the popup's inner HTML first to help garbage collection - const content = popup.querySelector('#popupTimelineContent'); - if (content) content.innerHTML = ''; - popup.remove(); - } -} - -function updateTimelineScale(value) { - const data = window._timelineData; - if (!data) return; - - data.pixelsPerHour = parseInt(value); - document.getElementById('scaleValue').textContent = value + 'px/h'; - - renderFullTimeline(data.pixelsPerHour); -} - -function renderFullTimeline(pixelsPerHour) { - const data = window._timelineData; - if (!data) return; - - const container = document.getElementById('popupTimelineContent'); - if (!container) return; - - const { lotsData, minTime, maxTime, stationColors, selectedStations } = data; - const MS_PER_HOUR = 3600000; - const totalHours = (maxTime - minTime) / MS_PER_HOUR; - const timelineWidth = Math.max(800, totalHours * pixelsPerHour); - - let html = ` -
-
- ${renderTimelineAxisFixed(minTime, maxTime, pixelsPerHour)} -
- `; - - lotsData.forEach((lotData) => { - html += ` -
-
- ${lotData.lotId} -
-
- `; - - lotData.steps.forEach((step, stepIdx) => { - if (!step.trackIn || step.trackIn <= 0) return; - if (!selectedStations.has(step.station)) return; - - const leftPx = ((step.trackIn - minTime) / MS_PER_HOUR) * pixelsPerHour; - const durationHours = (step.trackOut - step.trackIn) / MS_PER_HOUR; - const widthPx = Math.max(durationHours * pixelsPerHour, 40); - - const equipmentLabel = step.equipment || ''; - const durationStr = durationHours >= 1 - ? `${Math.floor(durationHours)}h ${Math.round((durationHours % 1) * 60)}m` - : `${Math.round(durationHours * 60)}m`; - const timeRangeStr = step.isOngoing ? '進行中' : formatDateTime(new Date(step.trackOut)); - const tooltipLines = [ - `${step.station} - ${equipmentLabel}`, - `${formatDateTime(new Date(step.trackIn))} ~ ${timeRangeStr}`, - `耗時: ${durationStr}`, - `規格: ${step.spec}` - ]; - - html += ` -
- ${equipmentLabel} - ${step.station} -
- `; - }); - - html += ` -
-
- `; - }); - - html += ` -
-
-
- `; - - container.innerHTML = html; -} - -function setTimelineScale(pixelsPerHour) { - const data = window._timelineData; - if (!data) return; - - // Update button states - ['scale25', 'scale50', 'scale100', 'scale200'].forEach(id => { - const btn = document.getElementById(id); - if (btn) { - btn.classList.toggle('active', id === `scale${pixelsPerHour}`); - } - }); - - const MS_PER_HOUR = 3600000; - const { minTime, maxTime } = data; - - // Recalculate timeline width - const totalHours = (maxTime - minTime) / MS_PER_HOUR; - const timelineWidth = Math.max(800, totalHours * pixelsPerHour); - - // Update container width - const inner = document.getElementById('timelineInner'); - if (inner) { - inner.style.width = timelineWidth + 'px'; - - // Update all bar widths (relative part) - const barContainers = inner.querySelectorAll('.timeline-bar').forEach(bar => { - const durationHours = parseFloat(bar.dataset.durationHours) || 0; - const leftHours = (parseFloat(bar.style.left) / (data.pixelsPerHour || 50)); - const newLeft = leftHours * pixelsPerHour; - const newWidth = Math.max(durationHours * pixelsPerHour, 30); - - bar.style.left = newLeft + 'px'; - bar.style.width = newWidth + 'px'; - }); - - // Update time axis - const axisContainer = inner.querySelector('.timeline-axis-container'); - if (axisContainer) { - axisContainer.outerHTML = renderTimelineAxisFixed(minTime, maxTime, pixelsPerHour); - } - - // Update lot row widths - inner.querySelectorAll('.timeline-bar').forEach(bar => { - const parent = bar.parentElement; - if (parent) { - parent.style.width = (timelineWidth - 180) + 'px'; - } - }); - } - - // Update stored pixelsPerHour - data.pixelsPerHour = pixelsPerHour; -} - -function renderTimelineAxisFixed(minTime, maxTime, pixelsPerHour) { - const MS_PER_HOUR = 3600000; - const totalHours = (maxTime - minTime) / MS_PER_HOUR; - const timelineWidth = totalHours * pixelsPerHour; - - // Generate tick marks - one per day or per 6 hours depending on scale - const tickIntervalHours = pixelsPerHour >= 100 ? 6 : (pixelsPerHour >= 50 ? 12 : 24); - const ticks = []; - - // Start from the first hour mark after minTime - const startDate = new Date(minTime); - startDate.setMinutes(0, 0, 0); - let tickTime = startDate.getTime(); - if (tickTime < minTime) tickTime += MS_PER_HOUR; - - // Align to tick interval - const tickHour = new Date(tickTime).getHours(); - const alignOffset = tickHour % tickIntervalHours; - if (alignOffset > 0) { - tickTime += (tickIntervalHours - alignOffset) * MS_PER_HOUR; - } - - while (tickTime <= maxTime) { - const date = new Date(tickTime); - const pos = ((tickTime - minTime) / MS_PER_HOUR) * pixelsPerHour; - const label = `${(date.getMonth() + 1).toString().padStart(2, '0')}/${date.getDate().toString().padStart(2, '0')} ${date.getHours().toString().padStart(2, '0')}:00`; - ticks.push({ pos, label, isDay: date.getHours() === 0 }); - tickTime += tickIntervalHours * MS_PER_HOUR; - } - - return ` -
- ${ticks.map(t => ` -
-
${t.label}
-
-
- `).join('')} -
- `; -} - -function showTimelineDetail(containerId, stepIndex) { - // Find the lot and show its detail - const lotIndex = QueryToolState.resolvedLots.findIndex(l => l.container_id === containerId); - if (lotIndex >= 0) { - selectLot(lotIndex); - // Scroll to the history row - setTimeout(() => { - const row = document.getElementById(`history-row-${stepIndex}`); - if (row) { - row.scrollIntoView({ behavior: 'smooth', block: 'center' }); - row.style.background = '#fff3cd'; - setTimeout(() => { row.style.background = ''; }, 2000); - } - }, 300); - } -} - -async function showAdjacentLots(equipmentId, equipmentName, targetTime) { - // Open modal or expand section to show adjacent lots - const modal = document.createElement('div'); - modal.className = 'modal-overlay'; - modal.innerHTML = ` -
-
-

前後批查詢

- -
-
-
- 設備: ${equipmentName} | 基準時間: ${formatDateTime(targetTime)} -
-
-
-
-
-
- `; - - // Add modal styles if not exists - if (!document.getElementById('modal-styles')) { - const style = document.createElement('style'); - style.id = 'modal-styles'; - style.textContent = ` - .modal-overlay { - position: fixed; - top: 0; - left: 0; - right: 0; - bottom: 0; - background: rgba(0,0,0,0.5); - display: flex; - align-items: center; - justify-content: center; - z-index: 10000; - } - .modal-content { - background: white; - border-radius: 8px; - box-shadow: 0 10px 30px rgba(0,0,0,0.3); - } - .modal-header { - display: flex; - justify-content: space-between; - align-items: center; - padding: 15px 20px; - border-bottom: 1px solid #e0e0e0; - } - .modal-header h3 { - margin: 0; - font-size: 18px; - } - .modal-close { - background: none; - border: none; - font-size: 24px; - cursor: pointer; - color: #666; - } - .modal-close:hover { - color: #333; - } - .modal-body { - padding: 20px; - } - `; - document.head.appendChild(style); - } - - document.body.appendChild(modal); - - // Load adjacent lots - try { - const result = await MesApi.get('/api/query-tool/adjacent-lots', { - params: { - equipment_id: equipmentId, - target_time: targetTime, - time_window: 24 - } - }); - - const content = document.getElementById('adjacentLotsContent'); - - if (result.error) { - content.innerHTML = `
${result.error}
`; - return; - } - - if (!result.data || result.data.length === 0) { - content.innerHTML = `

無前後批資料

`; - return; - } - - let html = ` -
- - - - - - - - - - - - - - - - - - `; - - result.data.forEach(lot => { - const pos = lot.RELATIVE_POSITION; - const posLabel = pos === 0 ? '目標批次' : (pos > 0 ? `+${pos}` : pos); - const rowClass = pos === 0 ? 'style="background: #fff3cd;"' : ''; - - html += ` - - - - - - - - - - - - - - `; - }); - - html += `
相對位置LOT ID產品類型BOPWafer Lot工單批次號上機時間下機時間上機數下機數
${posLabel}${lot.CONTAINERNAME || '-'}${lot.PJ_TYPE || '-'}${lot.PJ_BOP || '-'}${lot.WAFER_LOT_ID || '-'}${lot.PJ_WORKORDER || '-'}${lot.FINISHEDRUNCARD || ''}${formatDateTime(lot.TRACKINTIMESTAMP)}${formatDateTime(lot.TRACKOUTTIMESTAMP)}${lot.TRACKINQTY || ''}${lot.TRACKOUTQTY || ''}
`; - content.innerHTML = html; - - } catch (error) { - document.getElementById('adjacentLotsContent').innerHTML = `
查詢失敗: ${error.message}
`; - } -} - -// ============================================================ -// Workcenter Group Filter Functions -// ============================================================ - -async function loadWorkcenterGroups() { - try { - const result = await MesApi.get('/api/query-tool/workcenter-groups', { - silent: true - }); - if (result.error) { - console.error('Failed to load workcenter groups:', result.error); - return; - } - - QueryToolState.workcenterGroups = result.data || []; - console.log(`[QueryTool] Loaded ${QueryToolState.workcenterGroups.length} workcenter groups`); - } catch (error) { - console.error('Error loading workcenter groups:', error); - } -} - -function renderWorkcenterGroupSelector() { - const container = document.getElementById('workcenterGroupSelector'); - if (!container) return; - - const groups = QueryToolState.workcenterGroups; - const selected = QueryToolState.selectedWorkcenterGroups; - const count = selected.size; - - let html = ` -
- -
-
- - 已選 ${count} -
-
- -
-
- `; - - groups.forEach(group => { - const isSelected = selected.has(group.name); - html += ` -
- -
- `; - }); - - html += ` -
-
- - -
-
-
- `; - - container.innerHTML = html; -} - -function toggleWorkcenterGroup(groupName, checked) { - if (checked) { - QueryToolState.selectedWorkcenterGroups.add(groupName); - } else { - QueryToolState.selectedWorkcenterGroups.delete(groupName); - } - updateWorkcenterGroupUI(); -} - -function toggleAllWorkcenterGroups(checked) { - if (checked) { - QueryToolState.workcenterGroups.forEach(g => { - QueryToolState.selectedWorkcenterGroups.add(g.name); - }); - } else { - QueryToolState.selectedWorkcenterGroups.clear(); - } - renderWorkcenterGroupSelector(); -} - -function clearWorkcenterGroups() { - QueryToolState.selectedWorkcenterGroups.clear(); - renderWorkcenterGroupSelector(); -} - -function updateWorkcenterGroupUI() { - const count = QueryToolState.selectedWorkcenterGroups.size; - const display = document.getElementById('wcGroupDisplay'); - const badge = document.getElementById('wcGroupBadge'); - const countEl = document.getElementById('wcGroupSelectedCount'); - const selectAll = document.getElementById('wcGroupSelectAll'); - - // Update item visual state - document.querySelectorAll('.wc-group-item').forEach(item => { - const groupName = item.dataset.group; - const isSelected = QueryToolState.selectedWorkcenterGroups.has(groupName); - item.classList.toggle('selected', isSelected); - const checkbox = item.querySelector('input[type="checkbox"]'); - if (checkbox) checkbox.checked = isSelected; - }); - - // Update display text and badge - if (display) { - display.textContent = count === 0 ? '全部站點' : `${count} 個站點群組`; - } - if (badge) { - badge.textContent = count; - badge.style.display = count > 0 ? 'inline-block' : 'none'; - } - if (countEl) { - countEl.textContent = `已選 ${count}`; - } - if (selectAll) { - selectAll.checked = count === QueryToolState.workcenterGroups.length && count > 0; - } -} - -function toggleWorkcenterGroupDropdown() { - const dropdown = document.getElementById('wcGroupDropdown'); - if (dropdown) dropdown.classList.toggle('show'); -} - -function closeWorkcenterGroupDropdown() { - const dropdown = document.getElementById('wcGroupDropdown'); - if (dropdown) dropdown.classList.remove('show'); -} - -function applyWorkcenterFilter() { - // Close dropdown - closeWorkcenterGroupDropdown(); - - // Check if we have selected lots - if (QueryToolState.timelineSelectedLots.size === 0) { - Toast.warning('請先選擇批次'); - return; - } - - const wcGroups = QueryToolState.selectedWorkcenterGroups; - if (wcGroups.size > 0) { - Toast.info(`套用 ${wcGroups.size} 個站點群組篩選...`); - } else { - Toast.info('顯示全部站點資料...'); - } - - // Re-run confirmLotSelection to apply the filter - confirmLotSelection(); -} - -function filterWorkcenterGroups(searchText) { - const items = document.querySelectorAll('.wc-group-item'); - const search = searchText.toLowerCase(); - - items.forEach(item => { - const groupName = item.dataset.group.toLowerCase(); - item.style.display = groupName.includes(search) ? 'flex' : 'none'; - }); -} - -function showWorkcenterGroupSelector() { - // Show the entire selection bar - const selectionBar = document.getElementById('selectionBar'); - if (selectionBar) { - selectionBar.style.display = 'flex'; - } - - // Render the workcenter group selector if groups are available - if (QueryToolState.workcenterGroups.length > 0) { - renderWorkcenterGroupSelector(); - } -} - -// ============================================================ -// Equipment Query Functions -// ============================================================ - -async function loadEquipments() { - try { - const data = await MesApi.get('/api/query-tool/equipment-list'); - if (data.error) { - document.getElementById('equipmentList').innerHTML = `
${data.error}
`; - return; - } - - QueryToolState.allEquipments = data.data; - renderEquipmentList(data.data); - } catch (error) { - document.getElementById('equipmentList').innerHTML = `
載入失敗: ${error.message}
`; - } -} - -function renderEquipmentList(equipments) { - const container = document.getElementById('equipmentList'); - - if (!equipments || equipments.length === 0) { - container.innerHTML = '
無設備資料
'; - return; - } - - let html = ''; - let currentWorkcenter = null; - - equipments.forEach(eq => { - const isSelected = QueryToolState.selectedEquipments.has(eq.RESOURCEID); - - // Group header - if (eq.WORKCENTERNAME !== currentWorkcenter) { - currentWorkcenter = eq.WORKCENTERNAME; - html += `
${currentWorkcenter || '未分類'}
`; - } - - html += ` -
- -
-
${eq.RESOURCENAME}
-
${eq.RESOURCEFAMILYNAME || ''}
-
-
- `; - }); - - container.innerHTML = html; -} - -function toggleEquipmentDropdown() { - const dropdown = document.getElementById('equipmentDropdown'); - dropdown.classList.toggle('show'); -} - -function filterEquipments(query) { - const q = query.toLowerCase(); - const filtered = QueryToolState.allEquipments.filter(eq => - (eq.RESOURCENAME && eq.RESOURCENAME.toLowerCase().includes(q)) || - (eq.WORKCENTERNAME && eq.WORKCENTERNAME.toLowerCase().includes(q)) || - (eq.RESOURCEFAMILYNAME && eq.RESOURCEFAMILYNAME.toLowerCase().includes(q)) - ); - renderEquipmentList(filtered); -} - -function toggleEquipment(resourceId) { - if (QueryToolState.selectedEquipments.has(resourceId)) { - QueryToolState.selectedEquipments.delete(resourceId); - } else { - if (QueryToolState.selectedEquipments.size >= 20) { - Toast.warning('最多只能選擇 20 台設備'); - return; - } - QueryToolState.selectedEquipments.add(resourceId); - } - updateSelectedDisplay(); - - // Re-render with current filter - const search = document.querySelector('.equipment-search'); - if (search && search.value) { - filterEquipments(search.value); - } else { - renderEquipmentList(QueryToolState.allEquipments); - } -} - -function updateSelectedDisplay() { - const display = document.getElementById('equipmentDisplay'); - const count = document.getElementById('selectedCount'); - - if (QueryToolState.selectedEquipments.size === 0) { - display.textContent = '點擊選擇設備...'; - count.textContent = ''; - } else if (QueryToolState.selectedEquipments.size <= 3) { - const names = QueryToolState.allEquipments - .filter(eq => QueryToolState.selectedEquipments.has(eq.RESOURCEID)) - .map(eq => eq.RESOURCENAME) - .join(', '); - display.textContent = names; - count.textContent = `已選擇 ${QueryToolState.selectedEquipments.size} 台設備`; - } else { - display.textContent = `已選擇 ${QueryToolState.selectedEquipments.size} 台設備`; - count.textContent = ''; - } -} - -function setLast30Days() { - const today = new Date(); - const past = new Date(); - past.setDate(today.getDate() - 30); - - document.getElementById('dateFrom').value = past.toISOString().split('T')[0]; - document.getElementById('dateTo').value = today.toISOString().split('T')[0]; -} - -async function executeEquipmentQuery() { - if (QueryToolState.selectedEquipments.size === 0) { - Toast.error('請選擇至少一台設備'); - return; - } - - const dateFrom = document.getElementById('dateFrom').value; - const dateTo = document.getElementById('dateTo').value; - - if (!dateFrom || !dateTo) { - Toast.error('請指定日期範圍'); - return; - } - - // Validate date range - const from = new Date(dateFrom); - const to = new Date(dateTo); - - if (to < from) { - Toast.error('結束日期不可早於起始日期'); - return; - } - - const daysDiff = (to - from) / (1000 * 60 * 60 * 24); - if (daysDiff > 90) { - Toast.error('日期範圍不可超過 90 天'); - return; - } - - // Clear previous equipment results to free memory - QueryToolState.equipmentResults = null; - const eqContent = document.getElementById('eqResultsContent'); - if (eqContent) eqContent.innerHTML = ''; - - // Show loading - document.getElementById('eqEmptyState').style.display = 'none'; - document.getElementById('eqResultsContent').style.display = 'block'; - document.getElementById('eqResultsContent').innerHTML = ` -
- -
查詢中... -
- `; - - document.getElementById('eqQueryBtn').disabled = true; - - const equipmentIds = Array.from(QueryToolState.selectedEquipments); - const equipmentNames = QueryToolState.allEquipments - .filter(eq => QueryToolState.selectedEquipments.has(eq.RESOURCEID)) - .map(eq => eq.RESOURCENAME); - - try { - // Load status hours first - const statusResult = await MesApi.post('/api/query-tool/equipment-period', { - equipment_ids: equipmentIds, - equipment_names: equipmentNames, - start_date: dateFrom, - end_date: dateTo, - query_type: 'status_hours' - }); - - QueryToolState.equipmentResults = { - status_hours: statusResult, - equipment_ids: equipmentIds, - equipment_names: equipmentNames, - date_range: { start: dateFrom, end: dateTo } - }; - - renderEquipmentResults(); - - } catch (error) { - document.getElementById('eqResultsContent').innerHTML = `
查詢失敗: ${error.message}
`; - } finally { - document.getElementById('eqQueryBtn').disabled = false; - } -} - -function renderEquipmentResults() { - const results = QueryToolState.equipmentResults; - const content = document.getElementById('eqResultsContent'); - - let html = ` -
-
- ${QueryToolState.selectedEquipments.size} 台設備 | ${results.date_range.start} ~ ${results.date_range.end} -
-
- -
- - - - - -
- -
- `; - - content.innerHTML = html; - - // Render initial tab - renderEquipmentTab('status_hours', results.status_hours); -} - -async function loadEquipmentTab(tabType, tabElement) { - // Update tab states - document.querySelectorAll('#eqResultsContent .assoc-tab').forEach(t => t.classList.remove('active')); - if (tabElement) tabElement.classList.add('active'); - - const content = document.getElementById('eqTabContent'); - content.innerHTML = `
`; - - const results = QueryToolState.equipmentResults; - - // Check if already loaded - if (results[tabType]) { - renderEquipmentTab(tabType, results[tabType]); - return; - } - - try { - const result = await MesApi.post('/api/query-tool/equipment-period', { - equipment_ids: results.equipment_ids, - equipment_names: results.equipment_names, - start_date: results.date_range.start, - end_date: results.date_range.end, - query_type: tabType - }); - - results[tabType] = result; - renderEquipmentTab(tabType, result); - - } catch (error) { - content.innerHTML = `
載入失敗: ${error.message}
`; - } -} - -function renderEquipmentTab(tabType, result) { - const content = document.getElementById('eqTabContent'); - - if (result.error) { - content.innerHTML = `
${result.error}
`; - return; - } - - if (!result.data || result.data.length === 0) { - content.innerHTML = `

無資料

`; - return; - } - - const data = result.data; - - // Define columns based on tab type - const columnDefs = { - 'status_hours': { - cols: ['RESOURCENAME', 'PRD_HOURS', 'SBY_HOURS', 'UDT_HOURS', 'SDT_HOURS', 'EGT_HOURS', 'NST_HOURS', 'TOTAL_HOURS', 'OU_PERCENT'], - labels: { RESOURCENAME: '設備名稱', PRD_HOURS: '生產', SBY_HOURS: '待機', UDT_HOURS: '非計畫停機', SDT_HOURS: '計畫停機', EGT_HOURS: '工程', NST_HOURS: '非排程', TOTAL_HOURS: '總時數', OU_PERCENT: 'OU%' } - }, - 'lots': { - cols: ['EQUIPMENTNAME', 'WORKCENTERNAME', 'CONTAINERNAME', 'PJ_TYPE', 'PJ_BOP', 'WAFER_LOT_ID', 'FINISHEDRUNCARD', 'SPECNAME', 'TRACKINTIMESTAMP', 'TRACKOUTTIMESTAMP', 'TRACKINQTY', 'TRACKOUTQTY'], - labels: { EQUIPMENTNAME: '設備', WORKCENTERNAME: '站點', CONTAINERNAME: 'LOT ID', PJ_TYPE: '產品類型', PJ_BOP: 'BOP', WAFER_LOT_ID: 'Wafer Lot', FINISHEDRUNCARD: '批次號', SPECNAME: '規格', TRACKINTIMESTAMP: '上機時間', TRACKOUTTIMESTAMP: '下機時間', TRACKINQTY: '上機數', TRACKOUTQTY: '下機數' } - }, - 'materials': { - cols: ['EQUIPMENTNAME', 'MATERIALPARTNAME', 'TOTAL_CONSUMED', 'LOT_COUNT'], - labels: { EQUIPMENTNAME: '設備', MATERIALPARTNAME: '物料名稱', TOTAL_CONSUMED: '消耗總量', LOT_COUNT: '批次數' } - }, - 'rejects': { - cols: ['EQUIPMENTNAME', 'LOSSREASONNAME', 'TOTAL_DEFECT_QTY', 'TOTAL_REJECT_QTY', 'AFFECTED_LOT_COUNT'], - labels: { EQUIPMENTNAME: '設備', LOSSREASONNAME: '損失原因', TOTAL_DEFECT_QTY: '不良數量', TOTAL_REJECT_QTY: 'REJECT數量', AFFECTED_LOT_COUNT: '影響批次' } - }, - 'jobs': { - cols: ['RESOURCENAME', 'JOBID', 'JOBSTATUS', 'JOBMODELNAME', 'CREATEDATE', 'COMPLETEDATE', 'CAUSECODENAME', 'REPAIRCODENAME'], - labels: { RESOURCENAME: '設備', JOBID: 'JOB ID', JOBSTATUS: '狀態', JOBMODELNAME: '類型', CREATEDATE: '建立時間', COMPLETEDATE: '完成時間', CAUSECODENAME: '原因代碼', REPAIRCODENAME: '維修代碼' } - } - }; - - const def = columnDefs[tabType] || { cols: Object.keys(data[0]), labels: {} }; - - // Add export button - let html = ` -
- -
- `; - - // Show totals for status_hours - if (tabType === 'status_hours' && result.totals) { - const t = result.totals; - html += ` -
- 總計: PRD ${t.PRD_HOURS?.toFixed(1) || 0}h | SBY ${t.SBY_HOURS?.toFixed(1) || 0}h | - UDT ${t.UDT_HOURS?.toFixed(1) || 0}h | OU% ${t.OU_PERCENT?.toFixed(1) || 0}% -
- `; - } - - html += `
`; - - def.cols.forEach(col => { - html += ``; - }); - html += ``; - - data.forEach(row => { - html += ``; - def.cols.forEach(col => { - let value = row[col]; - if (col.includes('DATE') || col.includes('TIMESTAMP')) { - value = formatDateTime(value); - } - if (col === 'OU_PERCENT' && value !== null) { - value = `${value}%`; - } - if ((col.endsWith('_HOURS') || col === 'TOTAL_CONSUMED') && value !== null) { - value = Number(value).toFixed(2); - } - html += ``; - }); - html += ``; - }); - - html += `
${def.labels[col] || col}
${value !== null && value !== undefined ? value : ''}
`; - content.innerHTML = html; -} - -// ============================================================ -// Export Functions -// ============================================================ - -async function exportLotResults() { - if (QueryToolState.resolvedLots.length === 0) { - Toast.error('無資料可匯出'); - return; - } - - const lot = QueryToolState.resolvedLots[QueryToolState.selectedLotIndex]; - - try { - const response = await fetch('/api/query-tool/export-csv', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - export_type: 'lot_history', - params: { container_id: lot.container_id } - }) - }); - - if (!response.ok) { - const data = await response.json(); - throw new Error(data.error || '匯出失敗'); - } - - downloadBlob(response, `lot_history_${lot.lot_id || lot.input_value}.csv`); - Toast.success('CSV 匯出完成'); - - } catch (error) { - Toast.error('匯出失敗: ' + error.message); - } -} - -async function exportCombinedResults() { - const selectedIndices = QueryToolState.currentSelectedIndices || []; - if (selectedIndices.length === 0) { - Toast.error('無資料可匯出'); - return; - } - - // Collect all history data - const lots = QueryToolState.resolvedLots; - const allHistory = []; - - selectedIndices.forEach(idx => { - const lot = lots[idx]; - const history = QueryToolState.lotHistories[lot.container_id] || []; - history.forEach(step => { - allHistory.push({ - LOT_ID: lot.lot_id || lot.input_value, - ...step - }); - }); - }); - - if (allHistory.length === 0) { - Toast.error('無資料可匯出'); - return; - } - - // Generate CSV - const headers = ['LOT_ID', 'WORKCENTERNAME', 'EQUIPMENTNAME', 'SPECNAME', 'PJ_TYPE', 'PJ_BOP', 'WAFER_LOT_ID', 'TRACKINTIMESTAMP', 'TRACKOUTTIMESTAMP', 'TRACKINQTY', 'TRACKOUTQTY']; - let csv = headers.join(',') + '\n'; - - allHistory.forEach(row => { - csv += headers.map(h => { - let val = row[h] || ''; - if (typeof val === 'string' && (val.includes(',') || val.includes('"'))) { - val = '"' + val.replace(/"/g, '""') + '"'; - } - return val; - }).join(',') + '\n'; - }); - - // Download - const blob = new Blob(['\ufeff' + csv], { type: 'text/csv;charset=utf-8;' }); - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = `combined_lot_history_${selectedIndices.length}lots.csv`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - window.URL.revokeObjectURL(url); - - Toast.success('CSV 匯出完成'); -} - -// Export Production History (拆併批紀錄) -function exportProductionHistory(mode = 'single') { - let productionHistory = []; - let filename = 'production_split_merge_history'; - - if (mode === 'combined') { - // Get from combined splits data - productionHistory = QueryToolState.combinedSplitsData?.production_history || []; - const selectedCount = (QueryToolState.currentSelectedIndices || []).length; - filename = `production_split_merge_history_${selectedCount}lots`; - } else { - // Get from current LOT's splits data (cached in lotAssociations) - const containerId = QueryToolState.currentContainerId; - const cacheKey = `${containerId}_splits`; - const splitsData = QueryToolState.lotAssociations?.[cacheKey] || {}; - productionHistory = splitsData.production_history || []; - const lotId = QueryToolState.resolvedLots?.find(l => l.container_id === containerId)?.lot_id || containerId; - filename = `production_split_merge_history_${lotId}`; - } - - if (!productionHistory || productionHistory.length === 0) { - Toast.error('無生產拆併批紀錄可匯出'); - return; - } - - // Generate CSV - const headers = ['LOT_ID', '操作類型', '來源批次', '目標批次', '數量', '時間']; - const keys = ['LOT_ID', 'operation_type_display', 'source_lot', 'target_lot', 'target_qty', 'txn_date']; - let csv = headers.join(',') + '\n'; - - productionHistory.forEach(row => { - csv += keys.map(k => { - let val = row[k] || ''; - if (typeof val === 'string' && (val.includes(',') || val.includes('"'))) { - val = '"' + val.replace(/"/g, '""') + '"'; - } - return val; - }).join(',') + '\n'; - }); - - // Download - const blob = new Blob(['\ufeff' + csv], { type: 'text/csv;charset=utf-8;' }); - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = `${filename}.csv`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - window.URL.revokeObjectURL(url); - - Toast.success('CSV 匯出完成'); -} - -// Export Serial Numbers (成品流水號對應) -function exportSerialNumbers(mode = 'single') { - let serialNumbers = []; - let filename = 'serial_number_mapping'; - - if (mode === 'combined') { - // Get from combined splits data - serialNumbers = QueryToolState.combinedSplitsData?.serial_numbers || []; - const selectedCount = (QueryToolState.currentSelectedIndices || []).length; - filename = `serial_number_mapping_${selectedCount}lots`; - } else { - // Get from current LOT's splits data (cached in lotAssociations) - const containerId = QueryToolState.currentContainerId; - const cacheKey = `${containerId}_splits`; - const splitsData = QueryToolState.lotAssociations?.[cacheKey] || {}; - serialNumbers = splitsData.serial_numbers || []; - const lotId = QueryToolState.resolvedLots?.find(l => l.container_id === containerId)?.lot_id || containerId; - filename = `serial_number_mapping_${lotId}`; - } - - if (!serialNumbers || serialNumbers.length === 0) { - Toast.error('無成品流水號對應資料可匯出'); - return; - } - - // Flatten serial numbers data - const flatData = []; - serialNumbers.forEach(snGroup => { - const sn = snGroup.serial_number || ''; - const totalDie = snGroup.total_good_die || 0; - (snGroup.lots || []).forEach(lot => { - flatData.push({ - serial_number: sn, - total_good_die: totalDie, - lot_id: lot.lot_id || '', - combine_ratio_pct: lot.combine_ratio_pct || '', - good_die_qty: lot.good_die_qty || 0 - }); - }); - }); - - if (flatData.length === 0) { - Toast.error('無成品流水號對應資料可匯出'); - return; - } - - // Generate CSV - const headers = ['流水號', '總 Good Die', 'LOT ID', '佔比', 'Good Die 數']; - const keys = ['serial_number', 'total_good_die', 'lot_id', 'combine_ratio_pct', 'good_die_qty']; - let csv = headers.join(',') + '\n'; - - flatData.forEach(row => { - csv += keys.map(k => { - let val = row[k] || ''; - if (typeof val === 'string' && (val.includes(',') || val.includes('"'))) { - val = '"' + val.replace(/"/g, '""') + '"'; - } - return val; - }).join(',') + '\n'; - }); - - // Download - const blob = new Blob(['\ufeff' + csv], { type: 'text/csv;charset=utf-8;' }); - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = `${filename}.csv`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - window.URL.revokeObjectURL(url); - - Toast.success('CSV 匯出完成'); -} - -async function exportEquipmentTab(tabType) { - const results = QueryToolState.equipmentResults; - - if (!results || !results[tabType] || !results[tabType].data) { - Toast.error('無資料可匯出'); - return; - } - - try { - const params = { - start_date: results.date_range.start, - end_date: results.date_range.end - }; - - if (tabType === 'materials' || tabType === 'rejects') { - params.equipment_names = results.equipment_names; - } else { - params.equipment_ids = results.equipment_ids; - } - - const response = await fetch('/api/query-tool/export-csv', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - export_type: `equipment_${tabType}`, - params: params - }) - }); - - if (!response.ok) { - const data = await response.json(); - throw new Error(data.error || '匯出失敗'); - } - - downloadBlob(response, `equipment_${tabType}.csv`); - Toast.success('CSV 匯出完成'); - - } catch (error) { - Toast.error('匯出失敗: ' + error.message); - } -} - -async function downloadBlob(response, filename) { - const blob = await response.blob(); - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = filename; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - window.URL.revokeObjectURL(url); -} - -// ============================================================ -// Tab Navigation (Legacy - kept for compatibility) -// ============================================================ - -function switchMainTab(tabId) { - // This function is kept for compatibility but the main UI now uses - // query mode switching (batch vs equipment) instead of tabs -} - -// ============================================================ -// Utility Functions -// ============================================================ - -function formatDateTime(dateInput) { - if (!dateInput) return ''; - - // Handle Date objects - if (dateInput instanceof Date) { - const y = dateInput.getFullYear(); - const m = (dateInput.getMonth() + 1).toString().padStart(2, '0'); - const d = dateInput.getDate().toString().padStart(2, '0'); - const h = dateInput.getHours().toString().padStart(2, '0'); - const min = dateInput.getMinutes().toString().padStart(2, '0'); - const s = dateInput.getSeconds().toString().padStart(2, '0'); - return `${y}-${m}-${d} ${h}:${min}:${s}`; - } - - // Handle timestamps (numbers) - if (typeof dateInput === 'number') { - return formatDateTime(new Date(dateInput)); - } - - // Handle strings - if (typeof dateInput === 'string') { - return dateInput.replace('T', ' ').substring(0, 19); - } - - return ''; -} - -function truncateText(text, maxLength) { - if (!text) return ''; - if (text.length <= maxLength) return text; - return text.substring(0, maxLength) + '...'; -} diff --git a/tests/test_event_fetcher.py b/tests/test_event_fetcher.py new file mode 100644 index 0000000..9a81d3f --- /dev/null +++ b/tests/test_event_fetcher.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +"""Unit tests for EventFetcher.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pandas as pd + +from mes_dashboard.services.event_fetcher import EventFetcher + + +def test_cache_key_is_stable_for_sorted_ids(): + key1 = EventFetcher._cache_key("history", ["CID-B", "CID-A", "CID-A"]) + key2 = EventFetcher._cache_key("history", ["CID-A", "CID-B"]) + + assert key1 == key2 + assert key1.startswith("evt:history:") + + +def test_get_rate_limit_config_supports_env_override(monkeypatch): + monkeypatch.setenv("EVT_HISTORY_RATE_MAX_REQUESTS", "33") + monkeypatch.setenv("EVT_HISTORY_RATE_WINDOW_SECONDS", "77") + + config = EventFetcher._get_rate_limit_config("history") + + assert config["bucket"] == "event-history" + assert config["max_attempts"] == 33 + assert config["window_seconds"] == 77 + + +@patch("mes_dashboard.services.event_fetcher.read_sql_df") +@patch("mes_dashboard.services.event_fetcher.cache_get") +def test_fetch_events_cache_hit_skips_db(mock_cache_get, mock_read_sql_df): + mock_cache_get.return_value = {"CID-1": [{"CONTAINERID": "CID-1"}]} + + result = EventFetcher.fetch_events(["CID-1"], "materials") + + assert result["CID-1"][0]["CONTAINERID"] == "CID-1" + mock_read_sql_df.assert_not_called() + + +@patch("mes_dashboard.services.event_fetcher.cache_set") +@patch("mes_dashboard.services.event_fetcher.cache_get", return_value=None) +@patch("mes_dashboard.services.event_fetcher.read_sql_df") +@patch("mes_dashboard.services.event_fetcher.SQLLoader.load_with_params") +def test_fetch_events_upstream_history_branch( + mock_sql_load, + mock_read_sql_df, + _mock_cache_get, + mock_cache_set, +): + mock_sql_load.return_value = "SELECT * FROM UPSTREAM" + mock_read_sql_df.return_value = pd.DataFrame( + [ + {"CONTAINERID": "CID-1", "WORKCENTER_GROUP": "DB"}, + {"CONTAINERID": "CID-2", "WORKCENTER_GROUP": "WB"}, + ] + ) + + result = EventFetcher.fetch_events(["CID-1", "CID-2"], "upstream_history") + + assert sorted(result.keys()) == ["CID-1", "CID-2"] + assert mock_sql_load.call_args.args[0] == "mid_section_defect/upstream_history" + _, params = mock_read_sql_df.call_args.args + assert len(params) == 2 + mock_cache_set.assert_called_once() + assert mock_cache_set.call_args.args[0].startswith("evt:upstream_history:") + + +@patch("mes_dashboard.services.event_fetcher.cache_set") +@patch("mes_dashboard.services.event_fetcher.cache_get", return_value=None) +@patch("mes_dashboard.services.event_fetcher.read_sql_df") +@patch("mes_dashboard.services.event_fetcher.SQLLoader.load") +def test_fetch_events_history_branch_replaces_container_filter( + mock_sql_load, + mock_read_sql_df, + _mock_cache_get, + _mock_cache_set, +): + mock_sql_load.return_value = ( + "SELECT * FROM t WHERE h.CONTAINERID = :container_id {{ WORKCENTER_FILTER }}" + ) + mock_read_sql_df.return_value = pd.DataFrame([]) + + EventFetcher.fetch_events(["CID-1"], "history") + + sql, params = mock_read_sql_df.call_args.args + assert "h.CONTAINERID = :container_id" not in sql + assert "{{ WORKCENTER_FILTER }}" not in sql + assert params == {"p0": "CID-1"} diff --git a/tests/test_lineage_engine.py b/tests/test_lineage_engine.py new file mode 100644 index 0000000..4b12e2f --- /dev/null +++ b/tests/test_lineage_engine.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +"""Unit tests for LineageEngine.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pandas as pd + +from mes_dashboard.services.lineage_engine import LineageEngine + + +@patch("mes_dashboard.services.lineage_engine.read_sql_df") +def test_resolve_split_ancestors_batches_and_enforces_max_depth(mock_read_sql_df): + cids = [f"C{i:04d}" for i in range(1001)] + mock_read_sql_df.side_effect = [ + pd.DataFrame( + [ + { + "CONTAINERID": "C0000", + "SPLITFROMID": "P0000", + "CONTAINERNAME": "LOT-0000", + "SPLIT_DEPTH": 1, + }, + { + "CONTAINERID": "P0000", + "SPLITFROMID": None, + "CONTAINERNAME": "LOT-P0000", + "SPLIT_DEPTH": 2, + }, + ] + ), + pd.DataFrame( + [ + { + "CONTAINERID": "C1000", + "SPLITFROMID": "P1000", + "CONTAINERNAME": "LOT-1000", + "SPLIT_DEPTH": 1, + }, + { + "CONTAINERID": "C-TOO-DEEP", + "SPLITFROMID": "P-TOO-DEEP", + "CONTAINERNAME": "LOT-DEEP", + "SPLIT_DEPTH": 21, + }, + ] + ), + ] + + result = LineageEngine.resolve_split_ancestors(cids, {"INIT": "LOT-INIT"}) + + assert mock_read_sql_df.call_count == 2 + first_sql, first_params = mock_read_sql_df.call_args_list[0].args + second_sql, second_params = mock_read_sql_df.call_args_list[1].args + assert "LEVEL <= 20" in first_sql + assert "LEVEL <= 20" in second_sql + assert len(first_params) == 1000 + assert len(second_params) == 1 + + assert result["child_to_parent"]["C0000"] == "P0000" + assert result["child_to_parent"]["C1000"] == "P1000" + assert "C-TOO-DEEP" not in result["child_to_parent"] + assert result["cid_to_name"]["C0000"] == "LOT-0000" + assert result["cid_to_name"]["INIT"] == "LOT-INIT" + + +@patch("mes_dashboard.services.lineage_engine.read_sql_df") +def test_resolve_merge_sources_batches_and_returns_mapping(mock_read_sql_df): + names = [f"FN{i:04d}" for i in range(1001)] + mock_read_sql_df.side_effect = [ + pd.DataFrame( + [ + {"FINISHEDNAME": "FN0000", "SOURCE_CID": "SRC-A"}, + {"FINISHEDNAME": "FN0000", "SOURCE_CID": "SRC-B"}, + ] + ), + pd.DataFrame( + [ + {"FINISHEDNAME": "FN1000", "SOURCE_CID": "SRC-C"}, + {"FINISHEDNAME": "FN1000", "SOURCE_CID": "SRC-C"}, + {"FINISHEDNAME": None, "SOURCE_CID": "SRC-INVALID"}, + ] + ), + ] + + result = LineageEngine.resolve_merge_sources(names) + + assert mock_read_sql_df.call_count == 2 + first_sql, first_params = mock_read_sql_df.call_args_list[0].args + second_sql, second_params = mock_read_sql_df.call_args_list[1].args + assert "{{ FINISHED_NAME_FILTER }}" not in first_sql + assert "{{ FINISHED_NAME_FILTER }}" not in second_sql + assert len(first_params) == 1000 + assert len(second_params) == 1 + + assert result["FN0000"] == ["SRC-A", "SRC-B"] + assert result["FN1000"] == ["SRC-C"] + + +@patch("mes_dashboard.services.lineage_engine.LineageEngine.resolve_merge_sources") +@patch("mes_dashboard.services.lineage_engine.LineageEngine.resolve_split_ancestors") +def test_resolve_full_genealogy_combines_split_and_merge( + mock_resolve_split_ancestors, + mock_resolve_merge_sources, +): + mock_resolve_split_ancestors.side_effect = [ + { + "child_to_parent": { + "A": "B", + "B": "C", + }, + "cid_to_name": { + "A": "LOT-A", + "B": "LOT-B", + "C": "LOT-C", + }, + }, + { + "child_to_parent": { + "M1": "M0", + }, + "cid_to_name": { + "M1": "LOT-M1", + "M0": "LOT-M0", + }, + }, + ] + mock_resolve_merge_sources.return_value = {"LOT-B": ["M1"]} + + result = LineageEngine.resolve_full_genealogy(["A"], {"A": "LOT-A"}) + + assert result == {"A": {"B", "C", "M1", "M0"}} + assert mock_resolve_split_ancestors.call_count == 2 + mock_resolve_merge_sources.assert_called_once() + + +@patch("mes_dashboard.services.lineage_engine.read_sql_df") +def test_split_ancestors_matches_legacy_bfs_for_five_known_lots(mock_read_sql_df): + parent_by_cid = { + "L1": "L1P1", + "L1P1": "L1P2", + "L2": "L2P1", + "L3": None, + "L4": "L4P1", + "L4P1": "L4P2", + "L4P2": "L4P3", + "L5": "L5P1", + "L5P1": "L5P2", + "L5P2": "L5P1", + } + name_by_cid = { + "L1": "LOT-1", + "L1P1": "LOT-1-P1", + "L1P2": "LOT-1-P2", + "L2": "LOT-2", + "L2P1": "LOT-2-P1", + "L3": "LOT-3", + "L4": "LOT-4", + "L4P1": "LOT-4-P1", + "L4P2": "LOT-4-P2", + "L4P3": "LOT-4-P3", + "L5": "LOT-5", + "L5P1": "LOT-5-P1", + "L5P2": "LOT-5-P2", + } + seed_lots = ["L1", "L2", "L3", "L4", "L5"] + + def _connect_by_rows(start_cids): + rows = [] + for seed in start_cids: + current = seed + depth = 1 + visited = set() + while current and depth <= 20 and current not in visited: + visited.add(current) + rows.append( + { + "CONTAINERID": current, + "SPLITFROMID": parent_by_cid.get(current), + "CONTAINERNAME": name_by_cid.get(current), + "SPLIT_DEPTH": depth, + } + ) + current = parent_by_cid.get(current) + depth += 1 + return pd.DataFrame(rows) + + def _mock_read_sql(_sql, params): + requested = [value for value in params.values()] + return _connect_by_rows(requested) + + mock_read_sql_df.side_effect = _mock_read_sql + + connect_by_result = LineageEngine.resolve_split_ancestors(seed_lots) + + # Legacy BFS reference implementation from previous mid_section_defect_service. + legacy_child_to_parent = {} + legacy_cid_to_name = {} + frontier = list(seed_lots) + seen = set(seed_lots) + rounds = 0 + while frontier: + rounds += 1 + batch_rows = [] + for cid in frontier: + batch_rows.append( + { + "CONTAINERID": cid, + "SPLITFROMID": parent_by_cid.get(cid), + "CONTAINERNAME": name_by_cid.get(cid), + } + ) + new_parents = set() + for row in batch_rows: + cid = row["CONTAINERID"] + split_from = row["SPLITFROMID"] + name = row["CONTAINERNAME"] + if isinstance(name, str) and name: + legacy_cid_to_name[cid] = name + if isinstance(split_from, str) and split_from and split_from != cid: + legacy_child_to_parent[cid] = split_from + if split_from not in seen: + seen.add(split_from) + new_parents.add(split_from) + frontier = list(new_parents) + if rounds > 20: + break + + assert connect_by_result["child_to_parent"] == legacy_child_to_parent + assert connect_by_result["cid_to_name"] == legacy_cid_to_name diff --git a/tests/test_mid_section_defect_service.py b/tests/test_mid_section_defect_service.py index 5b7a74d..e4246c0 100644 --- a/tests/test_mid_section_defect_service.py +++ b/tests/test_mid_section_defect_service.py @@ -8,6 +8,7 @@ from unittest.mock import patch import pandas as pd from mes_dashboard.services.mid_section_defect_service import ( + build_trace_aggregation_from_events, query_analysis, query_analysis_detail, query_all_loss_reasons, @@ -126,3 +127,116 @@ def test_query_all_loss_reasons_cache_miss_queries_and_caches_sorted_values( {'loss_reasons': ['A_REASON', 'B_REASON']}, ttl=86400, ) + + +@patch('mes_dashboard.services.mid_section_defect_service.cache_set') +@patch('mes_dashboard.services.mid_section_defect_service.cache_get', return_value=None) +@patch('mes_dashboard.services.mid_section_defect_service.release_lock') +@patch('mes_dashboard.services.mid_section_defect_service.try_acquire_lock', return_value=True) +@patch('mes_dashboard.services.mid_section_defect_service._fetch_upstream_history') +@patch('mes_dashboard.services.mid_section_defect_service._resolve_full_genealogy') +@patch('mes_dashboard.services.mid_section_defect_service._fetch_tmtt_data') +def test_trace_aggregation_matches_query_analysis_summary( + mock_fetch_tmtt_data, + mock_resolve_genealogy, + mock_fetch_upstream_history, + _mock_lock, + _mock_release_lock, + _mock_cache_get, + _mock_cache_set, +): + tmtt_df = pd.DataFrame([ + { + 'CONTAINERID': 'CID-001', + 'CONTAINERNAME': 'LOT-001', + 'TRACKINQTY': 100, + 'REJECTQTY': 5, + 'LOSSREASONNAME': 'R1', + 'WORKFLOW': 'WF-A', + 'PRODUCTLINENAME': 'PKG-A', + 'PJ_TYPE': 'TYPE-A', + 'TMTT_EQUIPMENTNAME': 'TMTT-01', + 'TRACKINTIMESTAMP': '2025-01-10 10:00:00', + 'FINISHEDRUNCARD': 'FR-001', + }, + { + 'CONTAINERID': 'CID-002', + 'CONTAINERNAME': 'LOT-002', + 'TRACKINQTY': 120, + 'REJECTQTY': 6, + 'LOSSREASONNAME': 'R2', + 'WORKFLOW': 'WF-B', + 'PRODUCTLINENAME': 'PKG-B', + 'PJ_TYPE': 'TYPE-B', + 'TMTT_EQUIPMENTNAME': 'TMTT-02', + 'TRACKINTIMESTAMP': '2025-01-11 10:00:00', + 'FINISHEDRUNCARD': 'FR-002', + }, + ]) + + ancestors = { + 'CID-001': {'CID-101'}, + 'CID-002': set(), + } + upstream_normalized = { + 'CID-101': [{ + 'workcenter_group': '中段', + 'equipment_id': 'EQ-01', + 'equipment_name': 'EQ-01', + 'spec_name': 'SPEC-A', + 'track_in_time': '2025-01-09 08:00:00', + }], + 'CID-002': [{ + 'workcenter_group': '中段', + 'equipment_id': 'EQ-02', + 'equipment_name': 'EQ-02', + 'spec_name': 'SPEC-B', + 'track_in_time': '2025-01-11 08:00:00', + }], + } + upstream_events = { + 'CID-101': [{ + 'WORKCENTER_GROUP': '中段', + 'EQUIPMENTID': 'EQ-01', + 'EQUIPMENTNAME': 'EQ-01', + 'SPECNAME': 'SPEC-A', + 'TRACKINTIMESTAMP': '2025-01-09 08:00:00', + }], + 'CID-002': [{ + 'WORKCENTER_GROUP': '中段', + 'EQUIPMENTID': 'EQ-02', + 'EQUIPMENTNAME': 'EQ-02', + 'SPECNAME': 'SPEC-B', + 'TRACKINTIMESTAMP': '2025-01-11 08:00:00', + }], + } + + mock_fetch_tmtt_data.return_value = tmtt_df + mock_resolve_genealogy.return_value = ancestors + mock_fetch_upstream_history.return_value = upstream_normalized + + summary = query_analysis('2025-01-01', '2025-01-31') + staged_summary = build_trace_aggregation_from_events( + '2025-01-01', + '2025-01-31', + seed_container_ids=['CID-001', 'CID-002'], + lineage_ancestors={ + 'CID-001': ['CID-101'], + 'CID-002': [], + }, + upstream_events_by_cid=upstream_events, + ) + + assert staged_summary['available_loss_reasons'] == summary['available_loss_reasons'] + assert staged_summary['genealogy_status'] == summary['genealogy_status'] + assert staged_summary['detail_total_count'] == len(summary['detail']) + + assert staged_summary['kpi']['total_input'] == summary['kpi']['total_input'] + assert staged_summary['kpi']['lot_count'] == summary['kpi']['lot_count'] + assert staged_summary['kpi']['total_defect_qty'] == summary['kpi']['total_defect_qty'] + assert abs( + staged_summary['kpi']['total_defect_rate'] - summary['kpi']['total_defect_rate'] + ) <= 0.01 + + assert staged_summary['daily_trend'] == summary['daily_trend'] + assert staged_summary['charts'].keys() == summary['charts'].keys() diff --git a/tests/test_query_tool_routes.py b/tests/test_query_tool_routes.py index e07853c..bfff1b4 100644 --- a/tests/test_query_tool_routes.py +++ b/tests/test_query_tool_routes.py @@ -7,25 +7,35 @@ Tests the API endpoints with mocked service dependencies: - Error handling """ -import pytest -import json -from unittest.mock import patch, MagicMock - -from mes_dashboard import create_app +import pytest +import json +from unittest.mock import patch, MagicMock + +from mes_dashboard import create_app +from mes_dashboard.core.cache import NoOpCache +from mes_dashboard.core.rate_limit import reset_rate_limits_for_tests @pytest.fixture -def app(): - """Create test Flask application.""" - app = create_app() - app.config['TESTING'] = True - return app +def app(): + """Create test Flask application.""" + app = create_app() + app.config['TESTING'] = True + app.extensions["cache"] = NoOpCache() + return app -@pytest.fixture -def client(app): - """Create test client.""" - return app.test_client() +@pytest.fixture +def client(app): + """Create test client.""" + return app.test_client() + + +@pytest.fixture(autouse=True) +def _reset_rate_limits(): + reset_rate_limits_for_tests() + yield + reset_rate_limits_for_tests() class TestQueryToolPage: @@ -129,8 +139,8 @@ class TestResolveEndpoint: assert data['total'] == 1 assert data['data'][0]['lot_id'] == 'GA23100020-A00-001' - @patch('mes_dashboard.routes.query_tool_routes.resolve_lots') - def test_resolve_not_found(self, mock_resolve, client): + @patch('mes_dashboard.routes.query_tool_routes.resolve_lots') + def test_resolve_not_found(self, mock_resolve, client): """Should return not_found list for missing LOT IDs.""" mock_resolve.return_value = { 'data': [], @@ -148,8 +158,56 @@ class TestResolveEndpoint: ) assert response.status_code == 200 data = json.loads(response.data) - assert data['total'] == 0 - assert 'INVALID-LOT-ID' in data['not_found'] + assert data['total'] == 0 + assert 'INVALID-LOT-ID' in data['not_found'] + + @patch('mes_dashboard.routes.query_tool_routes.resolve_lots') + @patch('mes_dashboard.routes.query_tool_routes.cache_get') + def test_resolve_cache_hit_skips_service(self, mock_cache_get, mock_resolve, client): + mock_cache_get.return_value = { + 'data': [{'container_id': 'C1', 'input_value': 'LOT-1'}], + 'total': 1, + 'input_count': 1, + 'not_found': [], + } + + response = client.post( + '/api/query-tool/resolve', + json={'input_type': 'lot_id', 'values': ['LOT-1']}, + ) + + assert response.status_code == 200 + payload = response.get_json() + assert payload['total'] == 1 + mock_resolve.assert_not_called() + + @patch('mes_dashboard.routes.query_tool_routes.cache_set') + @patch('mes_dashboard.routes.query_tool_routes.cache_get', return_value=None) + @patch('mes_dashboard.routes.query_tool_routes.resolve_lots') + def test_resolve_success_caches_result( + self, + mock_resolve, + _mock_cache_get, + mock_cache_set, + client, + ): + mock_resolve.return_value = { + 'data': [{'container_id': 'C1', 'input_value': 'LOT-1'}], + 'total': 1, + 'input_count': 1, + 'not_found': [], + } + + response = client.post( + '/api/query-tool/resolve', + json={'input_type': 'lot_id', 'values': ['LOT-1']}, + ) + + assert response.status_code == 200 + mock_cache_set.assert_called_once() + cache_key = mock_cache_set.call_args.args[0] + assert cache_key.startswith('qt:resolve:lot_id:') + assert mock_cache_set.call_args.kwargs['ttl'] == 60 class TestLotHistoryEndpoint: @@ -267,7 +325,7 @@ class TestAdjacentLotsEndpoint: assert '2024-01-15' in call_args[0][1] # target_time -class TestLotAssociationsEndpoint: +class TestLotAssociationsEndpoint: """Tests for /api/query-tool/lot-associations endpoint.""" def test_missing_container_id(self, client): @@ -294,8 +352,8 @@ class TestLotAssociationsEndpoint: assert 'error' in data assert '不支援' in data['error'] or 'type' in data['error'].lower() - @patch('mes_dashboard.routes.query_tool_routes.get_lot_materials') - def test_lot_materials_success(self, mock_query, client): + @patch('mes_dashboard.routes.query_tool_routes.get_lot_materials') + def test_lot_materials_success(self, mock_query, client): """Should return lot materials on success.""" mock_query.return_value = { 'data': [ @@ -313,8 +371,137 @@ class TestLotAssociationsEndpoint: ) assert response.status_code == 200 data = json.loads(response.data) - assert 'data' in data - assert data['total'] == 1 + assert 'data' in data + assert data['total'] == 1 + + @patch('mes_dashboard.routes.query_tool_routes.get_lot_splits') + def test_lot_splits_default_fast_mode(self, mock_query, client): + mock_query.return_value = {'data': [], 'total': 0} + + response = client.get( + '/api/query-tool/lot-associations?container_id=488103800029578b&type=splits' + ) + + assert response.status_code == 200 + mock_query.assert_called_once_with('488103800029578b', full_history=False) + + @patch('mes_dashboard.routes.query_tool_routes.get_lot_splits') + def test_lot_splits_full_history_mode(self, mock_query, client): + mock_query.return_value = {'data': [], 'total': 0} + + response = client.get( + '/api/query-tool/lot-associations?' + 'container_id=488103800029578b&type=splits&full_history=true' + ) + + assert response.status_code == 200 + mock_query.assert_called_once_with('488103800029578b', full_history=True) + + +class TestQueryToolRateLimit: + """Rate-limit behavior for high-cost query-tool endpoints.""" + + @patch('mes_dashboard.routes.query_tool_routes.resolve_lots') + @patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 5)) + def test_resolve_rate_limited_returns_429(self, _mock_limit, mock_resolve, client): + response = client.post( + '/api/query-tool/resolve', + json={'input_type': 'lot_id', 'values': ['GA23100020-A00-001']}, + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '5' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + mock_resolve.assert_not_called() + + @patch('mes_dashboard.routes.query_tool_routes.get_lot_history') + @patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 6)) + def test_lot_history_rate_limited_returns_429(self, _mock_limit, mock_history, client): + response = client.get('/api/query-tool/lot-history?container_id=488103800029578b') + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '6' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + mock_history.assert_not_called() + + @patch('mes_dashboard.routes.query_tool_routes.get_lot_materials') + @patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 7)) + def test_lot_association_rate_limited_returns_429( + self, + _mock_limit, + mock_materials, + client, + ): + response = client.get( + '/api/query-tool/lot-associations?container_id=488103800029578b&type=materials' + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '7' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + mock_materials.assert_not_called() + + @patch('mes_dashboard.routes.query_tool_routes.get_adjacent_lots') + @patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 8)) + def test_adjacent_lots_rate_limited_returns_429( + self, + _mock_limit, + mock_adjacent, + client, + ): + response = client.get( + '/api/query-tool/adjacent-lots?equipment_id=EQ001&target_time=2024-01-15T10:30:00' + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '8' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + mock_adjacent.assert_not_called() + + @patch('mes_dashboard.routes.query_tool_routes.get_equipment_status_hours') + @patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 9)) + def test_equipment_period_rate_limited_returns_429( + self, + _mock_limit, + mock_equipment, + client, + ): + response = client.post( + '/api/query-tool/equipment-period', + json={ + 'equipment_ids': ['EQ001'], + 'start_date': '2024-01-01', + 'end_date': '2024-01-31', + 'query_type': 'status_hours', + }, + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '9' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + mock_equipment.assert_not_called() + + @patch('mes_dashboard.routes.query_tool_routes.get_lot_history') + @patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 10)) + def test_export_rate_limited_returns_429(self, _mock_limit, mock_history, client): + response = client.post( + '/api/query-tool/export-csv', + json={ + 'export_type': 'lot_history', + 'params': {'container_id': '488103800029578b'}, + }, + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '10' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + mock_history.assert_not_called() class TestEquipmentPeriodEndpoint: diff --git a/tests/test_query_tool_service.py b/tests/test_query_tool_service.py index 6511997..4c83dc6 100644 --- a/tests/test_query_tool_service.py +++ b/tests/test_query_tool_service.py @@ -8,15 +8,17 @@ Tests the core service functions without database dependencies: """ import pytest -from mes_dashboard.services.query_tool_service import ( - validate_date_range, - validate_lot_input, - validate_equipment_input, - _build_in_clause, - _build_in_filter, - BATCH_SIZE, - MAX_LOT_IDS, - MAX_SERIAL_NUMBERS, +from mes_dashboard.services.query_tool_service import ( + validate_date_range, + validate_lot_input, + validate_equipment_input, + _resolve_by_lot_id, + _resolve_by_serial_number, + _resolve_by_work_order, + get_lot_split_merge_history, + BATCH_SIZE, + MAX_LOT_IDS, + MAX_SERIAL_NUMBERS, MAX_WORK_ORDERS, MAX_EQUIPMENTS, MAX_DATE_RANGE_DAYS, @@ -184,86 +186,124 @@ class TestValidateEquipmentInput: assert result is None -class TestBuildInClause: - """Tests for _build_in_clause function.""" - - def test_empty_list(self): - """Should return empty list for empty input.""" - result = _build_in_clause([]) - assert result == [] - - def test_single_value(self): - """Should return single chunk for single value.""" - result = _build_in_clause(['VAL001']) - assert len(result) == 1 - assert result[0] == "'VAL001'" - - def test_multiple_values(self): - """Should join multiple values with comma.""" - result = _build_in_clause(['VAL001', 'VAL002', 'VAL003']) - assert len(result) == 1 - assert "'VAL001'" in result[0] - assert "'VAL002'" in result[0] - assert "'VAL003'" in result[0] - assert result[0] == "'VAL001', 'VAL002', 'VAL003'" - - def test_chunking(self): - """Should chunk when exceeding batch size.""" - # Create more than BATCH_SIZE values - values = [f'VAL{i:06d}' for i in range(BATCH_SIZE + 10)] - result = _build_in_clause(values) - assert len(result) == 2 - # First chunk should have BATCH_SIZE items - assert result[0].count("'") == BATCH_SIZE * 2 # 2 quotes per value - - def test_escape_single_quotes(self): - """Should escape single quotes in values.""" - result = _build_in_clause(["VAL'001"]) - assert len(result) == 1 - assert "VAL''001" in result[0] # Escaped - - def test_custom_chunk_size(self): - """Should respect custom chunk size.""" - values = ['V1', 'V2', 'V3', 'V4', 'V5'] - result = _build_in_clause(values, max_chunk_size=2) - assert len(result) == 3 # 2+2+1 - - -class TestBuildInFilter: - """Tests for _build_in_filter function.""" - - def test_empty_list(self): - """Should return 1=0 for empty input (no results).""" - result = _build_in_filter([], 'COL') - assert result == "1=0" - - def test_single_value(self): - """Should build simple IN clause for single value.""" - result = _build_in_filter(['VAL001'], 'COL') - assert "COL IN" in result - assert "'VAL001'" in result - - def test_multiple_values(self): - """Should build IN clause with multiple values.""" - result = _build_in_filter(['VAL001', 'VAL002'], 'COL') - assert "COL IN" in result - assert "'VAL001'" in result - assert "'VAL002'" in result - - def test_custom_column(self): - """Should use custom column name.""" - result = _build_in_filter(['VAL001'], 't.MYCOL') - assert "t.MYCOL IN" in result - - def test_large_list_uses_or(self): - """Should use OR for chunked results.""" - # Create more than BATCH_SIZE values - values = [f'VAL{i:06d}' for i in range(BATCH_SIZE + 10)] - result = _build_in_filter(values, 'COL') - assert " OR " in result - # Should have parentheses wrapping the OR conditions - assert result.startswith("(") - assert result.endswith(")") +class TestResolveQueriesUseBindParams: + """Queries with user input should always use bind params.""" + + def test_resolve_by_lot_id_uses_query_builder_params(self): + from unittest.mock import patch + import pandas as pd + + with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load: + with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read: + mock_load.return_value = "SELECT * FROM DUAL" + mock_read.return_value = pd.DataFrame([ + { + 'CONTAINERID': 'CID-1', + 'CONTAINERNAME': 'LOT-1', + 'SPECNAME': 'SPEC-1', + 'QTY': 100, + } + ]) + + result = _resolve_by_lot_id(['LOT-1']) + + assert result['total'] == 1 + mock_load.assert_called_once() + sql_params = mock_load.call_args.kwargs + assert 'CONTAINER_FILTER' in sql_params + assert ':p0' in sql_params['CONTAINER_FILTER'] + _, query_params = mock_read.call_args.args + assert query_params == {'p0': 'LOT-1'} + + def test_resolve_by_serial_number_uses_query_builder_params(self): + from unittest.mock import patch + import pandas as pd + + with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load: + with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read: + mock_load.return_value = "SELECT * FROM DUAL" + mock_read.return_value = pd.DataFrame([ + { + 'CONTAINERID': 'CID-1', + 'FINISHEDNAME': 'SN-1', + 'CONTAINERNAME': 'LOT-1', + 'SPECNAME': 'SPEC-1', + } + ]) + + result = _resolve_by_serial_number(['SN-1']) + + assert result['total'] == 1 + sql_params = mock_load.call_args.kwargs + assert ':p0' in sql_params['SERIAL_FILTER'] + _, query_params = mock_read.call_args.args + assert query_params == {'p0': 'SN-1'} + + def test_resolve_by_work_order_uses_query_builder_params(self): + from unittest.mock import patch + import pandas as pd + + with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load: + with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read: + mock_load.return_value = "SELECT * FROM DUAL" + mock_read.return_value = pd.DataFrame([ + { + 'CONTAINERID': 'CID-1', + 'PJ_WORKORDER': 'WO-1', + 'CONTAINERNAME': 'LOT-1', + 'SPECNAME': 'SPEC-1', + } + ]) + + result = _resolve_by_work_order(['WO-1']) + + assert result['total'] == 1 + sql_params = mock_load.call_args.kwargs + assert ':p0' in sql_params['WORK_ORDER_FILTER'] + _, query_params = mock_read.call_args.args + assert query_params == {'p0': 'WO-1'} + + +class TestSplitMergeHistoryMode: + """Fast mode should use read_sql_df, full mode should use read_sql_df_slow.""" + + def test_fast_mode_uses_time_window_and_row_limit(self): + from unittest.mock import patch + import pandas as pd + + with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load: + with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_fast: + with patch('mes_dashboard.services.query_tool_service.read_sql_df_slow') as mock_slow: + mock_load.return_value = "SELECT * FROM DUAL" + mock_fast.return_value = pd.DataFrame([]) + + result = get_lot_split_merge_history('WO-1', full_history=False) + + assert result['mode'] == 'fast' + kwargs = mock_load.call_args.kwargs + assert "ADD_MONTHS(SYSDATE, -6)" in kwargs['TIME_WINDOW'] + assert "FETCH FIRST 500 ROWS ONLY" == kwargs['ROW_LIMIT'] + mock_fast.assert_called_once() + mock_slow.assert_not_called() + + def test_full_mode_uses_slow_query_without_limits(self): + from unittest.mock import patch + import pandas as pd + + with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load: + with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_fast: + with patch('mes_dashboard.services.query_tool_service.read_sql_df_slow') as mock_slow: + mock_load.return_value = "SELECT * FROM DUAL" + mock_slow.return_value = pd.DataFrame([]) + + result = get_lot_split_merge_history('WO-1', full_history=True) + + assert result['mode'] == 'full' + kwargs = mock_load.call_args.kwargs + assert kwargs['TIME_WINDOW'] == '' + assert kwargs['ROW_LIMIT'] == '' + mock_fast.assert_not_called() + mock_slow.assert_called_once() class TestServiceConstants: @@ -323,98 +363,78 @@ class TestGetWorkcenterForGroups: assert result == [] -class TestGetLotHistoryWithWorkcenterFilter: - """Tests for get_lot_history with workcenter_groups filter.""" - - def test_no_filter_returns_all(self): - """When no workcenter_groups, should not add filter to SQL.""" - from unittest.mock import patch, MagicMock - import pandas as pd - - with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read: - with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader: - from mes_dashboard.services.query_tool_service import get_lot_history - - mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}' - mock_read.return_value = pd.DataFrame({ - 'CONTAINERID': ['abc123'], - 'WORKCENTERNAME': ['DB_1'], - }) - - result = get_lot_history('abc123', workcenter_groups=None) - - assert 'error' not in result - assert result['filtered_by_groups'] == [] - # Verify SQL does not contain WORKCENTERNAME IN - sql_called = mock_read.call_args[0][0] - assert 'WORKCENTERNAME IN' not in sql_called - assert '{{ WORKCENTER_FILTER }}' not in sql_called - - def test_with_filter_adds_condition(self): - """When workcenter_groups provided, should filter by workcenters.""" - from unittest.mock import patch - import pandas as pd - - with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read: - with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader: - with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc: - from mes_dashboard.services.query_tool_service import get_lot_history - - mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}' - mock_get_wc.return_value = ['DB_1', 'DB_2'] - mock_read.return_value = pd.DataFrame({ - 'CONTAINERID': ['abc123'], - 'WORKCENTERNAME': ['DB_1'], - }) - - result = get_lot_history('abc123', workcenter_groups=['DB']) - - mock_get_wc.assert_called_once_with(['DB']) - assert result['filtered_by_groups'] == ['DB'] - # Verify SQL contains filter - sql_called = mock_read.call_args[0][0] - assert 'WORKCENTERNAME' in sql_called - - def test_empty_groups_list_no_filter(self): - """Empty groups list should return all (no filter).""" - from unittest.mock import patch - import pandas as pd - - with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read: - with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader: - from mes_dashboard.services.query_tool_service import get_lot_history - - mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}' - mock_read.return_value = pd.DataFrame({ - 'CONTAINERID': ['abc123'], - 'WORKCENTERNAME': ['DB_1'], - }) - - result = get_lot_history('abc123', workcenter_groups=[]) - - assert result['filtered_by_groups'] == [] - # Verify SQL does not contain WORKCENTERNAME IN - sql_called = mock_read.call_args[0][0] - assert 'WORKCENTERNAME IN' not in sql_called - - def test_filter_with_empty_workcenters_result(self): - """When group has no workcenters, should not add filter.""" - from unittest.mock import patch - import pandas as pd - - with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read: - with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader: - with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc: - from mes_dashboard.services.query_tool_service import get_lot_history - - mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}' - mock_get_wc.return_value = [] # No workcenters for this group - mock_read.return_value = pd.DataFrame({ - 'CONTAINERID': ['abc123'], - 'WORKCENTERNAME': ['DB_1'], - }) - - result = get_lot_history('abc123', workcenter_groups=['UNKNOWN']) - - # Should still succeed, just no filter applied - assert 'error' not in result +class TestGetLotHistoryWithWorkcenterFilter: + """Tests for get_lot_history with workcenter_groups filter.""" + + def test_no_filter_returns_all(self): + """When no workcenter_groups, should not add filter to SQL.""" + from unittest.mock import patch + from mes_dashboard.services.query_tool_service import get_lot_history + + with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch: + mock_fetch.return_value = { + 'abc123': [ + {'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'}, + {'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'WB_1'}, + ] + } + + result = get_lot_history('abc123', workcenter_groups=None) + + assert 'error' not in result + assert result['filtered_by_groups'] == [] + assert result['total'] == 2 + + def test_with_filter_adds_condition(self): + """When workcenter_groups provided, should filter by workcenters.""" + from unittest.mock import patch + from mes_dashboard.services.query_tool_service import get_lot_history + + with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch: + with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc: + mock_fetch.return_value = { + 'abc123': [ + {'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'}, + {'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'WB_1'}, + ] + } + mock_get_wc.return_value = ['DB_1'] + + result = get_lot_history('abc123', workcenter_groups=['DB']) + + mock_get_wc.assert_called_once_with(['DB']) + assert result['filtered_by_groups'] == ['DB'] + assert result['total'] == 1 + assert result['data'][0]['WORKCENTERNAME'] == 'DB_1' + + def test_empty_groups_list_no_filter(self): + """Empty groups list should return all (no filter).""" + from unittest.mock import patch + from mes_dashboard.services.query_tool_service import get_lot_history + + with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch: + mock_fetch.return_value = { + 'abc123': [{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'}] + } + + result = get_lot_history('abc123', workcenter_groups=[]) + + assert result['filtered_by_groups'] == [] + assert result['total'] == 1 + + def test_filter_with_empty_workcenters_result(self): + """When group has no workcenters, should not add filter.""" + from unittest.mock import patch + from mes_dashboard.services.query_tool_service import get_lot_history + + with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch: + with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc: + mock_fetch.return_value = { + 'abc123': [{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'}] + } + mock_get_wc.return_value = [] + + result = get_lot_history('abc123', workcenter_groups=['UNKNOWN']) + + assert 'error' not in result + assert result['total'] == 1 diff --git a/tests/test_trace_routes.py b/tests/test_trace_routes.py new file mode 100644 index 0000000..02f8e1d --- /dev/null +++ b/tests/test_trace_routes.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- +"""Route tests for staged trace API endpoints.""" + +from __future__ import annotations + +from unittest.mock import patch + +import mes_dashboard.core.database as db +from mes_dashboard.app import create_app +from mes_dashboard.core.cache import NoOpCache +from mes_dashboard.core.rate_limit import reset_rate_limits_for_tests + + +def _client(): + db._ENGINE = None + app = create_app('testing') + app.config['TESTING'] = True + app.extensions["cache"] = NoOpCache() + return app.test_client() + + +def setup_function(): + reset_rate_limits_for_tests() + + +def teardown_function(): + reset_rate_limits_for_tests() + + +@patch('mes_dashboard.routes.trace_routes.resolve_lots') +def test_seed_resolve_query_tool_success(mock_resolve_lots): + mock_resolve_lots.return_value = { + 'data': [ + { + 'container_id': 'CID-001', + 'lot_id': 'LOT-001', + } + ] + } + + client = _client() + response = client.post( + '/api/trace/seed-resolve', + json={ + 'profile': 'query_tool', + 'params': { + 'resolve_type': 'lot_id', + 'values': ['LOT-001'], + }, + }, + ) + + assert response.status_code == 200 + payload = response.get_json() + assert payload['stage'] == 'seed-resolve' + assert payload['seed_count'] == 1 + assert payload['seeds'][0]['container_id'] == 'CID-001' + assert payload['seeds'][0]['container_name'] == 'LOT-001' + assert payload['cache_key'].startswith('trace:seed:query_tool:') + + +def test_seed_resolve_invalid_profile_returns_400(): + client = _client() + response = client.post( + '/api/trace/seed-resolve', + json={ + 'profile': 'invalid', + 'params': {}, + }, + ) + + assert response.status_code == 400 + payload = response.get_json() + assert payload['error']['code'] == 'INVALID_PROFILE' + + +@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 8)) +def test_seed_resolve_rate_limited_returns_429(_mock_rate_limit): + client = _client() + response = client.post( + '/api/trace/seed-resolve', + json={ + 'profile': 'query_tool', + 'params': {'resolve_type': 'lot_id', 'values': ['LOT-001']}, + }, + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '8' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + + +@patch('mes_dashboard.routes.trace_routes.LineageEngine.resolve_full_genealogy') +def test_lineage_success_returns_snake_case(mock_resolve_genealogy): + mock_resolve_genealogy.return_value = { + 'CID-001': {'CID-A', 'CID-B'} + } + + client = _client() + response = client.post( + '/api/trace/lineage', + json={ + 'profile': 'query_tool', + 'container_ids': ['CID-001'], + }, + ) + + assert response.status_code == 200 + payload = response.get_json() + assert payload['stage'] == 'lineage' + assert sorted(payload['ancestors']['CID-001']) == ['CID-A', 'CID-B'] + assert payload['total_nodes'] == 3 + assert 'totalNodes' not in payload + + +@patch( + 'mes_dashboard.routes.trace_routes.LineageEngine.resolve_full_genealogy', + side_effect=TimeoutError('lineage timed out'), +) +def test_lineage_timeout_returns_504(_mock_resolve_genealogy): + client = _client() + response = client.post( + '/api/trace/lineage', + json={ + 'profile': 'query_tool', + 'container_ids': ['CID-001'], + }, + ) + + assert response.status_code == 504 + payload = response.get_json() + assert payload['error']['code'] == 'LINEAGE_TIMEOUT' + + +@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 6)) +def test_lineage_rate_limited_returns_429(_mock_rate_limit): + client = _client() + response = client.post( + '/api/trace/lineage', + json={ + 'profile': 'query_tool', + 'container_ids': ['CID-001'], + }, + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '6' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS' + + +@patch('mes_dashboard.routes.trace_routes.EventFetcher.fetch_events') +def test_events_partial_failure_returns_200_with_code(mock_fetch_events): + def _side_effect(_container_ids, domain): + if domain == 'history': + return { + 'CID-001': [{'CONTAINERID': 'CID-001', 'EVENTTYPE': 'TRACK_IN'}] + } + raise RuntimeError('domain failed') + + mock_fetch_events.side_effect = _side_effect + + client = _client() + response = client.post( + '/api/trace/events', + json={ + 'profile': 'query_tool', + 'container_ids': ['CID-001'], + 'domains': ['history', 'materials'], + }, + ) + + assert response.status_code == 200 + payload = response.get_json() + assert payload['stage'] == 'events' + assert payload['code'] == 'EVENTS_PARTIAL_FAILURE' + assert 'materials' in payload['failed_domains'] + assert payload['results']['history']['count'] == 1 + + +@patch('mes_dashboard.routes.trace_routes.build_trace_aggregation_from_events') +@patch('mes_dashboard.routes.trace_routes.EventFetcher.fetch_events') +def test_events_mid_section_defect_with_aggregation( + mock_fetch_events, + mock_build_aggregation, +): + mock_fetch_events.return_value = { + 'CID-001': [ + { + 'CONTAINERID': 'CID-001', + 'WORKCENTER_GROUP': '測試', + 'EQUIPMENTID': 'EQ-01', + 'EQUIPMENTNAME': 'EQ-01', + } + ] + } + mock_build_aggregation.return_value = { + 'kpi': {'total_input': 100}, + 'charts': {'by_station': []}, + 'daily_trend': [], + 'available_loss_reasons': [], + 'genealogy_status': 'ready', + 'detail_total_count': 0, + } + + client = _client() + response = client.post( + '/api/trace/events', + json={ + 'profile': 'mid_section_defect', + 'container_ids': ['CID-001'], + 'domains': ['upstream_history'], + 'params': { + 'start_date': '2025-01-01', + 'end_date': '2025-01-31', + }, + 'lineage': {'ancestors': {'CID-001': ['CID-A']}}, + 'seed_container_ids': ['CID-001'], + }, + ) + + assert response.status_code == 200 + payload = response.get_json() + assert payload['aggregation']['kpi']['total_input'] == 100 + assert payload['aggregation']['genealogy_status'] == 'ready' + mock_build_aggregation.assert_called_once() + + +@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 5)) +def test_events_rate_limited_returns_429(_mock_rate_limit): + client = _client() + response = client.post( + '/api/trace/events', + json={ + 'profile': 'query_tool', + 'container_ids': ['CID-001'], + 'domains': ['history'], + }, + ) + + assert response.status_code == 429 + assert response.headers.get('Retry-After') == '5' + payload = response.get_json() + assert payload['error']['code'] == 'TOO_MANY_REQUESTS'