egg/DashBoard
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(docs): add OOM cross-apply analysis and agent work plan, clean up obsolete artifacts

Browse Source 
Add OOM/cache cross-tool gap analysis and structured agent work plan for
Material Trace, Query Tool, and Mid-Section Defect hardening. Remove
obsolete migration blueprint and portal-no-iframe baseline artifacts.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
egg
2026-03-04 19:23:31 +08:00
parent 5517f7e85c
commit 981ae5614e
18 changed files with 516 additions and 1080 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
docs/migration/full-modernization-architecture-blueprint/asset_readiness_manifest.json
View File

@@ -1,20 +0,0 @@
{
"in_scope_required_assets": {
"/wip-overview": ["wip-overview.js"],
"/wip-detail": ["wip-detail.js"],
"/hold-overview": ["hold-overview.js"],
"/hold-detail": ["hold-detail.js"],
"/hold-history": ["hold-history.js"],
"/reject-history": ["reject-history.js"],
"/resource": ["resource-status.js"],
"/resource-history": ["resource-history.js"],
"/qc-gate": ["qc-gate.js"],
"/job-query": ["job-query.js"],
"/admin/performance": ["admin-performance.js"],
"/tables": ["tables.js"],
"/excel-query": ["excel-query.js"],
"/query-tool": ["query-tool.js"],
"/mid-section-defect": ["mid-section-defect.js"]
},
"deferred_routes": []
}

3
docs/migration/full-modernization-architecture-blueprint/bug_revalidation_records.json
View File

@@ -1,3 +0,0 @@
{
"records": []
}

116
docs/migration/full-modernization-architecture-blueprint/exception_registry.json
View File

@@ -1,116 +0,0 @@
{
"entries": [
{
"id": "style-legacy-wip-overview",
"scope": "/wip-overview",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-wip-detail",
"scope": "/wip-detail",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-hold-overview",
"scope": "/hold-overview",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-hold-detail",
"scope": "/hold-detail",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-hold-history",
"scope": "/hold-history",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-reject-history",
"scope": "/reject-history",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-resource",
"scope": "/resource",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-resource-history",
"scope": "/resource-history",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-qc-gate",
"scope": "/qc-gate",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-job-query",
"scope": "/job-query",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-admin-pages",
"scope": "/admin/pages",
"owner": "frontend-platform-admin",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-admin-performance",
"scope": "/admin/performance",
"owner": "frontend-platform-admin",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-tables",
"scope": "/tables",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-excel-query",
"scope": "/excel-query",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-query-tool",
"scope": "/query-tool",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
},
{
"id": "style-legacy-mid-section-defect",
"scope": "/mid-section-defect",
"owner": "frontend-mes-reporting",
"milestone": "full-modernization-phase2",
"reason": "Legacy styles pending full token and scope migration"
}
]
}

20
docs/migration/full-modernization-architecture-blueprint/known_bug_baseline.json
View File

@@ -1,20 +0,0 @@
{
"routes": {
"/wip-overview": { "known_issues": [] },
"/wip-detail": { "known_issues": [] },
"/hold-overview": { "known_issues": [] },
"/hold-detail": { "known_issues": [] },
"/hold-history": { "known_issues": [] },
"/reject-history": { "known_issues": [] },
"/resource": { "known_issues": [] },
"/resource-history": { "known_issues": [] },
"/qc-gate": { "known_issues": [] },
"/job-query": { "known_issues": [] },
"/tables": { "known_issues": [] },
"/excel-query": { "known_issues": [] },
"/query-tool": { "known_issues": [] },
"/mid-section-defect": { "known_issues": [] },
"/admin/pages": { "known_issues": [] },
"/admin/performance": { "known_issues": [] }
}
}

3
docs/migration/full-modernization-architecture-blueprint/manual_acceptance_records.json
View File

@@ -1,3 +0,0 @@
{
"records": []
}

6
docs/migration/full-modernization-architecture-blueprint/quality_gate_policy.json
View File

@@ -1,6 +0,0 @@
{
"severity_mode": {
"current": "block"
},
"deferred_routes_excluded": []
}

9
docs/migration/full-modernization-architecture-blueprint/quality_gate_report.json
View File

@@ -1,9 +0,0 @@
{
"mode": "block",
"errors": [],
"warnings": [
"/excel-query uses shell tokens without fallback ['--portal-shadow-panel'] in frontend/src/excel-query/style.css with approved exception"
],
"info": [],
"passed": true
}

208
docs/migration/full-modernization-architecture-blueprint/route_contracts.json
View File

@@ -1,208 +0,0 @@
{
"routes": [
{
"route": "/wip-overview",
"route_id": "wip-overview",
"title": "WIP Overview",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/wip-overview",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/wip-detail",
"route_id": "wip-detail",
"title": "WIP Detail",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/wip-detail",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/hold-overview",
"route_id": "hold-overview",
"title": "Hold Overview",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/hold-overview",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/hold-detail",
"route_id": "hold-detail",
"title": "Hold Detail",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/hold-detail",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/hold-history",
"route_id": "hold-history",
"title": "Hold History",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/hold-history",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/reject-history",
"route_id": "reject-history",
"title": "Reject History",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/reject-history",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/resource",
"route_id": "resource",
"title": "Resource",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/resource",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/resource-history",
"route_id": "resource-history",
"title": "Resource History",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/resource-history",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/qc-gate",
"route_id": "qc-gate",
"title": "QC Gate",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/qc-gate",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/job-query",
"route_id": "job-query",
"title": "Job Query",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/job-query",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/admin/pages",
"route_id": "admin-pages",
"title": "Admin Pages",
"scope": "in-scope",
"render_mode": "external",
"owner": "frontend-platform-admin",
"visibility_policy": "admin_only",
"canonical_shell_path": "/portal-shell/admin/pages",
"rollback_strategy": "external_route_reversion",
"compatibility_policy": "external_target_redirect"
},
{
"route": "/admin/performance",
"route_id": "admin-performance",
"title": "Admin Performance",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-platform-admin",
"visibility_policy": "admin_only",
"canonical_shell_path": "/portal-shell/admin/performance",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/tables",
"route_id": "tables",
"title": "Tables",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/tables",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/excel-query",
"route_id": "excel-query",
"title": "Excel Query",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/excel-query",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/query-tool",
"route_id": "query-tool",
"title": "Query Tool",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/query-tool",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/mid-section-defect",
"route_id": "mid-section-defect",
"title": "Mid Section Defect",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/mid-section-defect",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
},
{
"route": "/material-trace",
"route_id": "material-trace",
"title": "Material Trace",
"scope": "in-scope",
"render_mode": "native",
"owner": "frontend-mes-reporting",
"visibility_policy": "released_or_admin",
"canonical_shell_path": "/portal-shell/material-trace",
"rollback_strategy": "fallback_to_legacy_route",
"compatibility_policy": "redirect_to_shell_when_spa_enabled"
}
]
}

21
docs/migration/full-modernization-architecture-blueprint/route_scope_matrix.json
View File

@@ -1,21 +0,0 @@
{
"in_scope": [
{ "route": "/wip-overview", "category": "report" },
{ "route": "/wip-detail", "category": "report" },
{ "route": "/hold-overview", "category": "report" },
{ "route": "/hold-detail", "category": "report" },
{ "route": "/hold-history", "category": "report" },
{ "route": "/reject-history", "category": "report" },
{ "route": "/resource", "category": "report" },
{ "route": "/resource-history", "category": "report" },
{ "route": "/qc-gate", "category": "report" },
{ "route": "/job-query", "category": "report" },
{ "route": "/tables", "category": "report" },
{ "route": "/excel-query", "category": "report" },
{ "route": "/query-tool", "category": "report" },
{ "route": "/mid-section-defect", "category": "report" },
{ "route": "/admin/pages", "category": "admin" },
{ "route": "/admin/performance", "category": "admin" }
],
"deferred": []
}

4
docs/migration/full-modernization-architecture-blueprint/style_inventory.json
View File

@@ -1,4 +0,0 @@
{
"routes": {},
"notes": "Baseline placeholder inventory"
}

46
docs/migration/portal-no-iframe/baseline_api_payload_contracts.json
View File

@@ -1,46 +0,0 @@
{
"source": "current frontend API consumption contracts",
"apis": {
"/api/wip/overview/summary": {
"required_keys": [
"dataUpdateDate",
"runLots",
"queueLots",
"holdLots"
],
"notes": "summary header and cards depend on these fields"
},
"/api/wip/overview/matrix": {
"required_keys": [
"workcenters",
"packages",
"matrix",
"workcenter_totals"
],
"notes": "matrix table rendering contract"
},
"/api/wip/hold-detail/summary": {
"required_keys": [
"workcenterCount",
"packageCount",
"lotCount"
],
"notes": "hold detail summary cards contract"
},
"/api/resource/history/summary": {
"required_keys": [
"kpi",
"trend",
"heatmap",
"workcenter_comparison"
],
"notes": "resource history chart summary contract"
},
"/api/resource/history/detail": {
"required_keys": [
"data"
],
"notes": "detail table contract (plus truncated/max_records metadata when present)"
}
}
}

4
docs/migration/portal-no-iframe/baseline_drawer_contract_validation.json
View File

@@ -1,4 +0,0 @@
{
"source": "data/page_status.json",
"errors": []
}

207
docs/migration/portal-no-iframe/baseline_drawer_visibility.json
View File

@@ -1,207 +0,0 @@
{
"source": "data/page_status.json",
"admin": [
{
"id": "reports",
"name": "即時報表",
"order": 1,
"admin_only": false,
"pages": [
{
"route": "/wip-overview",
"name": "WIP 即時概況",
"status": "released",
"order": 1
},
{
"route": "/hold-overview",
"name": "Hold 即時概況",
"status": "released",
"order": 2
},
{
"route": "/resource",
"name": "設備即時概況",
"status": "released",
"order": 4
},
{
"route": "/qc-gate",
"name": "QC-GATE 狀態",
"status": "released",
"order": 6
}
]
},
{
"id": "drawer-2",
"name": "歷史報表",
"order": 2,
"admin_only": false,
"pages": [
{
"route": "/hold-history",
"name": "Hold 歷史績效",
"status": "released",
"order": 3
},
{
"route": "/reject-history",
"name": "報廢歷史查詢",
"status": "released",
"order": 4
},
{
"route": "/resource-history",
"name": "設備歷史績效",
"status": "released",
"order": 5
}
]
},
{
"id": "drawer",
"name": "查詢工具",
"order": 3,
"admin_only": false,
"pages": [
{
"route": "/job-query",
"name": "設備維修查詢",
"status": "released",
"order": 1
},
{
"route": "/query-tool",
"name": "批次追蹤工具",
"status": "released",
"order": 2
},
{
"route": "/mid-section-defect",
"name": "製程不良追溯分析",
"status": "released",
"order": 3
}
]
},
{
"id": "dev-tools",
"name": "開發工具",
"order": 4,
"admin_only": true,
"pages": [
{
"route": "/tables",
"name": "表格總覽",
"status": "dev",
"order": 1
},
{
"route": "/admin/pages",
"name": "頁面管理",
"status": "released",
"order": 1
},
{
"route": "/excel-query",
"name": "Excel 批次查詢",
"status": "dev",
"order": 2
},
{
"route": "/admin/performance",
"name": "效能監控",
"status": "dev",
"order": 2
}
]
}
],
"non_admin": [
{
"id": "reports",
"name": "即時報表",
"order": 1,
"admin_only": false,
"pages": [
{
"route": "/wip-overview",
"name": "WIP 即時概況",
"status": "released",
"order": 1
},
{
"route": "/hold-overview",
"name": "Hold 即時概況",
"status": "released",
"order": 2
},
{
"route": "/resource",
"name": "設備即時概況",
"status": "released",
"order": 4
},
{
"route": "/qc-gate",
"name": "QC-GATE 狀態",
"status": "released",
"order": 6
}
]
},
{
"id": "drawer-2",
"name": "歷史報表",
"order": 2,
"admin_only": false,
"pages": [
{
"route": "/hold-history",
"name": "Hold 歷史績效",
"status": "released",
"order": 3
},
{
"route": "/reject-history",
"name": "報廢歷史查詢",
"status": "released",
"order": 4
},
{
"route": "/resource-history",
"name": "設備歷史績效",
"status": "released",
"order": 5
}
]
},
{
"id": "drawer",
"name": "查詢工具",
"order": 3,
"admin_only": false,
"pages": [
{
"route": "/job-query",
"name": "設備維修查詢",
"status": "released",
"order": 1
},
{
"route": "/query-tool",
"name": "批次追蹤工具",
"status": "released",
"order": 2
},
{
"route": "/mid-section-defect",
"name": "製程不良追溯分析",
"status": "released",
"order": 3
}
]
}
]
}

46
docs/migration/portal-no-iframe/baseline_route_query_contracts.json
View File

@@ -1,46 +0,0 @@
{
"source": "frontend route parsing and current parity matrix",
"routes": {
"/wip-overview": {
"query_keys": [
"workorder",
"lotid",
"package",
"type",
"status"
],
"notes": "filters + status URL state must remain compatible"
},
"/wip-detail": {
"query_keys": [
"workcenter",
"workorder",
"lotid",
"package",
"type",
"status"
],
"notes": "workcenter deep-link and back-link query continuity"
},
"/hold-detail": {
"query_keys": [
"reason"
],
"notes": "reason required for normal access flow"
},
"/resource-history": {
"query_keys": [
"start_date",
"end_date",
"granularity",
"workcenter_groups",
"families",
"resource_ids",
"is_production",
"is_key",
"is_monitor"
],
"notes": "query/export params must remain compatible"
}
}
}

273
docs/oom_agent_work_plan.md Normal file
View File

@@ -0,0 +1,273 @@
# OOM 防護跨工具改善 — Agent 工作架構
> 基於 `oom_cache_cross_apply_analysis.md` 痛點分析 + 4 個研究 agent 的代碼深度掃描結果
---
## Agent 架構總覽
```
┌─────────────────────┐
│ agent-shared (P0) │ ← 必須先完成
│ 共用記憶體守衛抽離 │
└─────────┬───────────┘
│
┌───────────────┼───────────────┐
▼ ▼ ▼
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
│ agent-mt (P1) │ │ agent-qt (P1) │ │ agent-md (P1) │ ← 可平行
│ Material Trace│ │ Query Tool │ │ Mid-Defect │
└───────┬───────┘ └───────┬───────┘ └───────┬───────┘
│ │ │
└─────────────────┼─────────────────┘
▼
┌─────────────────────┐
│ agent-qa (P2) │ ← 最後驗證
│ 測試 + 壓力測試 │
└─────────────────────┘
```
---
## Agent 0: agent-shared — 共用記憶體守衛抽離
### 目標
將 `reject_dataset_cache.py` 的私有 `_enforce_interactive_memory_guard()` 抽離為 `core/` 層可重用元件。
### 發現
- `_enforce_interactive_memory_guard` 目前是 `reject_dataset_cache.py` 的 **module-private 函式**
- 依賴 `_df_memory_mb()` 和 `_process_rss_mb()` 兩個 helper（也是同檔案 private）
- `_process_rss_mb()` 與 `worker_memory_guard._current_rss_mb()` **完全重複**
- 4 個常數全部硬編碼為 reject-history 專用（`REJECT_DERIVE_*`）
### 任務清單
| # | 任務 | 改動檔案 | 行數估計 |
|---|------|---------|---------|
| S1 | 建立 `core/interactive_memory_guard.py` | 新建 | ~60 行 |
| S2 | 抽離 `df_memory_mb()` + `process_rss_mb()` 為 public | 新檔案 | 含在 S1 |
| S3 | 主函式 `enforce_dataset_memory_guard()` — 參數化閾值（帶預設值） | 新檔案 | 含在 S1 |
| S4 | `reject_dataset_cache.py` 改為 import 新元件 | 修改 | ~15 行 |
| S5 | 合併 `worker_memory_guard._current_rss_mb()` 重複邏輯 | 修改 | ~10 行 |
| S6 | 新增單元測試 | 新建 | ~80 行 |
### 新元件 API 設計（草案）
```python
# core/interactive_memory_guard.py
def df_memory_mb(df: pd.DataFrame) -> float: ...
def process_rss_mb() -> Optional[float]: ...
def enforce_dataset_memory_guard(
df: pd.DataFrame,
*,
operation: str,
query_id: str = "",
max_input_mb: float = 96.0,
max_projected_rss_mb: float = 1100.0,
working_set_factor: float = 1.8,
) -> None:
"""跨工具通用 — DataFrame + RSS 投影守衛。超限 raise MemoryError。"""
```
---
## Agent 1: agent-mt — Material Trace
### 痛點（代碼驗證）
| 痛點 | 嚴重度 | 代碼位置 | 驗證結果 |
|------|:------:|---------|---------|
| 分頁重查 Oracle | 🔴 | `service.py:246` `forward_query()` | 每次翻頁走 `_execute_batched_query()` 完整重查 |
| Export 全量 materialize | 🔴 | `service.py:315` `export_csv()` | `read_sql_df_slow` + `to_csv().encode()` 全在記憶體 |
| 正向查詢無行數上限 | 🟡 | `forward_by_lot.sql` / `forward_by_workorder.sql` | 確認無 FETCH FIRST |
| 記憶體守衛不投影 RSS | 🟡 | `service.py:156` `_check_memory_guard()` | 只檢 DataFrame 大小，不看 RSS |
| 完全零快取 | 🔴 | service.py 全域 import | 無任何 Redis/LRU/DuckDB import |
### 任務清單
| # | 任務 | 改動檔案 | 行數估計 | 優先序 |
|---|------|---------|---------|:---:|
| MT1 | Redis 查詢結果快取 — key=(mode,values,wc_groups) hash, TTL=5min | `service.py` | ~80 行 | P0 |
| MT2 | 分頁/匯出讀快取 — `_paginate()` 從 Redis 讀取而非重查 | `service.py` | ~40 行 | P0 |
| MT3 | 升級記憶體守衛 — `_check_memory_guard` 改用 `enforce_dataset_memory_guard` | `service.py` | ~15 行 | P1 |
| MT4 | 正向 SQL 加 `FETCH FIRST 50001 ROWS ONLY` + truncation 標記 | `forward_by_*.sql` + `service.py` | ~25 行 | P1 |
| MT5 | Export 串流化 — 改用 `read_sql_df_slow_iter` + generator yield | `service.py` | ~40 行 | P2 |
| MT6 | 強制 GC — `_execute_batched_query` 後 `gc.collect()` | `service.py` | ~5 行 | P2 |
| MT7 | 新增/更新測試 | `tests/test_material_trace_*.py` | ~100 行 | — |
### 快取架構設計
```
POST /query (page=1)
→ hash = md5(mode + sorted(values) + sorted(wc_groups))
→ cache_key = "mt:result:{hash}"
→ cache MISS → Oracle 查詢 → DataFrame → store Redis (parquet bytes, TTL=5min)
→ _paginate(df, page=1, per_page=50)
POST /query (page=2) ← 翻頁
→ same cache_key
→ cache HIT → Redis load DataFrame
→ _paginate(df, page=2, per_page=50) ← 不打 Oracle
POST /export
→ same cache_key
→ cache HIT → Redis load DataFrame → streaming CSV
```
---
## Agent 2: agent-qt — Query Tool
### 痛點（代碼驗證）
| 痛點 | 嚴重度 | 代碼位置 | 驗證結果 |
|------|:------:|---------|---------|
| 明細全量回傳 | 🔴 | `lot_history.sql` 無 LIMIT | lot-history/associations/equipment-lots 全無 server-side 分頁 |
| EventFetcher 無累積上限 | 🔴 | `event_fetcher.py:251` `_fetch_and_group_batch` | `grouped` dict 無限累積，fetchmany 只節省 Oracle 端 |
| equipment-lots 無快取+全量 | 🔴 | `service.py:1604` | 365天×20台→10萬+行全量 DataFrame，零快取 |
| CSV export 全量 materialize | 🟡 | `routes.py:816` | 先 `get_lot_history_batch()` 全量，再 yield CSV |
| 無互動式 RSS 守衛 | 🟡 | 全部服務函數 | 完全依賴被動式 worker_memory_guard |
| 48M 無索引表 | 🟡 | `lot_split_merge_history.sql` | fast mode 有 6月+500行限制（部分緩解） |
### 任務清單
| # | 任務 | 改動檔案 | 行數估計 | 優先序 |
|---|------|---------|---------|:---:|
| QT1 | EventFetcher 加 total-result 守衛 — 累積 > N 行截斷+標記 | `event_fetcher.py` | ~25 行 | P0 |
| QT2 | 重型端點前加 RSS 投影守衛 | `service.py` (5 處) | ~30 行 | P0 |
| QT3 | lot-history SQL 加 OFFSET/FETCH 分頁 | `lot_history.sql` + `service.py` + `routes.py` | ~60 行 | P1 |
| QT4 | equipment-lots SQL 加 OFFSET/FETCH 分頁 | `equipment_lots.sql` + `service.py` + `routes.py` | ~60 行 | P1 |
| QT5 | equipment-lots 加 Redis 快取（同 equipment_status_hours 模式） | `service.py` | ~30 行 | P1 |
| QT6 | split_merge_history full mode 預設 fast | `service.py:1128` | ~10 行 | P2 |
| QT7 | 重型端點 response 後 gc.collect() | `routes.py` | ~10 行 | P2 |
| QT8 | 新增/更新測試 | `tests/test_query_tool_*.py` | ~150 行 | — |
### EventFetcher 守衛設計
```python
# event_fetcher.py — _fetch_and_group_batch 內
_TOTAL_RESULT_MAX_ROWS = int(os.getenv("EVENT_FETCHER_MAX_TOTAL_ROWS", "500000"))
total_row_count += 1
if total_row_count > _TOTAL_RESULT_MAX_ROWS:
logger.warning("EventFetcher total rows %d exceeds limit %d, truncating",
total_row_count, _TOTAL_RESULT_MAX_ROWS)
meta["truncated"] = True
break # 停止累積，回傳已有資料
```
---
## Agent 3: agent-md — Mid-Section Defect
### 痛點（代碼驗證）
| 痛點 | 嚴重度 | 代碼位置 | 驗證結果 |
|------|:------:|---------|---------|
| RQ 健康檢查只看 import | 🔴 | `trace_job_service.py:51` | `_check_rq_available()` 只做 `import rq`，不 ping Redis，不檢 worker |
| Sync fallback 無 RSS 守衛 | 🔴 | `trace_routes.py:662` | enqueue 失敗→直接走 sync，MSD 甚至不拒絕 >50K CIDs |
| Stampede lock 只在 legacy path | 🟡 | `service.py:169` | 三階段 events 端點完全無 stampede 防護 |
| Lock wait 90s fail-open | 🟡 | `service.py:195` | 等 90s 後直接執行，可能 stampede |
| Aggregation 全量在記憶體 | 🟡 | `service.py:292` | `build_trace_aggregation_from_events` 需完整 events dict |
| Export 全量 materialize | 🟡 | `service.py:644` | `query_analysis()` 全量 → yield CSV |
### 任務清單
| # | 任務 | 改動檔案 | 行數估計 | 優先序 |
|---|------|---------|---------|:---:|
| MD1 | RQ 健康監控升級 — 加 `conn.ping()` + worker 存活檢查 + TTL cache | `trace_job_service.py` | ~40 行 | P0 |
| MD2 | Sync 路徑 RSS 投影守衛 — events 入口處檢查，超限返回 503 | `trace_routes.py` | ~35 行 | P0 |
| MD3 | RQ 不可用時前端降級提示 — 健康端點 + 前端提示 | `health_routes.py` + Vue | ~30 行 | P1 |
| MD4 | Stampede lock timeout 延長 — 90s → 180s | `service.py` 常數 | ~5 行 | P1 |
| MD5 | Events 端點加 stampede lock | `trace_routes.py` | ~20 行 | P1 |
| MD6 | Aggregation 前 RSS checkpoint | `service.py:292` | ~15 行 | P2 |
| MD7 | 新增/更新測試 | `tests/test_mid_section_defect_*.py` | ~120 行 | — |
### RQ 健康檢查升級設計
```python
# trace_job_service.py
_RQ_HEALTH_TTL = 60 # 秒
_rq_health_cache = {"available": None, "checked_at": 0}
def is_async_available() -> bool:
now = time.monotonic()
if now - _rq_health_cache["checked_at"] < _RQ_HEALTH_TTL:
return _rq_health_cache["available"]
if not _check_rq_available():
_update_cache(False, now)
return False
conn = get_redis_client()
if conn is None:
_update_cache(False, now)
return False
try:
conn.ping() # ← 新增：實際 ping Redis
except Exception:
_update_cache(False, now)
return False
# 新增：檢查 worker 存活
try:
from rq import Queue
q = Queue("trace", connection=conn)
if q.count > 100: # queue 堵塞
logger.warning("RQ queue backed up: %d jobs", q.count)
workers = rq.Worker.all(queue=q)
if not workers:
_update_cache(False, now)
return False
except Exception:
pass # fail-open for worker check
_update_cache(True, now)
return True
```
---
## Agent 4: agent-qa — 測試驗證
### 任務清單
| # | 任務 | 範圍 |
|---|------|------|
| QA1 | 跑既有測試套件確認無回歸 | `pytest tests/` |
| QA2 | 驗證新 `core/interactive_memory_guard.py` 單元測試 | 新測試 |
| QA3 | 驗證 Material Trace 快取行為（hit/miss/TTL） | `test_material_trace_*.py` |
| QA4 | 驗證 EventFetcher 截斷邏輯 | `test_event_fetcher.py` |
| QA5 | 驗證 RQ 健康檢查各場景（worker 存活/掉線/Redis 斷線） | `test_trace_job_service.py` |
| QA6 | 壓力測試：模擬高 CID 數量的記憶體用量 | `tests/stress/` |
---
## 執行順序與依賴
```
Week 1:
Day 1-2: agent-shared (S1-S6) ← 阻塞後續
Day 2: 開始 agent-mt, agent-qt, agent-md (平行)
Week 1-2:
agent-mt: MT1→MT2→MT3→MT4→MT5→MT6→MT7
agent-qt: QT1→QT2→QT3→QT4→QT5→QT6→QT7→QT8
agent-md: MD1→MD2→MD3→MD4→MD5→MD6→MD7
Week 2:
agent-qa: QA1→QA6 (全部完成後)
```
---
## 風險與注意事項
1. **Material Trace Redis 快取需要 `core/redis_df_store.py`** — 已被 equipment_status_hours 使用，API 可直接沿用
2. **EventFetcher 截斷會影響中段缺陷** — mid-section defect 的 events 也走 EventFetcher，需確保截斷標記正確傳遞
3. **SQL OFFSET/FETCH 需要 Oracle 12c+** — 確認生產環境版本支援
4. **RQ worker 檢查有效能開銷** — 使用 60s TTL cache 避免每次請求都查

243
docs/oom_cache_cross_apply_analysis.md Normal file
View File

@@ -0,0 +1,243 @@
# 防 OOM 與快取機制跨工具套用分析
> 基於報廢歷史 (reject-history) 已建立的防護體系，評估物料追溯、查詢工具、中段缺陷三個工具的現況缺口與改善建議。
## 標竿：報廢歷史已具備的機制
### 防 OOM（6 層縱深防禦）
| 層級 | 機制 | 檔案 | 作用 |
|------|------|------|------|
| A | Worker RSS 記憶體守衛 | `core/worker_memory_guard.py` | daemon 每 15s 檢查 RSS；70% 告警 → 85% 清快取+GC → 95% SIGTERM 重啟 |
| B | 互動式記憶體守衛 | `services/reject_dataset_cache.py` | 每次快取派生前：DataFrame > 96MB 拒絕；投影 RSS > 1100MB 拒絕 |
| C | 強制 GC | `services/reject_dataset_cache.py` | 每次互動計算後 `gc.collect()` |
| D | Chunk 記憶體限制 | `services/batch_query_engine.py` | 每個 Oracle 查詢分塊 > 192MB 丟棄 |
| E | 大結果溢出至 Parquet | `core/query_spool_store.py` | > 20 萬行或 48MB 寫磁碟，Redis 僅存指標；2GB 總量上限 |
| F | Gunicorn Worker 回收 | `gunicorn.conf.py` | 每 1200±300 request 自動回收 worker |
### 快取（4 層 + 2 加速器）
| 層級 | 機制 | 作用 |
|------|------|------|
| L1 | ProcessLevelCache（進程內 LRU+TTL） | dataset TTL=15min/8 entries |
| L2 | Redis（跨 worker） | DataFrame 存為 parquet bytes |
| L3 | Parquet Spool Store（磁碟溢出） | 大型結果寫磁碟，背景每 5 分鐘清理 |
| 加速器 1 | DuckDB SQL Runtime | 對 Parquet 跑 SQL 算 batch-pareto/view/export |
| 加速器 2 | Materialized Pareto 聚合 | 預計算 6 維指標 cube |
| 批次引擎 | Batch Query Engine | 長日期範圍分月拆解 + Redis 分塊快取 |
---
## 現況對照矩陣
| 防護機制 | 報廢歷史 | 物料追溯 | 查詢工具 | 中段缺陷 |
|----------|:---:|:---:|:---:|:---:|
| 互動式記憶體守衛 (RSS 投影) | ✅ | ⚠️ 僅 DataFrame 大小 | ❌ | ❌ |
| Parquet 磁碟溢出 | ✅ | ❌ | ❌ | ❌ |
| DuckDB SQL Runtime | ✅ | ❌ | ❌ | ❌ |
| 物化聚合快照 | ✅ | ❌ | ❌ | ❌ |
| 批次引擎 (時間分片) | ✅ | ❌ | ❌ | ✅ 偵測查詢 |
| fetchmany 迭代器 | ❌ | ❌ | ✅ EventFetcher | ✅ EventFetcher |
| 非同步 Job Queue (RQ) | ❌ | ❌ | ❌ | ✅ >20K CIDs |
| NDJSON 串流 | ❌ | ❌ | ❌ | ✅ |
| 分散式鎖防 stampede | ❌ | ❌ | ❌ | ✅ |
| 強制 GC | ✅ | ❌ | ❌ | ⚠️ 僅 >10K CIDs |
| Redis 快取 | ✅ 三層 | ❌ 完全無 | ✅ resolve+events | ✅ 多層 |
| SQL 行數上限 | ✅ | ⚠️ 僅反向 | ❌ | ❌ |
| 分頁/匯出重查 Oracle | ❌ cache-based | ✅ 每次重查 | 部分 | ❌ cache-based |
---
## 工具一：物料追溯 (Material Trace)
### 架構概述
- 路由：`routes/material_trace_routes.py`（3 端點：query / export / filter-options）
- 服務：`services/material_trace_service.py`（正向/反向查詢、CSV 匯出）
- 主表：`DWH.DW_MES_LOTMATERIALSHISTORY`（18M 行）
- 查詢模式：同步、無快取、每次分頁/匯出都完整重查 Oracle
### 已有防護
- DataFrame 記憶體守衛 `_check_memory_guard()` — 256MB 上限（僅 DataFrame 大小，不投影 RSS）
- IN 子句批次拆分 `_IN_BATCH_SIZE=1000`
- 輸入數量限制：正向 200 筆、反向 50 筆
- 反向查詢 SQL 行數上限 `FETCH FIRST 10001 ROWS ONLY`
- 匯出行數上限 50,000
- Wildcard 前綴安全 `CONTAINER_RESOLVE_PATTERN_MIN_PREFIX_LEN=2`
- 限流：query 30/min、export 10/min
### 痛點
| 風險 | 嚴重度 | 說明 |
|------|:------:|------|
| 分頁重查 Oracle | 🔴 高 | 每翻一頁都重新跑完整 Oracle 查詢 + DataFrame 全量建構再切片 |
| Export 重查 Oracle | 🔴 高 | 獨立重查一次，5 萬行上限但整個 DataFrame 一次載入 |
| 正向查詢無行數上限 | 🟡 中 | 反向有 FETCH FIRST 但正向無 SQL 級行數限制 |
| 記憶體守衛不投影 RSS | 🟡 中 | 不考慮當前進程 RSS 狀態 |
| 無 GC | 🟢 低 | 靠 Worker RSS 守衛兜底 |
### 建議
| 優先序 | 措施 | 改動量 | 效益 |
|:---:|------|:---:|------|
| P0 | 加入查詢結果快取 — 首次結果存 Redis (key=mode+values hash, TTL=5min)，分頁/匯出讀快取 | ~80 行 | 消除 N+1 次 Oracle 查詢 |
| P1 | 升級記憶體守衛 — 加上 RSS 投影檢查 | ~30 行 | 高壓下拒絕大查詢 |
| P1 | 正向查詢加 SQL 行數上限 `FETCH FIRST 50001 ROWS ONLY` + truncation 標記 | ~25 行 | 防意外全表掃描 |
| P2 | Export 串流化 — 改為 generator yield rows | ~40 行 | 降低 export 瞬時記憶體 |
| P2 | 加強制 GC | ~5 行 | 及時釋放臨時 DataFrame |
---
## 工具二：查詢工具 (Query Tool)
### 架構概述
- 路由：`routes/query_tool_routes.py`（10 端點）
- 服務：`services/query_tool_service.py`（1868 行）+ `services/event_fetcher.py`
- 三大功能：批次追蹤（正向）、流水批反查（反向）、設備生產批次追蹤
- 主表：`DW_MES_LOTWIPHISTORY`（53M）、`DW_MES_LOTMATERIALSHISTORY`、`DW_MES_HM_LOTMOVEOUT`（48M, 無索引）
- 已有 EventFetcher fetchmany 迭代器 + Redis 快取（resolve 60s / events 180-300s）
### 已有防護
- Slow-query semaphore `DB_SLOW_MAX_CONCURRENT=5`
- fetchmany 迭代器避免 cursor.fetchall() 三重 materialize
- 輸入限制：LOT 100 / Serial 100 / WO 50 / Equipment 20 / 日期 365 天
- Container IDs 批次上限 `QUERY_TOOL_MAX_CONTAINER_IDS=200`
- Wildcard 展開限制：每 token 2000 / 總計 30,000
- EventFetcher 大量 CIDs 跳過快取寫入 (>10K)
- 限流：resolve 10/min、equipment 5/min、export 3/min
### 痛點
| 風險 | 嚴重度 | 說明 |
|------|:------:|------|
| 明細全量回傳 | 🔴 高 | lot-history / associations / equipment-lots 全部一次回傳，無 server-side 分頁 |
| 48M 行無索引表 | 🔴 高 | `DW_MES_HM_LOTMOVEOUT` split_merge_history 走全表掃描，30-120s |
| CSV export 全量 materialize | 🟡 中 | 先完整查詢建 DataFrame 再轉 CSV stream |
| 無互動式記憶體守衛 | 🟡 中 | EventFetcher 逐批但無整體 RSS 投影 |
| 無 Parquet 溢出 | 🟡 中 | 大結果只能在 Redis (512MB) 或記憶體中 |
### 建議
| 優先序 | 措施 | 改動量 | 效益 |
|:---:|------|:---:|------|
| P0 | detail 端點加 server-side 分頁 — SQL 加 OFFSET/FETCH | ~60 行/端點 | 防大 LOT 數千筆 history 一次灌入 |
| P0 | EventFetcher 加 total-result 記憶體守衛 — 累積結果超限截斷+標記 | ~25 行 | 防累積結果超安全線 |
| P1 | 拓展 Parquet 溢出到 EventFetcher — 大結果寫 spool + DuckDB 讀取 | ~120 行 | equipment 365 天查詢安全運作 |
| P1 | 加 RSS 投影守衛 — 重型端點前檢查 projected RSS | ~30 行 | 跨端點整體防護 |
| P2 | split_merge_history 預設 fast 模式 — full 需手動啟用 | ~10 行 | 降低 48M 表衝擊 |
| P2 | 加強制 GC — heavy 端點 response 後 gc.collect() | ~10 行 | 配合 Worker RSS 守衛 |
---
## 工具三：中段缺陷 (Mid-Section Defect)
### 架構概述
- 路由：`routes/mid_section_defect_routes.py`（5 端點）+ `routes/trace_routes.py`（三階段管線）
- 服務：`services/mid_section_defect_service.py`（1350 行）
- 三階段漸進式管線：seed-resolve → lineage → events（含歸因統計）
- 主表：`DW_MES_LOTWIPHISTORY`（53M）、`DW_MES_LOTREJECTHISTORY`、`DW_MES_CONTAINER`（5.2M）
- 已有：RQ async job queue、NDJSON 串流、fetchmany 迭代器、Redis 分散式鎖、BatchQueryEngine
### 已有防護
- Worker RSS 記憶體守衛（全域）
- fetchmany 迭代器避免三重 materialize（2026-02-25 OOM 事件後加入）
- 非同步 RQ Job Queue — >20K CIDs 導向獨立 worker process
- NDJSON 串流 — 大結果漸進式傳送
- 分散式鎖防 cache stampede（120s lock, 90s wait）
- BatchQueryEngine — 偵測查詢 >10 天分月拆解
- EventFetcher 大量 CIDs 跳過快取
- 顯式 gc.collect() — >10K CIDs 後觸發
- systemd MemoryMax=6G 硬保護
- 限流：analysis 6/min、detail 15/min、export 3/min
### 痛點
| 風險 | 嚴重度 | 說明 |
|------|:------:|------|
| 無 CID 硬拒絕 | 🔴 高 | 設計上刻意不拒（資料完整性需求），114K CIDs 是真實場景，sync 路徑峰值 4-6GB |
| RQ 不可用時 fallback 到 sync | 🔴 高 | RQ worker 掛掉時 >20K CIDs 仍由 gunicorn worker 同步處理 |
| Aggregation 全量在記憶體 | 🟡 中 | `build_trace_aggregation_from_events()` 需完整 events dict 做歸因 |
| Cache stampede fail-open 90s | 🟡 中 | 管線 > 90s 時第二個 request 也開始跑，雙重 Oracle 負載 |
| Export 全量 materialize | 🟡 中 | 從快取讀全部 detail list 再 stream |
| SQL/Python workcenter 分類漂移 | 🟢 低 | CASE WHEN 與 Python dict 需手動同步 |
### 建議
| 優先序 | 措施 | 改動量 | 效益 |
|:---:|------|:---:|------|
| P0 | RQ 健康監控 + 降級提示 — 定期驗證 RQ worker 存活，不可用時前端顯示警告 | ~50 行 | 防靜默 fallback 到 sync 撐爆 worker |
| P0 | Sync 路徑加 RSS 投影守衛 — 超限返回 503 + retry-after | ~35 行 | 保護 gunicorn worker |
| P1 | Parquet 溢出 for events — 大結果寫 spool，aggregation 改用 DuckDB GROUP BY | ~200 行 | 114K CIDs 場景記憶體從 ~4GB 降到 ~500MB |
| P1 | 延長 stampede lock timeout — 從 90s 提高到 180s 或改 pub/sub 通知 | ~20 行 | 減少雙重查詢 |
| P2 | Workcenter 分類統一源 — 從 Python dict 自動生成 SQL CASE WHEN | ~60 行 | 消除隱性資料遺漏 |
| P2 | Export 改 DuckDB 串流 — spool 存在時用 fetchmany generator | ~40 行 | 降低 export 峰值記憶體 |
---
## 跨工具共用方案
### 從報廢歷史推廣的通用元件
| 元件 | 推廣對象 | 理由 |
|------|---------|------|
| `_enforce_interactive_memory_guard()` | 全部三個 | 零改架構，插入式防護，~30 行 |
| Parquet spool + DuckDB SQL | 查詢工具、中段缺陷 | EventFetcher 大結果目前僅 in-memory |
| 查詢結果 Redis 快取 | 物料追溯 | 唯一完全零快取的工具 |
### 從中段缺陷推廣的元件
| 元件 | 推廣對象 | 理由 |
|------|---------|------|
| RQ async job queue | 查詢工具 (equipment 長查詢) | 365 天設備查詢可能跑 2-3 分鐘 |
| 分散式鎖防 stampede | 物料追溯（若加快取） | 避免快取冷啟動時多 request 同時打 Oracle |
| sessionStorage 前端快取 | 物料追溯、查詢工具 | 頁面切換回頭不需重查 |
---
## 建議執行順序
```
Phase 1（快速加固, 1-2 天）:
├─ 全部三個: 加 _enforce_interactive_memory_guard + 強制 GC
├─ 物料追溯: 加 Redis 查詢結果快取（消除分頁/匯出重查）
└─ 物料追溯: 正向查詢加 SQL 行數上限
Phase 2（結構性改善, 3-5 天）:
├─ 查詢工具: detail 端點加 server-side 分頁
├─ 中段缺陷: RQ 健康監控 + sync 路徑 RSS 守衛
└─ 中段缺陷: stampede lock timeout 延長
Phase 3（進階優化, 5-7 天）:
├─ 共用: EventFetcher 拓展 Parquet spool 支援
├─ 中段缺陷: aggregation 改用 DuckDB SQL
└─ 查詢工具: 大結果走 spool + DuckDB
```
---
## 相關檔案索引
| 類別 | 關鍵檔案路徑 |
|------|-------------|
| 標竿：報廢歷史快取 | `services/reject_dataset_cache.py` |
| 標竿：DuckDB runtime | `services/reject_cache_sql_runtime.py` |
| 標竿：Parquet spool | `core/query_spool_store.py` |
| 標竿：記憶體守衛 | `core/worker_memory_guard.py` |
| 標竿：批次引擎 | `services/batch_query_engine.py` |
| 物料追溯 service | `services/material_trace_service.py` |
| 物料追溯 routes | `routes/material_trace_routes.py` |
| 查詢工具 service | `services/query_tool_service.py` |
| 查詢工具 routes | `routes/query_tool_routes.py` |
| 查詢工具 EventFetcher | `services/event_fetcher.py` |
| 中段缺陷 service | `services/mid_section_defect_service.py` |
| 中段缺陷 routes | `routes/mid_section_defect_routes.py` |
| 中段缺陷 trace 管線 | `routes/trace_routes.py` |
| 中段缺陷 async job | `services/trace_job_service.py` |
| 中段缺陷 lineage 引擎 | `services/lineage_engine.py` |
| 共用快取層 | `core/cache.py` |
| 共用 database | `core/database.py` |

125
docs/reject_history_performance.md
View File

@@ -1,125 +0,0 @@
# Reject 歷史績效表設計說明
## 目標
使用 `DW_MES_LOTREJECTHISTORY` 為主，輔以其他維度表，建立可直接用於報表的 `reject` 歷史績效表（按日彙總），解決原始資料直接查詢時的績效與一致性問題。
## 使用資料表
- `DWH.DW_MES_LOTREJECTHISTORY`: 不良/報廢事實表（主來源）
- `DWH.DW_MES_CONTAINER`: 補齊 `PJ_TYPE`、`PRODUCTLINENAME`、`MFGORDERNAME`
- `DWH.DW_MES_SPEC_WORKCENTER_V`: 對應 `WORKCENTER_GROUP` 與排序欄位
- `DWH.ERP_PJ_WIP_SCRAP_REASONS_EXCLUDE`: 良率排除政策表（`ENABLE_FLAG='Y'` 代表不納入良率計算）
## 資料評估重點（2026-02-13，近 30 天樣本）
- `DW_MES_LOTREJECTHISTORY` 共 `230,074` 筆；`HISTORYMAINLINEID` 僅 `75,683` 個。
- `HISTORYMAINLINEID` 多筆情況明顯（`30,784` 個主事件，平均每主事件 `6.02` 筆），代表同主事件會拆成多個 `LOSSREASONNAME`。
- 若直接加總 `MOVEINQTY`，分母會被重複計算。近 30 天樣本中：
- `NAIVE_MOVEIN = 44,836,693,831`
- `DEDUP_MOVEIN = 35,658,750,247`
- 膨脹比 `1.2574`（約高估 25.74%）
- 指標定義依業務規則分開處理：
- `REJECT_TOTAL_QTY = REJECTQTY + STANDBYQTY + QTYTOPROCESS + INPROCESSQTY + PROCESSEDQTY`（扣帳報廢）
- `DEFECT_QTY = DEFECTQTY`（不扣帳報廢）
- `DW_MES_SPEC_WORKCENTER_V` 若直接以 `WORK_CENTER` join 會放大筆數；需先彙整為唯一 `WORK_CENTER -> GROUP/SEQUENCE` 對照表再 join。
## 績效表欄位與計算邏輯
- 粒度：`日 + 工站群組 + 工站 + 站點規格 + 設備 + 產品維度 + 不良原因`
- 核心指標：
- `REJECT_EVENT_ROWS`: 原始 reject 紀錄筆數
- `AFFECTED_LOT_COUNT`: 受影響 lot 數（distinct `CONTAINERID`）
- `MOVEIN_QTY`: 以 `HISTORYMAINLINEID` 去重後的投入量
- `REJECT_QTY`: 原始 `REJECTQTY` 加總（五欄之一）
- `REJECT_TOTAL_QTY`: 五個 reject 相關欄位加總（扣帳報廢）
- `DEFECT_QTY`: `DEFECTQTY` 加總（不扣帳報廢）
- `REJECT_RATE_PCT = REJECT_TOTAL_QTY / MOVEIN_QTY * 100`
- `DEFECT_RATE_PCT = DEFECT_QTY / MOVEIN_QTY * 100`
- `REJECT_SHARE_PCT = REJECT_TOTAL_QTY / (REJECT_TOTAL_QTY + DEFECT_QTY) * 100`
## 排除政策與前端開關
- 預設模式：排除 `ERP_PJ_WIP_SCRAP_REASONS_EXCLUDE` 中 `ENABLE_FLAG='Y'` 的報廢原因。
- 可切換模式：提供 `include_excluded_scrap=true|false` 讓使用者決定是否納入。
- 前端頁面提供「納入不計良率報廢」開關，並同步影響 summary/trend/pareto/list/export。
- 排除原因清單採全表快取，預設每日刷新一次（Redis 優先、記憶體 fallback）。
## API 與欄位契約
- `GET /api/reject-history/options`
- 回傳 `workcenter_groups`、`reasons` 與政策 `meta`
- `GET /api/reject-history/summary`
- 回傳 `MOVEIN_QTY`、`REJECT_TOTAL_QTY`、`DEFECT_QTY`、`REJECT_RATE_PCT`、`DEFECT_RATE_PCT`、`REJECT_SHARE_PCT`、`AFFECTED_LOT_COUNT`、`AFFECTED_WORKORDER_COUNT`
- `GET /api/reject-history/trend`
- 回傳趨勢 `items[]`，每筆含 `bucket_date`、`REJECT_TOTAL_QTY`、`DEFECT_QTY`、`REJECT_RATE_PCT`、`DEFECT_RATE_PCT`
- `GET /api/reject-history/reason-pareto`
- 支援 `metric_mode=reject_total|defect`
- 支援 `pareto_scope=top80|all`（預設 `top80`）
- `GET /api/reject-history/list`
- 分頁回傳 `items[]` 與 `pagination`
- 明細保留五個 reject 欄位（`REJECT_QTY`、`STANDBY_QTY`、`QTYTOPROCESS_QTY`、`INPROCESS_QTY`、`PROCESSED_QTY`）與 `DEFECT_QTY`
- `GET /api/reject-history/export`
- CSV 欄位與 list 語義一致，含 `REJECT_TOTAL_QTY` 與 `DEFECT_QTY`
## 前端視覺與互動
- 主要區塊：
- Header（語義 badge + 更新時間）
- 篩選區（時間、原因、`WORKCENTER_GROUP`、政策開關、Pareto 前 80% 開關）
- KPI（8 張卡，Reject 暖色語義 / Defect 冷色語義）
- 趨勢圖（報廢量與報廢率分圖）
- Pareto（柱狀 + 累積線）與明細表
- 互動規則：
- Pareto 點選原因後，會套用為 active filter chip 並重查
- 再次點選同原因會取消篩選
- 預設僅顯示累計前 80%，可切換顯示完整 Pareto
- 匯出 CSV 使用目前畫面相同篩選條件
## 交付檔案
- 建表 + 刷新 SQL：`docs/reject_history_performance.sql`
- 可被應用層直接載入的查詢 SQL：`src/mes_dashboard/sql/reject_history/performance_daily.sql`
## 上線與回滾策略
- 上線策略：
- 先維持 `data/page_status.json` 中 `/reject-history` 為 `dev`
- 完成 UAT 後再改為 `released`
- 回滾策略：
- 將 `/reject-history` 狀態切回 `dev` 或移除導航入口
- 保留 API 與既有頁面，不影響既有報表
- 快取策略：
- 排除政策表每日全表刷新（預設 86400 秒）
- Redis 異常時退回記憶體快取，不阻斷查詢
## 驗證紀錄（2026-02-13）
- 後端/整合測試：
- `pytest -q tests/test_reject_history_service.py tests/test_scrap_reason_exclusion_cache.py tests/test_reject_history_routes.py tests/test_reject_history_shell_coverage.py tests/test_portal_shell_wave_b_native_smoke.py::test_reject_history_native_smoke_query_sections_and_export tests/test_app_factory.py::AppFactoryTests::test_routes_registered`
- 結果：`22 passed`
- 前端建置：
- `cd frontend && npm run build`
- 結果：成功產出 `reject-history.html/js/css`，並完成 dist 複製流程
## 建議排程
- 每日跑前一日增量：
- `:start_date = TRUNC(SYSDATE - 1)`
- `:end_date = TRUNC(SYSDATE - 1)`
- 每月第一天補跑前 31 天，避免補數漏失。
## Cache-SQL Rollout（DuckDB）
- 階段順序：`batch-pareto -> view -> export-cached`
- 預設全開：
- `REJECT_CACHE_SQL_ENABLED=true`
- `REJECT_CACHE_SQL_BATCH_PARETO_ENABLED=true`
- `REJECT_CACHE_SQL_VIEW_ENABLED=true`
- `REJECT_CACHE_SQL_EXPORT_ENABLED=true`
- 回退策略（預設回退 legacy）：
- `REJECT_CACHE_SQL_FALLBACK_LEGACY_ENABLED=true`
- 可用 endpoint 級旗標單獨控制 `*_FALLBACK_LEGACY_ENABLED`
- 灰度建議：
- 先只開 `batch-pareto`，觀察 `pareto_source` 與 fallback reason
- 再開 `view`，確認 `detail.pagination` 與舊版一致
- 最後開 `export-cached`，確認 CSV 欄位/筆數與 view detail 篩選範圍一致
## 驗證清單（前端提示與匯出一致性）
- 前端「柏拉圖顯示限制（80%/Top20）」僅影響畫面顯示，不影響明細與匯出範圍。
- 使用相同 `query_id + filters` 比對：
- `GET /api/reject-history/view` 的 `detail.pagination.total`
- `GET /api/reject-history/export-cached` 的匯出總筆數
- 兩者應一致（display-only truncation 不得裁掉 export row）。
- 若觸發 cache-SQL fail-fast（`*_FALLBACK_LEGACY_ENABLED=false`），前端需提示使用者重試或縮小條件。
## 已知環境備註
- `tests/test_navigation_contract.py` 需要 `docs/migration/portal-no-iframe/baseline_drawer_visibility.json`。目前工作區缺少此 baseline 檔案，屬既有環境缺口，與本次 reject-history 開發內容無直接耦合。

242
docs/reject_history_performance.sql
View File

@@ -1,242 +0,0 @@
/*
Reject 歷史績效表建置腳本
目的:
- 以 DW_MES_LOTREJECTHISTORY 為主，建立可直接報表化的日彙總績效表
- 補齊產品/工站維度，並區分扣帳報廢與不扣帳報廢
*/
/* ============================================================
1) 建表 (執行一次)
============================================================ */
CREATE TABLE DWH.DW_PJ_REJECT_HISTORY_PERF_D (
TXN_DAY DATE NOT NULL,
TXN_MONTH VARCHAR2(7) NOT NULL,
WORKCENTER_GROUP VARCHAR2(40) NOT NULL,
WORKCENTERSEQUENCE_GROUP NUMBER(10) NOT NULL,
WORKCENTERNAME VARCHAR2(40) NOT NULL,
SPECNAME VARCHAR2(40) NOT NULL,
EQUIPMENTNAME VARCHAR2(255) NOT NULL,
PRIMARY_EQUIPMENTNAME VARCHAR2(40) NOT NULL,
PRODUCTLINENAME VARCHAR2(40) NOT NULL,
PJ_TYPE VARCHAR2(40) NOT NULL,
LOSSREASONNAME VARCHAR2(40) NOT NULL,
REJECTCATEGORYNAME VARCHAR2(40) NOT NULL,
REJECT_EVENT_ROWS NUMBER NOT NULL,
AFFECTED_LOT_COUNT NUMBER NOT NULL,
AFFECTED_WORKORDER_COUNT NUMBER NOT NULL,
MOVEIN_QTY NUMBER NOT NULL,
REJECT_QTY NUMBER NOT NULL,
REJECT_TOTAL_QTY NUMBER NOT NULL,
DEFECT_QTY NUMBER NOT NULL,
STANDBY_QTY NUMBER NOT NULL,
QTYTOPROCESS_QTY NUMBER NOT NULL,
INPROCESS_QTY NUMBER NOT NULL,
PROCESSED_QTY NUMBER NOT NULL,
REJECT_RATE_PCT NUMBER(18, 4) NOT NULL,
DEFECT_RATE_PCT NUMBER(18, 4) NOT NULL,
REJECT_SHARE_PCT NUMBER(18, 4) NOT NULL,
LAST_REFRESH_TS DATE NOT NULL
);
CREATE INDEX DWH.IDX_RJH_PERF_D_01 ON DWH.DW_PJ_REJECT_HISTORY_PERF_D (TXN_DAY);
CREATE INDEX DWH.IDX_RJH_PERF_D_02 ON DWH.DW_PJ_REJECT_HISTORY_PERF_D (WORKCENTER_GROUP, TXN_DAY);
CREATE INDEX DWH.IDX_RJH_PERF_D_03 ON DWH.DW_PJ_REJECT_HISTORY_PERF_D (PRIMARY_EQUIPMENTNAME, TXN_DAY);
CREATE INDEX DWH.IDX_RJH_PERF_D_04 ON DWH.DW_PJ_REJECT_HISTORY_PERF_D (LOSSREASONNAME, TXN_DAY);
/* ============================================================
2) 區間刷新 (可每日排程)
綁定參數:
:start_date (YYYY-MM-DD)
:end_date (YYYY-MM-DD)
============================================================ */
DELETE FROM DWH.DW_PJ_REJECT_HISTORY_PERF_D
WHERE TXN_DAY >= TO_DATE(:start_date, 'YYYY-MM-DD')
AND TXN_DAY < TO_DATE(:end_date, 'YYYY-MM-DD') + 1;
INSERT /*+ APPEND */ INTO DWH.DW_PJ_REJECT_HISTORY_PERF_D (
TXN_DAY,
TXN_MONTH,
WORKCENTER_GROUP,
WORKCENTERSEQUENCE_GROUP,
WORKCENTERNAME,
SPECNAME,
EQUIPMENTNAME,
PRIMARY_EQUIPMENTNAME,
PRODUCTLINENAME,
PJ_TYPE,
LOSSREASONNAME,
REJECTCATEGORYNAME,
REJECT_EVENT_ROWS,
AFFECTED_LOT_COUNT,
AFFECTED_WORKORDER_COUNT,
MOVEIN_QTY,
REJECT_QTY,
REJECT_TOTAL_QTY,
DEFECT_QTY,
STANDBY_QTY,
QTYTOPROCESS_QTY,
INPROCESS_QTY,
PROCESSED_QTY,
REJECT_RATE_PCT,
DEFECT_RATE_PCT,
REJECT_SHARE_PCT,
LAST_REFRESH_TS
)
WITH workcenter_map AS (
SELECT
WORK_CENTER,
MIN(WORK_CENTER_GROUP) KEEP (
DENSE_RANK FIRST ORDER BY WORKCENTERSEQUENCE_GROUP
) AS WORKCENTER_GROUP,
MIN(WORKCENTERSEQUENCE_GROUP) AS WORKCENTERSEQUENCE_GROUP
FROM DWH.DW_MES_SPEC_WORKCENTER_V
WHERE WORK_CENTER IS NOT NULL
GROUP BY WORK_CENTER
),
reject_raw AS (
SELECT
TRUNC(r.TXNDATE) AS TXN_DAY,
TO_CHAR(TRUNC(r.TXNDATE), 'YYYY-MM') AS TXN_MONTH,
r.CONTAINERID,
NVL(TRIM(r.PJ_WORKORDER), TRIM(c.MFGORDERNAME)) AS PJ_WORKORDER,
NVL(TRIM(c.PJ_TYPE), '(NA)') AS PJ_TYPE,
NVL(TRIM(c.PRODUCTLINENAME), '(NA)') AS PRODUCTLINENAME,
NVL(TRIM(r.WORKCENTERNAME), '(NA)') AS WORKCENTERNAME,
NVL(TRIM(wm.WORKCENTER_GROUP), NVL(TRIM(r.WORKCENTERNAME), '(NA)')) AS WORKCENTER_GROUP,
NVL(wm.WORKCENTERSEQUENCE_GROUP, 999) AS WORKCENTERSEQUENCE_GROUP,
NVL(TRIM(r.SPECNAME), '(NA)') AS SPECNAME,
NVL(TRIM(r.EQUIPMENTNAME), '(NA)') AS EQUIPMENTNAME,
NVL(
TRIM(REGEXP_SUBSTR(r.EQUIPMENTNAME, '[^,]+', 1, 1)),
NVL(TRIM(r.EQUIPMENTNAME), '(NA)')
) AS PRIMARY_EQUIPMENTNAME,
NVL(TRIM(r.LOSSREASONNAME), '(未填寫)') AS LOSSREASONNAME,
NVL(TRIM(r.REJECTCATEGORYNAME), '(未填寫)') AS REJECTCATEGORYNAME,
NVL(r.MOVEINQTY, 0) AS MOVEINQTY,
NVL(r.REJECTQTY, 0) AS REJECT_QTY,
NVL(r.STANDBYQTY, 0) AS STANDBY_QTY,
NVL(r.QTYTOPROCESS, 0) AS QTYTOPROCESS_QTY,
NVL(r.INPROCESSQTY, 0) AS INPROCESS_QTY,
NVL(r.PROCESSEDQTY, 0) AS PROCESSED_QTY,
NVL(r.REJECTQTY, 0)
+ NVL(r.STANDBYQTY, 0)
+ NVL(r.QTYTOPROCESS, 0)
+ NVL(r.INPROCESSQTY, 0)
+ NVL(r.PROCESSEDQTY, 0) AS REJECT_TOTAL_QTY,
NVL(r.DEFECTQTY, 0) AS DEFECT_QTY,
ROW_NUMBER() OVER (
PARTITION BY NVL(
TRIM(r.HISTORYMAINLINEID),
TRIM(r.CONTAINERID) || ':' || TO_CHAR(r.TXNDATE, 'YYYYMMDDHH24MISS') || ':' || NVL(TRIM(r.SPECID), '-')
)
ORDER BY NVL(TRIM(r.LOSSREASONNAME), ' ')
) AS EVENT_RN
FROM DWH.DW_MES_LOTREJECTHISTORY r
LEFT JOIN DWH.DW_MES_CONTAINER c
ON c.CONTAINERID = r.CONTAINERID
LEFT JOIN workcenter_map wm
ON wm.WORK_CENTER = r.WORKCENTERNAME
WHERE r.TXNDATE >= TO_DATE(:start_date, 'YYYY-MM-DD')
AND r.TXNDATE < TO_DATE(:end_date, 'YYYY-MM-DD') + 1
),
daily_agg AS (
SELECT
TXN_DAY,
TXN_MONTH,
WORKCENTER_GROUP,
WORKCENTERSEQUENCE_GROUP,
WORKCENTERNAME,
SPECNAME,
EQUIPMENTNAME,
PRIMARY_EQUIPMENTNAME,
PRODUCTLINENAME,
PJ_TYPE,
LOSSREASONNAME,
REJECTCATEGORYNAME,
COUNT(*) AS REJECT_EVENT_ROWS,
COUNT(DISTINCT CONTAINERID) AS AFFECTED_LOT_COUNT,
COUNT(DISTINCT PJ_WORKORDER) AS AFFECTED_WORKORDER_COUNT,
SUM(CASE WHEN EVENT_RN = 1 THEN MOVEINQTY ELSE 0 END) AS MOVEIN_QTY,
SUM(REJECT_QTY) AS REJECT_QTY,
SUM(REJECT_TOTAL_QTY) AS REJECT_TOTAL_QTY,
SUM(DEFECT_QTY) AS DEFECT_QTY,
SUM(STANDBY_QTY) AS STANDBY_QTY,
SUM(QTYTOPROCESS_QTY) AS QTYTOPROCESS_QTY,
SUM(INPROCESS_QTY) AS INPROCESS_QTY,
SUM(PROCESSED_QTY) AS PROCESSED_QTY
FROM reject_raw
GROUP BY
TXN_DAY,
TXN_MONTH,
WORKCENTER_GROUP,
WORKCENTERSEQUENCE_GROUP,
WORKCENTERNAME,
SPECNAME,
EQUIPMENTNAME,
PRIMARY_EQUIPMENTNAME,
PRODUCTLINENAME,
PJ_TYPE,
LOSSREASONNAME,
REJECTCATEGORYNAME
)
SELECT
TXN_DAY,
TXN_MONTH,
WORKCENTER_GROUP,
WORKCENTERSEQUENCE_GROUP,
WORKCENTERNAME,
SPECNAME,
EQUIPMENTNAME,
PRIMARY_EQUIPMENTNAME,
PRODUCTLINENAME,
PJ_TYPE,
LOSSREASONNAME,
REJECTCATEGORYNAME,
REJECT_EVENT_ROWS,
AFFECTED_LOT_COUNT,
AFFECTED_WORKORDER_COUNT,
MOVEIN_QTY,
REJECT_QTY,
REJECT_TOTAL_QTY,
DEFECT_QTY,
STANDBY_QTY,
QTYTOPROCESS_QTY,
INPROCESS_QTY,
PROCESSED_QTY,
CASE
WHEN MOVEIN_QTY = 0 THEN 0
ELSE ROUND(REJECT_TOTAL_QTY * 100 / MOVEIN_QTY, 4)
END AS REJECT_RATE_PCT,
CASE
WHEN MOVEIN_QTY = 0 THEN 0
ELSE ROUND(DEFECT_QTY * 100 / MOVEIN_QTY, 4)
END AS DEFECT_RATE_PCT,
CASE
WHEN (REJECT_TOTAL_QTY + DEFECT_QTY) = 0 THEN 0
ELSE ROUND(REJECT_TOTAL_QTY * 100 / (REJECT_TOTAL_QTY + DEFECT_QTY), 4)
END AS REJECT_SHARE_PCT,
SYSDATE AS LAST_REFRESH_TS
FROM daily_agg;
COMMIT;
/* ============================================================
3) 快速驗證查詢
============================================================ */
SELECT
TXN_DAY,
WORKCENTER_GROUP,
SUM(MOVEIN_QTY) AS MOVEIN_QTY,
SUM(REJECT_QTY) AS REJECT_QTY,
SUM(REJECT_TOTAL_QTY) AS REJECT_TOTAL_QTY,
SUM(DEFECT_QTY) AS DEFECT_QTY,
ROUND(SUM(REJECT_TOTAL_QTY) * 100 / NULLIF(SUM(MOVEIN_QTY), 0), 4) AS REJECT_RATE_PCT
FROM DWH.DW_PJ_REJECT_HISTORY_PERF_D
WHERE TXN_DAY BETWEEN TO_DATE(:start_date, 'YYYY-MM-DD') AND TO_DATE(:end_date, 'YYYY-MM-DD')
GROUP BY TXN_DAY, WORKCENTER_GROUP
ORDER BY TXN_DAY DESC, WORKCENTER_GROUP;