feat(lineage): unified LineageEngine, EventFetcher, and progressive trace API

Introduce a unified Seed→Lineage→Event pipeline replacing per-page Python
BFS with Oracle CONNECT BY NOCYCLE queries, add staged /api/trace/*
endpoints with rate limiting and L2 Redis caching, and wire progressive
frontend loading via useTraceProgress composable.

Key changes:
- Add LineageEngine (split ancestors / merge sources / full genealogy)
  with QueryBuilder bind-param safety and batched IN clauses
- Add EventFetcher with 6-domain support and L2 Redis cache
- Add trace_routes Blueprint (seed-resolve, lineage, events) with
  profile dispatch, rate limiting, and Redis TTL=300s caching
- Refactor query_tool_service to use LineageEngine and QueryBuilder,
  removing raw string interpolation (SQL injection fix)
- Add rate limits and resolve cache to query_tool_routes
- Integrate useTraceProgress into mid-section-defect with skeleton
  placeholders and fade-in transitions
- Add lineageCache and on-demand lot lineage to query-tool
- Add TraceProgressBar shared component
- Remove legacy query-tool.js static script (3k lines)
- Fix MatrixTable package column truncation (.slice(0,15) removed)
- Archive unified-lineage-engine change, add trace-progressive-ui specs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
egg
2026-02-12 16:30:24 +08:00
parent c38b5f646a
commit 519f8ae2f4
52 changed files with 5074 additions and 4047 deletions

View File

@@ -1,20 +1,22 @@
<script setup>
import { reactive, ref } from 'vue';
import { apiGet, ensureMesApiAvailable } from '../core/api.js';
import { useAutoRefresh } from '../shared-composables/useAutoRefresh.js';
import FilterBar from './components/FilterBar.vue';
import KpiCards from './components/KpiCards.vue';
import ParetoChart from './components/ParetoChart.vue';
import TrendChart from './components/TrendChart.vue';
import DetailTable from './components/DetailTable.vue';
ensureMesApiAvailable();
const API_TIMEOUT = 120000; // 2min (genealogy can be slow)
const PAGE_SIZE = 200;
<script setup>
import { computed, reactive, ref } from 'vue';
import { apiGet, ensureMesApiAvailable } from '../core/api.js';
import { useAutoRefresh } from '../shared-composables/useAutoRefresh.js';
import { useTraceProgress } from '../shared-composables/useTraceProgress.js';
import TraceProgressBar from '../shared-composables/TraceProgressBar.vue';
import FilterBar from './components/FilterBar.vue';
import KpiCards from './components/KpiCards.vue';
import ParetoChart from './components/ParetoChart.vue';
import TrendChart from './components/TrendChart.vue';
import DetailTable from './components/DetailTable.vue';
ensureMesApiAvailable();
const API_TIMEOUT = 120000;
const PAGE_SIZE = 200;
const filters = reactive({
startDate: '',
endDate: '',
@@ -25,58 +27,94 @@ const committedFilters = ref({
endDate: '',
lossReasons: [],
});
const availableLossReasons = ref([]);
const analysisData = ref({
kpi: {},
charts: {},
daily_trend: [],
genealogy_status: 'ready',
detail_total_count: 0,
});
const detailData = ref([]);
const detailPagination = ref({
page: 1,
page_size: PAGE_SIZE,
total_count: 0,
total_pages: 1,
});
const detailLoading = ref(false);
const loading = reactive({
initial: false,
querying: false,
});
const hasQueried = ref(false);
const queryError = ref('');
function setDefaultDates() {
const today = new Date();
const endDate = new Date(today);
endDate.setDate(endDate.getDate() - 1);
const startDate = new Date(endDate);
startDate.setDate(startDate.getDate() - 6);
filters.startDate = toDateString(startDate);
filters.endDate = toDateString(endDate);
}
function toDateString(value) {
return value.toISOString().slice(0, 10);
}
function unwrapApiResult(result, fallbackMessage) {
if (result?.success === true) {
return result;
}
throw new Error(result?.error || fallbackMessage);
}
const availableLossReasons = ref([]);
const trace = useTraceProgress({ profile: 'mid_section_defect' });
const analysisData = ref({
kpi: {},
charts: {},
daily_trend: [],
genealogy_status: 'ready',
detail_total_count: 0,
});
const detailData = ref([]);
const detailPagination = ref({
page: 1,
page_size: PAGE_SIZE,
total_count: 0,
total_pages: 1,
});
const detailLoading = ref(false);
const loading = reactive({
querying: false,
});
const hasQueried = ref(false);
const queryError = ref('');
const hasTraceError = computed(() => (
Boolean(trace.stage_errors.seed)
|| Boolean(trace.stage_errors.lineage)
|| Boolean(trace.stage_errors.events)
));
const showTraceProgress = computed(() => (
loading.querying
|| trace.completed_stages.value.length > 0
|| hasTraceError.value
));
const eventsAggregation = computed(() => trace.stage_results.events?.aggregation || null);
const showAnalysisSkeleton = computed(() => hasQueried.value && loading.querying && !eventsAggregation.value);
const showAnalysisCharts = computed(() => hasQueried.value && Boolean(eventsAggregation.value));
function emptyAnalysisData() {
return {
kpi: {},
charts: {},
daily_trend: [],
genealogy_status: 'ready',
detail_total_count: 0,
};
}
function setDefaultDates() {
const today = new Date();
const endDate = new Date(today);
endDate.setDate(endDate.getDate() - 1);
const startDate = new Date(endDate);
startDate.setDate(startDate.getDate() - 6);
filters.startDate = toDateString(startDate);
filters.endDate = toDateString(endDate);
}
function toDateString(value) {
return value.toISOString().slice(0, 10);
}
function unwrapApiResult(result, fallbackMessage) {
if (result?.success === true) {
return result;
}
throw new Error(result?.error || fallbackMessage);
}
function buildFilterParams() {
const snapshot = committedFilters.value;
const params = {
start_date: snapshot.startDate,
end_date: snapshot.endDate,
};
if (snapshot.lossReasons.length) {
params.loss_reasons = snapshot.lossReasons;
}
return params;
}
function buildDetailParams() {
const snapshot = committedFilters.value;
const params = {
start_date: snapshot.startDate,
@@ -95,22 +133,27 @@ function snapshotFilters() {
lossReasons: [...filters.lossReasons],
};
}
async function loadLossReasons() {
try {
const result = await apiGet('/api/mid-section-defect/loss-reasons');
const unwrapped = unwrapApiResult(result, '載入不良原因失敗');
availableLossReasons.value = unwrapped.data?.loss_reasons || [];
} catch {
// Non-blocking — dropdown will be empty until first query
}
}
function firstStageErrorMessage() {
const stageError = trace.stage_errors.seed || trace.stage_errors.lineage || trace.stage_errors.events;
return stageError?.message || '';
}
async function loadLossReasons() {
try {
const result = await apiGet('/api/mid-section-defect/loss-reasons');
const unwrapped = unwrapApiResult(result, '載入不良原因失敗');
availableLossReasons.value = unwrapped.data?.loss_reasons || [];
} catch {
// Non-blocking, dropdown remains empty.
}
}
async function loadDetail(page = 1, signal = null) {
detailLoading.value = true;
try {
const params = {
...buildFilterParams(),
...buildDetailParams(),
page,
page_size: PAGE_SIZE,
};
@@ -119,11 +162,14 @@ async function loadDetail(page = 1, signal = null) {
timeout: API_TIMEOUT,
signal,
});
const unwrapped = unwrapApiResult(result, '載入明細失敗');
detailData.value = unwrapped.data?.detail || [];
detailPagination.value = unwrapped.data?.pagination || {
page: 1, page_size: PAGE_SIZE, total_count: 0, total_pages: 1,
};
const unwrapped = unwrapApiResult(result, '載入明細失敗');
detailData.value = unwrapped.data?.detail || [];
detailPagination.value = unwrapped.data?.pagination || {
page: 1,
page_size: PAGE_SIZE,
total_count: 0,
total_pages: 1,
};
} catch (err) {
if (err?.name === 'AbortError') {
return;
@@ -133,35 +179,40 @@ async function loadDetail(page = 1, signal = null) {
} finally {
detailLoading.value = false;
}
}
}
async function loadAnalysis() {
queryError.value = '';
trace.abort();
trace.reset();
loading.querying = true;
const signal = createAbortSignal('msd-analysis');
hasQueried.value = true;
analysisData.value = emptyAnalysisData();
try {
const params = buildFilterParams();
await trace.execute(params);
// Fire summary and detail page 1 in parallel
const [summaryResult] = await Promise.all([
apiGet('/api/mid-section-defect/analysis', {
params,
timeout: API_TIMEOUT,
signal,
}),
loadDetail(1, signal),
]);
const unwrapped = unwrapApiResult(summaryResult, '查詢失敗');
analysisData.value = unwrapped.data;
hasQueried.value = true;
// Start auto-refresh after first successful query
if (!autoRefreshStarted) {
autoRefreshStarted = true;
startAutoRefresh();
}
if (eventsAggregation.value) {
analysisData.value = {
...analysisData.value,
...eventsAggregation.value,
};
}
const stageError = firstStageErrorMessage();
if (stageError) {
queryError.value = stageError;
}
if (!stageError || trace.completed_stages.value.includes('events')) {
await loadDetail(1, createAbortSignal('msd-detail'));
}
if (!autoRefreshStarted) {
autoRefreshStarted = true;
startAutoRefresh();
}
} catch (err) {
if (err?.name === 'AbortError') {
return;
@@ -171,24 +222,24 @@ async function loadAnalysis() {
loading.querying = false;
}
}
function handleUpdateFilters(updated) {
Object.assign(filters, updated);
}
function handleUpdateFilters(updated) {
Object.assign(filters, updated);
}
function handleQuery() {
snapshotFilters();
loadAnalysis();
void loadAnalysis();
}
function prevPage() {
if (detailPagination.value.page <= 1) return;
loadDetail(detailPagination.value.page - 1, createAbortSignal('msd-detail'));
void loadDetail(detailPagination.value.page - 1, createAbortSignal('msd-detail'));
}
function nextPage() {
if (detailPagination.value.page >= detailPagination.value.total_pages) return;
loadDetail(detailPagination.value.page + 1, createAbortSignal('msd-detail'));
void loadDetail(detailPagination.value.page + 1, createAbortSignal('msd-detail'));
}
function exportCsv() {
@@ -202,7 +253,7 @@ function exportCsv() {
}
const link = document.createElement('a');
link.href = `/api/mid-section-defect/export?${params}`;
link.href = `/api/mid-section-defect/export?${params.toString()}`;
link.download = `mid_section_defect_${snapshot.startDate}_to_${snapshot.endDate}.csv`;
document.body.appendChild(link);
link.click();
@@ -211,7 +262,10 @@ function exportCsv() {
let autoRefreshStarted = false;
const { createAbortSignal, startAutoRefresh } = useAutoRefresh({
onRefresh: () => loadAnalysis(),
onRefresh: async () => {
trace.abort();
await loadAnalysis();
},
intervalMs: 5 * 60 * 1000,
autoStart: false,
refreshOnVisible: true,
@@ -220,70 +274,87 @@ const { createAbortSignal, startAutoRefresh } = useAutoRefresh({
function initPage() {
setDefaultDates();
snapshotFilters();
loadLossReasons();
void loadLossReasons();
}
void initPage();
</script>
<template>
<div class="page-container">
<header class="page-header">
<h1>中段製程不良追溯分析</h1>
<p class="header-desc">TMTT 測試站不良回溯至上游機台 / 站點 / 製程</p>
</header>
<FilterBar
:filters="filters"
:loading="loading.querying"
:available-loss-reasons="availableLossReasons"
@update-filters="handleUpdateFilters"
@query="handleQuery"
/>
<div v-if="queryError" class="error-banner">{{ queryError }}</div>
<template v-if="hasQueried">
<div v-if="analysisData.genealogy_status === 'error'" class="warning-banner">
追溯分析未完成genealogy 查詢失敗圖表僅顯示 TMTT 站點數據
</div>
<KpiCards :kpi="analysisData.kpi" :loading="loading.querying" />
<div class="charts-section">
<div class="charts-row">
<ParetoChart title="依站點歸因" :data="analysisData.charts?.by_station" />
<ParetoChart title="依不良原因" :data="analysisData.charts?.by_loss_reason" />
</div>
<div class="charts-row">
<ParetoChart title="依上游機台歸因" :data="analysisData.charts?.by_machine" />
<ParetoChart title="依 TMTT 機台" :data="analysisData.charts?.by_tmtt_machine" />
</div>
<div class="charts-row">
<ParetoChart title="依製程 (WORKFLOW)" :data="analysisData.charts?.by_workflow" />
<ParetoChart title="依封裝 (PACKAGE)" :data="analysisData.charts?.by_package" />
</div>
<div class="charts-row charts-row-full">
<TrendChart :data="analysisData.daily_trend" />
</div>
</div>
<DetailTable
:data="detailData"
:loading="detailLoading"
:pagination="detailPagination"
@export-csv="exportCsv"
@prev-page="prevPage"
@next-page="nextPage"
/>
</template>
<div v-else-if="!loading.querying" class="empty-state">
<p>請選擇日期範圍與不良原因點擊查詢開始分析</p>
</div>
<div class="loading-overlay" :class="{ hidden: !loading.querying }">
<div class="loading-spinner"></div>
</div>
</div>
</template>
void initPage();
</script>
<template>
<div class="page-container">
<header class="page-header">
<h1>中段製程不良追溯分析</h1>
<p class="header-desc">TMTT 測試站不良回溯至上游機台 / 站點 / 製程</p>
</header>
<FilterBar
:filters="filters"
:loading="loading.querying"
:available-loss-reasons="availableLossReasons"
@update-filters="handleUpdateFilters"
@query="handleQuery"
/>
<TraceProgressBar
v-if="showTraceProgress"
:current_stage="trace.current_stage.value"
:completed_stages="trace.completed_stages.value"
:stage_errors="trace.stage_errors"
/>
<div v-if="queryError" class="error-banner">{{ queryError }}</div>
<template v-if="hasQueried">
<div v-if="analysisData.genealogy_status === 'error'" class="warning-banner">
追溯分析未完成genealogy 查詢失敗圖表僅顯示 TMTT 站點數據
</div>
<div v-if="showAnalysisSkeleton" class="trace-skeleton-section">
<div class="trace-skeleton-kpi-grid">
<div v-for="index in 6" :key="`kpi-${index}`" class="trace-skeleton-card trace-skeleton-pulse"></div>
</div>
<div class="trace-skeleton-chart-grid">
<div v-for="index in 6" :key="`chart-${index}`" class="trace-skeleton-chart trace-skeleton-pulse"></div>
<div class="trace-skeleton-chart trace-skeleton-trend trace-skeleton-pulse"></div>
</div>
</div>
<transition name="trace-fade">
<div v-if="showAnalysisCharts">
<KpiCards :kpi="analysisData.kpi" :loading="false" />
<div class="charts-section">
<div class="charts-row">
<ParetoChart title="依站點歸因" :data="analysisData.charts?.by_station" />
<ParetoChart title="依不良原因" :data="analysisData.charts?.by_loss_reason" />
</div>
<div class="charts-row">
<ParetoChart title="依上游機台歸因" :data="analysisData.charts?.by_machine" />
<ParetoChart title="依 TMTT 機台" :data="analysisData.charts?.by_tmtt_machine" />
</div>
<div class="charts-row">
<ParetoChart title="依製程 (WORKFLOW)" :data="analysisData.charts?.by_workflow" />
<ParetoChart title="依封裝 (PACKAGE)" :data="analysisData.charts?.by_package" />
</div>
<div class="charts-row charts-row-full">
<TrendChart :data="analysisData.daily_trend" />
</div>
</div>
</div>
</transition>
<DetailTable
:data="detailData"
:loading="detailLoading"
:pagination="detailPagination"
@export-csv="exportCsv"
@prev-page="prevPage"
@next-page="nextPage"
/>
</template>
<div v-else-if="!loading.querying" class="empty-state">
<p>請選擇日期範圍與不良原因點擊查詢開始分析</p>
</div>
</div>
</template>

View File

@@ -456,43 +456,79 @@ body {
}
/* ====== Empty State ====== */
.empty-state {
display: flex;
align-items: center;
justify-content: center;
padding: 80px 20px;
color: var(--msd-muted);
font-size: 15px;
}
/* ====== Loading Overlay ====== */
.loading-overlay {
position: fixed;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
background: rgba(255, 255, 255, 0.7);
z-index: 999;
transition: opacity 0.2s;
}
.loading-overlay.hidden {
opacity: 0;
pointer-events: none;
}
.loading-spinner {
width: 40px;
height: 40px;
border: 3px solid #e2e8f0;
border-top-color: var(--msd-primary);
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
@keyframes spin {
to {
transform: rotate(360deg);
}
}
.empty-state {
display: flex;
align-items: center;
justify-content: center;
padding: 80px 20px;
color: var(--msd-muted);
font-size: 15px;
}
/* ====== Trace Skeleton ====== */
.trace-skeleton-section {
margin-bottom: 16px;
}
.trace-skeleton-kpi-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
gap: 12px;
margin-bottom: 12px;
}
.trace-skeleton-card {
min-height: 100px;
border-radius: 10px;
background: #e5eaf2;
box-shadow: var(--msd-shadow);
}
.trace-skeleton-chart-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 12px;
}
.trace-skeleton-chart {
min-height: 300px;
border-radius: 10px;
background: #e5eaf2;
box-shadow: var(--msd-shadow);
}
.trace-skeleton-trend {
grid-column: 1 / -1;
}
.trace-skeleton-pulse {
animation: trace-skeleton-pulse 1.2s ease-in-out infinite;
}
.trace-fade-enter-active,
.trace-fade-leave-active {
transition: opacity 0.3s ease-in;
}
.trace-fade-enter-from,
.trace-fade-leave-to {
opacity: 0;
}
@keyframes trace-skeleton-pulse {
0% {
opacity: 0.5;
}
50% {
opacity: 1;
}
100% {
opacity: 0.5;
}
}
@media (max-width: 1200px) {
.trace-skeleton-chart-grid {
grid-template-columns: 1fr;
}
}

View File

@@ -1,5 +1,5 @@
<script setup>
import { computed, onMounted } from 'vue';
import { computed, onMounted, ref } from 'vue';
import MultiSelect from '../resource-shared/components/MultiSelect.vue';
import FilterToolbar from '../shared-ui/components/FilterToolbar.vue';
@@ -20,12 +20,15 @@ const {
resetEquipmentDateRange,
bootstrap,
resolveLots,
loadLotLineage,
loadLotHistory,
loadAssociations,
queryEquipmentPeriod,
exportCurrentCsv,
} = useQueryToolData();
const expandedLineageIds = ref(new Set());
const equipmentOptions = computed(() =>
equipment.options.map((item) => ({
value: item.RESOURCEID,
@@ -47,6 +50,50 @@ function formatCell(value) {
return String(value);
}
function rowContainerId(row) {
return String(row?.container_id || row?.CONTAINERID || '').trim();
}
function isLineageExpanded(containerId) {
return expandedLineageIds.value.has(containerId);
}
function lineageState(containerId) {
return batch.lineageCache[containerId] || null;
}
function lineageAncestors(containerId) {
const state = lineageState(containerId);
const values = state?.ancestors?.[containerId];
if (!Array.isArray(values)) {
return [];
}
return values;
}
async function handleResolveLots() {
expandedLineageIds.value = new Set();
await resolveLots();
}
function toggleLotLineage(row) {
const containerId = rowContainerId(row);
if (!containerId) {
return;
}
const next = new Set(expandedLineageIds.value);
if (next.has(containerId)) {
next.delete(containerId);
expandedLineageIds.value = next;
return;
}
next.add(containerId);
expandedLineageIds.value = next;
void loadLotLineage(containerId);
}
onMounted(async () => {
hydrateFromUrl();
if (!equipment.startDate || !equipment.endDate) {
@@ -89,7 +136,7 @@ onMounted(async () => {
/>
</label>
<template #actions>
<button type="button" class="query-tool-btn query-tool-btn-primary" :disabled="loading.resolving" @click="resolveLots">
<button type="button" class="query-tool-btn query-tool-btn-primary" :disabled="loading.resolving" @click="handleResolveLots">
{{ loading.resolving ? '解析中...' : '解析' }}
</button>
</template>
@@ -110,22 +157,60 @@ onMounted(async () => {
</tr>
</thead>
<tbody>
<tr
<template
v-for="(row, index) in batch.resolvedLots"
:key="row.container_id || row.CONTAINERID || index"
:class="{ selected: batch.selectedContainerId === (row.container_id || row.CONTAINERID) }"
>
<td>
<button
type="button"
class="query-tool-btn query-tool-btn-ghost"
@click="loadLotHistory(row.container_id || row.CONTAINERID)"
>
載入歷程
</button>
</td>
<td v-for="column in resolvedColumns" :key="column">{{ formatCell(row[column]) }}</td>
</tr>
<tr :class="{ selected: batch.selectedContainerId === (row.container_id || row.CONTAINERID) }">
<td>
<div class="query-tool-row-actions">
<button
type="button"
class="query-tool-btn query-tool-btn-ghost"
@click="loadLotHistory(rowContainerId(row))"
>
載入歷程
</button>
<button
type="button"
class="query-tool-btn query-tool-btn-ghost"
@click="toggleLotLineage(row)"
>
{{ isLineageExpanded(rowContainerId(row)) ? '收合血緣' : '展開血緣' }}
</button>
</div>
</td>
<td v-for="column in resolvedColumns" :key="column">{{ formatCell(row[column]) }}</td>
</tr>
<tr
v-if="isLineageExpanded(rowContainerId(row))"
:key="`lineage-${row.container_id || row.CONTAINERID || index}`"
class="query-tool-lineage-row"
>
<td :colspan="resolvedColumns.length + 1">
<div v-if="lineageState(rowContainerId(row))?.loading" class="query-tool-empty">
血緣追溯中...
</div>
<div v-else-if="lineageState(rowContainerId(row))?.error" class="query-tool-error-inline">
{{ lineageState(rowContainerId(row)).error }}
</div>
<div v-else-if="lineageAncestors(rowContainerId(row)).length === 0" class="query-tool-empty">
無上游血緣資料
</div>
<div v-else class="query-tool-lineage-content">
<strong>上游節點 ({{ lineageAncestors(rowContainerId(row)).length }})</strong>
<ul class="query-tool-lineage-list">
<li
v-for="ancestorId in lineageAncestors(rowContainerId(row))"
:key="`${rowContainerId(row)}-${ancestorId}`"
>
{{ ancestorId }}
</li>
</ul>
</div>
</td>
</tr>
</template>
</tbody>
</table>
</div>

View File

@@ -85,6 +85,7 @@ export function useQueryToolData() {
lotHistoryRows: [],
associationType: 'materials',
associationRows: [],
lineageCache: {},
});
const equipment = reactive({
@@ -185,6 +186,7 @@ export function useQueryToolData() {
batch.selectedContainerId = '';
batch.lotHistoryRows = [];
batch.associationRows = [];
batch.lineageCache = {};
syncBatchUrlState();
try {
const payload = await apiPost(
@@ -236,6 +238,52 @@ export function useQueryToolData() {
}
}
async function loadLotLineage(containerId) {
const id = String(containerId || '').trim();
if (!id) {
return false;
}
const cached = batch.lineageCache[id];
if (cached?.loading || cached?.ancestors) {
return true;
}
batch.lineageCache[id] = {
ancestors: null,
merges: null,
loading: true,
error: '',
};
try {
const payload = await apiPost(
'/api/trace/lineage',
{
profile: 'query_tool',
container_ids: [id],
},
{ timeout: 60000, silent: true },
);
batch.lineageCache[id] = {
ancestors: payload?.ancestors || {},
merges: payload?.merges || {},
loading: false,
error: '',
};
return true;
} catch (error) {
batch.lineageCache[id] = {
ancestors: null,
merges: null,
loading: false,
error: error?.message || '血緣查詢失敗',
};
return false;
}
}
async function loadAssociations() {
if (!batch.selectedContainerId) {
errorMessage.value = '請先選擇一筆 CONTAINERID';
@@ -391,6 +439,7 @@ export function useQueryToolData() {
resetEquipmentDateRange,
bootstrap,
resolveLots,
loadLotLineage,
loadLotHistory,
loadAssociations,
queryEquipmentPeriod,

View File

@@ -80,6 +80,12 @@
color: #334155;
}
.query-tool-row-actions {
display: flex;
gap: 6px;
align-items: center;
}
.query-tool-table-wrap {
margin-top: 12px;
overflow: auto;
@@ -112,6 +118,34 @@
background: #eef2ff;
}
.query-tool-lineage-row td {
background: #f8fafc;
border-top: 1px dashed #dbeafe;
}
.query-tool-lineage-content {
display: flex;
flex-direction: column;
gap: 8px;
padding: 6px 0;
}
.query-tool-lineage-list {
margin: 0;
padding-left: 18px;
color: #334155;
font-size: 12px;
}
.query-tool-error-inline {
padding: 10px;
border-radius: 8px;
background: #fef2f2;
border: 1px solid #fecaca;
color: #b91c1c;
font-size: 12px;
}
.query-tool-empty {
margin: 0;
padding: 12px;

View File

@@ -0,0 +1,166 @@
<script setup>
import { computed } from 'vue';
const props = defineProps({
current_stage: {
type: String,
default: null,
},
completed_stages: {
type: Array,
default: () => [],
},
stage_errors: {
type: Object,
default: () => ({}),
},
});
const STAGES = Object.freeze([
{ id: 'seed-resolve', key: 'seed', label: '批次解析' },
{ id: 'lineage', key: 'lineage', label: '血緣追溯' },
{ id: 'events', key: 'events', label: '事件查詢' },
]);
const completedSet = computed(() => new Set(props.completed_stages || []));
function hasStageError(stage) {
const error = props.stage_errors?.[stage.key];
return Boolean(error?.message || error?.code);
}
function stageState(stage) {
if (hasStageError(stage)) {
return 'error';
}
if (completedSet.value.has(stage.id)) {
return 'complete';
}
if (props.current_stage === stage.id) {
return 'active';
}
return 'pending';
}
const firstError = computed(() => {
for (const stage of STAGES) {
const error = props.stage_errors?.[stage.key];
if (error?.message) {
return `[${stage.label}] ${error.message}`;
}
}
return '';
});
</script>
<template>
<div class="trace-progress">
<div class="trace-progress-track">
<div
v-for="(stage, index) in STAGES"
:key="stage.id"
class="trace-progress-step"
:class="`is-${stageState(stage)}`"
>
<div class="trace-progress-dot"></div>
<span class="trace-progress-label">{{ stage.label }}</span>
<div v-if="index < STAGES.length - 1" class="trace-progress-line"></div>
</div>
</div>
<p v-if="firstError" class="trace-progress-error">{{ firstError }}</p>
</div>
</template>
<style scoped>
.trace-progress {
margin: 12px 0 16px;
padding: 12px 14px;
border: 1px solid #dbeafe;
border-radius: 10px;
background: #f8fbff;
}
.trace-progress-track {
display: flex;
gap: 0;
}
.trace-progress-step {
display: flex;
align-items: center;
flex: 1;
min-width: 0;
color: #94a3b8;
}
.trace-progress-dot {
width: 10px;
height: 10px;
border-radius: 999px;
background: currentColor;
flex-shrink: 0;
}
.trace-progress-label {
margin-left: 8px;
font-size: 12px;
font-weight: 600;
white-space: nowrap;
}
.trace-progress-line {
height: 2px;
background: currentColor;
opacity: 0.35;
flex: 1;
margin: 0 10px;
}
.trace-progress-step.is-complete {
color: #16a34a;
}
.trace-progress-step.is-active {
color: #2563eb;
}
.trace-progress-step.is-active .trace-progress-dot {
animation: trace-progress-pulse 1s ease-in-out infinite;
}
.trace-progress-step.is-error {
color: #dc2626;
}
.trace-progress-error {
margin: 10px 0 0;
color: #b91c1c;
font-size: 12px;
}
@keyframes trace-progress-pulse {
0% {
transform: scale(1);
opacity: 1;
}
50% {
transform: scale(1.35);
opacity: 0.6;
}
100% {
transform: scale(1);
opacity: 1;
}
}
@media (max-width: 840px) {
.trace-progress-track {
flex-direction: column;
gap: 8px;
}
.trace-progress-line {
display: none;
}
}
</style>

View File

@@ -2,3 +2,4 @@ export { useAutoRefresh } from './useAutoRefresh.js';
export { useAutocomplete } from './useAutocomplete.js';
export { usePaginationState } from './usePaginationState.js';
export { readQueryState, writeQueryState } from './useQueryState.js';
export { useTraceProgress } from './useTraceProgress.js';

View File

@@ -0,0 +1,181 @@
import { reactive, ref } from 'vue';
import { apiPost, ensureMesApiAvailable } from '../core/api.js';
ensureMesApiAvailable();
const DEFAULT_STAGE_TIMEOUT_MS = 60000;
const PROFILE_DOMAINS = Object.freeze({
query_tool: ['history', 'materials', 'rejects', 'holds', 'jobs'],
mid_section_defect: ['upstream_history'],
});
function stageKey(stageName) {
if (stageName === 'seed-resolve') return 'seed';
if (stageName === 'lineage') return 'lineage';
return 'events';
}
function normalizeSeedContainerIds(seedPayload) {
const rows = Array.isArray(seedPayload?.seeds) ? seedPayload.seeds : [];
const seen = new Set();
const containerIds = [];
rows.forEach((row) => {
const id = String(row?.container_id || '').trim();
if (!id || seen.has(id)) {
return;
}
seen.add(id);
containerIds.push(id);
});
return containerIds;
}
function collectAllContainerIds(seedContainerIds, lineagePayload) {
const seen = new Set(seedContainerIds);
const merged = [...seedContainerIds];
const ancestors = lineagePayload?.ancestors || {};
Object.values(ancestors).forEach((values) => {
if (!Array.isArray(values)) {
return;
}
values.forEach((value) => {
const id = String(value || '').trim();
if (!id || seen.has(id)) {
return;
}
seen.add(id);
merged.push(id);
});
});
return merged;
}
export function useTraceProgress({ profile } = {}) {
const current_stage = ref(null);
const completed_stages = ref([]);
const is_running = ref(false);
const stage_results = reactive({
seed: null,
lineage: null,
events: null,
});
const stage_errors = reactive({
seed: null,
lineage: null,
events: null,
});
let activeController = null;
function reset() {
completed_stages.value = [];
current_stage.value = null;
stage_results.seed = null;
stage_results.lineage = null;
stage_results.events = null;
stage_errors.seed = null;
stage_errors.lineage = null;
stage_errors.events = null;
}
function abort() {
if (activeController) {
activeController.abort();
activeController = null;
}
}
async function execute(params = {}) {
const domains = PROFILE_DOMAINS[profile];
if (!domains) {
throw new Error(`Unsupported trace profile: ${profile}`);
}
abort();
reset();
is_running.value = true;
const controller = new AbortController();
activeController = controller;
try {
current_stage.value = 'seed-resolve';
const seedPayload = await apiPost(
'/api/trace/seed-resolve',
{ profile, params },
{ timeout: DEFAULT_STAGE_TIMEOUT_MS, signal: controller.signal },
);
stage_results.seed = seedPayload;
completed_stages.value = [...completed_stages.value, 'seed-resolve'];
const seedContainerIds = normalizeSeedContainerIds(seedPayload);
if (seedContainerIds.length === 0) {
return stage_results;
}
current_stage.value = 'lineage';
const lineagePayload = await apiPost(
'/api/trace/lineage',
{
profile,
container_ids: seedContainerIds,
cache_key: seedPayload?.cache_key || null,
},
{ timeout: DEFAULT_STAGE_TIMEOUT_MS, signal: controller.signal },
);
stage_results.lineage = lineagePayload;
completed_stages.value = [...completed_stages.value, 'lineage'];
const allContainerIds = collectAllContainerIds(seedContainerIds, lineagePayload);
current_stage.value = 'events';
const eventsPayload = await apiPost(
'/api/trace/events',
{
profile,
container_ids: allContainerIds,
domains,
cache_key: seedPayload?.cache_key || null,
params,
seed_container_ids: seedContainerIds,
lineage: {
ancestors: lineagePayload?.ancestors || {},
},
},
{ timeout: DEFAULT_STAGE_TIMEOUT_MS, signal: controller.signal },
);
stage_results.events = eventsPayload;
completed_stages.value = [...completed_stages.value, 'events'];
return stage_results;
} catch (error) {
if (error?.name === 'AbortError') {
return stage_results;
}
const key = stageKey(current_stage.value);
stage_errors[key] = {
code: error?.errorCode || null,
message: error?.message || '追溯查詢失敗',
};
return stage_results;
} finally {
if (activeController === controller) {
activeController = null;
}
current_stage.value = null;
is_running.value = false;
}
}
return {
current_stage,
completed_stages,
stage_results,
stage_errors,
is_running,
execute,
reset,
abort,
};
}

View File

@@ -11,7 +11,7 @@ const props = defineProps({
const emit = defineEmits(['drilldown']);
const workcenters = computed(() => props.data?.workcenters || []);
const packages = computed(() => (props.data?.packages || []).slice(0, 15));
const packages = computed(() => props.data?.packages || []);
function formatNumber(value) {
if (!value) {

View File

@@ -0,0 +1,2 @@
schema: spec-driven
status: proposal

View File

@@ -0,0 +1,202 @@
## Context
兩個高查詢複雜度頁面(`/mid-section-defect``/query-tool`)各自實作了 LOT 血緣追溯邏輯。mid-section-defect 使用 Python BFS`_bfs_split_chain()` + `_fetch_merge_sources()`query-tool 使用 `_build_in_filter()` 字串拼接。兩者共用的底層資料表為 `DWH.DW_MES_CONTAINER`5.2M rows, CONTAINERID UNIQUE index`DWH.DW_MES_PJ_COMBINEDASSYLOTS`1.97M rows, FINISHEDNAME indexed
現行問題:
- BFS 每輪一次 DB round-trip3-16 輪),加上 `genealogy_records.sql` 全掃描 `HM_LOTMOVEOUT`48M rows
- `_build_in_filter()` 字串拼接存在 SQL injection 風險
- query-tool 無 rate limit / cache可打爆 DB pool (pool_size=10, max_overflow=20)
- 兩份 service 各 1200-1300 行,血緣邏輯重複
既有安全基礎設施:
- `QueryBuilder``sql/builder.py``add_in_condition()` 支援 bind params `:p0, :p1, ...`
- `SQLLoader``sql/loader.py``load_with_params()` 支援結構參數 `{{ PARAM }}`
- `configured_rate_limit()``core/rate_limit.py`per-client rate limit with `Retry-After` header
- `LayeredCache``core/cache.py`L1 MemoryTTL + L2 Redis
## Goals / Non-Goals
**Goals:**
-`CONNECT BY NOCYCLE` 取代 Python BFS將 3-16 次 DB round-trip 縮減為 1 次
- 建立 `LineageEngine` 統一模組,消除血緣邏輯重複
- 消除 `_build_in_filter()` SQL injection 風險
- 為 query-tool 加入 rate limit + cache對齊 mid-section-defect
-`lot_split_merge_history` 加入 fast/full 雙模式
**Non-Goals:**
- 不新增 API endpoint由後續 `trace-progressive-ui` 負責)
- 不改動前端
- 不建立 materialized view / 不使用 PARALLEL hints
- 不改動其他頁面wip-detail, lot-detail 等)
## Decisions
### D1: CONNECT BY NOCYCLE 作為主要遞迴查詢策略
**選擇**: Oracle `CONNECT BY NOCYCLE` with `LEVEL <= 20`
**替代方案**: Recursive `WITH` (recursive subquery factoring)
**理由**:
- `CONNECT BY` 是 Oracle 原生遞迴語法,在 Oracle 19c 上執行計劃最佳化最成熟
- `LEVEL <= 20` 等價於現行 BFS `bfs_round > 20` 防護
- `NOCYCLE` 處理循環引用(`SPLITFROMID` 可能存在資料錯誤的循環)
- recursive `WITH` 作為 SQL 檔案內的註解替代方案,若 execution plan 不佳可快速切換
**SQL 設計**`sql/lineage/split_ancestors.sql`:
```sql
SELECT
c.CONTAINERID,
c.SPLITFROMID,
c.CONTAINERNAME,
LEVEL AS SPLIT_DEPTH
FROM DWH.DW_MES_CONTAINER c
START WITH {{ CID_FILTER }}
CONNECT BY NOCYCLE PRIOR c.SPLITFROMID = c.CONTAINERID
AND LEVEL <= 20
```
- `{{ CID_FILTER }}``QueryBuilder.get_conditions_sql()` 生成bind params 注入
- Oracle IN clause 上限透過 `ORACLE_IN_BATCH_SIZE=1000` 分批,多批結果合併
### D2: LineageEngine 模組結構
```
src/mes_dashboard/services/lineage_engine.py
├── resolve_split_ancestors(container_ids: List[str]) -> Dict
│ └── 回傳 {child_to_parent: {cid: parent_cid}, cid_to_name: {cid: name}}
├── resolve_merge_sources(container_names: List[str]) -> Dict
│ └── 回傳 {finished_name: [{source_cid, source_name}]}
└── resolve_full_genealogy(container_ids: List[str], initial_names: Dict) -> Dict
└── 組合 split + merge回傳 {cid: Set[ancestor_cids]}
src/mes_dashboard/sql/lineage/
├── split_ancestors.sql (CONNECT BY NOCYCLE)
└── merge_sources.sql (from merge_lookup.sql)
```
**函數簽名設計**:
- profile-agnostic接受 `container_ids: List[str]`,不綁定頁面邏輯
- 回傳原生 Python 資料結構dict/set不回傳 DataFrame
- 內部使用 `QueryBuilder` + `SQLLoader.load_with_params()` + `read_sql_df()`
- batch 邏輯封裝在模組內caller 不需處理 `ORACLE_IN_BATCH_SIZE`
### D3: EventFetcher 模組結構
```
src/mes_dashboard/services/event_fetcher.py
├── fetch_events(container_ids: List[str], domain: str) -> List[Dict]
│ └── 支援 domain: history, materials, rejects, holds, jobs, upstream_history
├── _cache_key(domain: str, container_ids: List[str]) -> str
│ └── 格式: evt:{domain}:{sorted_cids_hash}
└── _get_rate_limit_config(domain: str) -> Dict
└── 回傳 {bucket, max_attempts, window_seconds}
```
**快取策略**:
- L2 Redis cache對齊 `core/cache.py` 模式TTL 依 domain 配置
- cache key 使用 `hashlib.md5(sorted(cids).encode()).hexdigest()[:12]` 避免超長 key
- mid-section-defect 既有的 `_fetch_upstream_history()` 遷移到 `fetch_events(cids, "upstream_history")`
### D4: query-tool SQL injection 修復策略
**修復範圍**6 個呼叫點):
1. `_resolve_by_lot_id()` (line 262): `_build_in_filter(lot_ids, 'CONTAINERNAME')` + `read_sql_df(sql, {})`
2. `_resolve_by_serial_number()` (line ~320): 同上模式
3. `_resolve_by_work_order()` (line ~380): 同上模式
4. `get_lot_history()` 內部的 IN 子句
5. `get_lot_associations()` 內部的 IN 子句
6. `lot_split_merge_history` 查詢
**修復模式**(統一):
```python
# Before (unsafe)
in_filter = _build_in_filter(lot_ids, 'CONTAINERNAME')
sql = f"SELECT ... WHERE {in_filter}"
df = read_sql_df(sql, {})
# After (safe)
builder = QueryBuilder()
builder.add_in_condition("CONTAINERNAME", lot_ids)
sql = SQLLoader.load_with_params(
"query_tool/lot_resolve_id",
CONTAINER_FILTER=builder.get_conditions_sql(),
)
df = read_sql_df(sql, builder.params)
```
**`_build_in_filter()``_build_in_clause()` 完全刪除**(非 deprecated直接刪除因為這是安全漏洞
### D5: query-tool rate limit + cache 配置
**Rate limit**(對齊 `configured_rate_limit()` 模式):
| Endpoint | Bucket | Max/Window | Env Override |
|----------|--------|------------|-------------|
| `/resolve` | `query-tool-resolve` | 10/60s | `QT_RESOLVE_RATE_*` |
| `/lot-history` | `query-tool-history` | 20/60s | `QT_HISTORY_RATE_*` |
| `/lot-associations` | `query-tool-association` | 20/60s | `QT_ASSOC_RATE_*` |
| `/adjacent-lots` | `query-tool-adjacent` | 20/60s | `QT_ADJACENT_RATE_*` |
| `/equipment-period` | `query-tool-equipment` | 5/60s | `QT_EQUIP_RATE_*` |
| `/export-csv` | `query-tool-export` | 3/60s | `QT_EXPORT_RATE_*` |
**Cache**:
- resolve result: L2 Redis, TTL=60s, key=`qt:resolve:{input_type}:{values_hash}`
- 其他 GET endpoints: 暫不加 cache結果依賴動態 CONTAINERID 參數cache 命中率低)
### D6: lot_split_merge_history fast/full 雙模式
**Fast mode**(預設):
```sql
-- lot_split_merge_history.sql 加入條件
AND h.TXNDATE >= ADD_MONTHS(SYSDATE, -6)
...
FETCH FIRST 500 ROWS ONLY
```
**Full mode**`full_history=true`:
- SQL variant 不含時間窗和 row limit
- 使用 `read_sql_df_slow()` (120s timeout) 取代 `read_sql_df()` (55s timeout)
- Route 層透過 `request.args.get('full_history', 'false').lower() == 'true'` 判斷
### D7: 重構順序與 regression 防護
**Phase 1**: mid-section-defect較安全有 cache + distributed lock 保護)
1. 建立 `lineage_engine.py` + SQL files
2.`mid_section_defect_service.py` 中以 `LineageEngine` 取代 BFS 三函數
3. golden test 驗證 BFS vs CONNECT BY 結果一致
4. 廢棄 `genealogy_records.sql` + `split_chain.sql`(標記 deprecated
**Phase 2**: query-tool風險較高無既有保護
1. 修復所有 `_build_in_filter()``QueryBuilder`
2. 刪除 `_build_in_filter()` + `_build_in_clause()`
3. 加入 route-level rate limit
4. 加入 resolve cache
5. 加入 `lot_split_merge_history` fast/full mode
**Phase 3**: EventFetcher
1. 建立 `event_fetcher.py`
2. 遷移 `_fetch_upstream_history()``EventFetcher`
3. 遷移 query-tool event fetch paths → `EventFetcher`
## Risks / Trade-offs
| Risk | Mitigation |
|------|-----------|
| CONNECT BY 對超大血緣樹 (>10000 nodes) 可能產生不預期的 execution plan | `LEVEL <= 20` 硬上限 + SQL 檔案內含 recursive `WITH` 替代方案可快速切換 |
| golden test 覆蓋率不足導致 regression 漏網 | 選取 ≥5 個已知血緣結構的 LOT含多層 split + merge 交叉CI gate 強制通過 |
| `_build_in_filter()` 刪除後漏改呼叫點 | Phase 2 完成後 `grep -r "_build_in_filter\|_build_in_clause" src/` 必須 0 結果 |
| fast mode 6 個月時間窗可能截斷需要完整歷史的追溯 | 提供 `full_history=true` 切換完整模式,前端預設不加此參數 = fast mode |
| QueryBuilder `add_in_condition()` 對 >1000 值不自動分批 | LineageEngine 內部封裝分批邏輯(`for i in range(0, len(ids), 1000)`),呼叫者無感 |
## Migration Plan
1. **建立新模組**`lineage_engine.py`, `event_fetcher.py`, `sql/lineage/*.sql` — 無副作用,可安全部署
2. **Phase 1 切換**mid-section-defect 內部呼叫改用 `LineageEngine` — 有 cache/lock 保護regression 可透過 golden test + 手動比對驗證
3. **Phase 2 切換**query-tool 修復 + rate limit + cache — 需重新跑 query-tool 路由測試
4. **Phase 3 切換**EventFetcher 遷移 — 最後執行,影響範圍最小
5. **清理**:確認 deprecated SQL files 無引用後刪除
**Rollback**: 每個 Phase 獨立,可單獨 revert。`LineageEngine``EventFetcher` 為新模組,不影響既有程式碼直到各 Phase 的切換 commit。
## Open Questions
- `DW_MES_CONTAINER.SPLITFROMID` 欄位是否有 index若無`CONNECT BY``START WITH` 性能可能依賴全表掃描而非 CONTAINERID index。需確認 Oracle execution plan。
- `ORACLE_IN_BATCH_SIZE=1000``CONNECT BY START WITH ... IN (...)` 的行為是否與普通 `WHERE ... IN (...)` 一致?需在開發環境驗證。
- EventFetcher 的 cache TTL 各 domain 是否需要差異化(如 `upstream_history` 較長、`holds` 較短)?暫統一 300s後續視使用模式調整。

View File

@@ -0,0 +1,110 @@
## Why
批次追蹤工具 (`/query-tool`) 與中段製程不良追溯分析 (`/mid-section-defect`) 是本專案中查詢複雜度最高的兩個頁面。兩者都需要解析 LOT 血緣關係(拆批 split + 併批 merge但各自實作了獨立的追溯邏輯導致
1. **效能瓶頸**mid-section-defect 使用 Python 多輪 BFS 追溯 split chain`_bfs_split_chain()`,每次 3-16 次 DB round-trip加上 `genealogy_records.sql` 對 48M 行的 `HM_LOTMOVEOUT` 全表掃描30-120 秒)。
2. **安全風險**query-tool 的 `_build_in_filter()` 使用字串拼接建構 IN 子句(`query_tool_service.py:156-174``_resolve_by_lot_id()` / `_resolve_by_serial_number()` / `_resolve_by_work_order()` 系列函數傳入空 params `read_sql_df(sql, {})`——值直接嵌入 SQL 字串中,存在 SQL 注入風險。
3. **缺乏防護**query-tool 無 rate limit、無 cache高併發時可打爆 DB connection poolProduction pool_size=10, max_overflow=20
4. **重複程式碼**:兩個 service 各自維護 split chain 追溯、merge lookup、batch IN 分段等相同邏輯。
Oracle 19c 的 `CONNECT BY NOCYCLE` 可以用一條 SQL 取代整套 Python BFS將 3-16 次 DB round-trip 縮減為 1 次。備選方案為 Oracle 19c 支援的 recursive `WITH` (recursive subquery factoring)功能等價但可讀性更好。split/merge 的資料來源 (`DW_MES_CONTAINER.SPLITFROMID` + `DW_MES_PJ_COMBINEDASSYLOTS`) 完全不需碰 `HM_LOTMOVEOUT`,可消除 48M 行全表掃描。
**邊界聲明**:本變更為純後端內部重構,不新增任何 API endpoint不改動前端。既有 API contract 向下相容URL、request/response 格式不變),僅新增可選的 `full_history` query param 作為向下相容擴展。後續的前端分段載入和新增 API endpoints 列入獨立的 `trace-progressive-ui` 變更。
## What Changes
- 建立統一的 `LineageEngine` 模組(`src/mes_dashboard/services/lineage_engine.py`),提供 LOT 血緣解析共用核心:
- `resolve_split_ancestors()` — 使用 `CONNECT BY NOCYCLE` 單次 SQL 查詢取代 Python BFS備選: recursive `WITH`,於 SQL 檔案中以註解標註替代寫法)
- `resolve_merge_sources()` — 從 `DW_MES_PJ_COMBINEDASSYLOTS` 查詢併批來源
- `resolve_full_genealogy()` — 組合 split + merge 為完整血緣圖
- 設計為 profile-agnostic 的公用函數未來其他頁面wip-detail、lot-detail可直接呼叫但本變更僅接入 mid-section-defect 和 query-tool
- 建立統一的 `EventFetcher` 模組,提供帶 cache + rate limit 的批次事件查詢,封裝既有的 domain 查詢history、materials、rejects、holds、jobs、upstream_history
- 重構 `mid_section_defect_service.py`:以 `LineageEngine` 取代 `_bfs_split_chain()` + `_fetch_merge_sources()` + `_resolve_full_genealogy()`;以 `EventFetcher` 取代 `_fetch_upstream_history()`
- 重構 `query_tool_service.py`:以 `QueryBuilder` bind params 全面取代 `_build_in_filter()` 字串拼接;加入 route-level rate limit 和 cache 對齊 mid-section-defect 既有模式。
- 新增 SQL 檔案:
- `sql/lineage/split_ancestors.sql`CONNECT BY NOCYCLE 實作,檔案內包含 recursive WITH 替代寫法作為 Oracle 版本兼容備註)
- `sql/lineage/merge_sources.sql`(從 `sql/mid_section_defect/merge_lookup.sql` 遷移)
- 廢棄 SQL 檔案(標記 deprecated保留一個版本後刪除
- `sql/mid_section_defect/genealogy_records.sql`48M row HM_LOTMOVEOUT 全掃描不再需要)
- `sql/mid_section_defect/split_chain.sql`(由 lineage CONNECT BY 取代)
- 為 query-tool 的 `lot_split_merge_history.sql` 加入雙模式查詢:
- **fast mode**(預設):`TXNDATE >= ADD_MONTHS(SYSDATE, -6)` + `FETCH FIRST 500 ROWS ONLY`——涵蓋近半年追溯,回應 <5s
- **full mode**前端傳入 `full_history=true` 時不加時間窗保留完整歷史追溯能力 `read_sql_df_slow` (120s timeout)
- query-tool route 新增 `full_history` boolean query paramservice 依此選擇 SQL variant
## Capabilities
### New Capabilities
- `lineage-engine-core`: 統一 LOT 血緣解析引擎提供 `resolve_split_ancestors()`CONNECT BY NOCYCLE`LEVEL <= 20` 上限)、`resolve_merge_sources()``resolve_full_genealogy()` 三個公用函數全部使用 `QueryBuilder` bind params支援批次 IN 分段`ORACLE_IN_BATCH_SIZE=1000`)。函數簽名設計為 profile-agnostic接受 `container_ids: List[str]` 並回傳字典結構不綁定特定頁面邏輯
- `event-fetcher-unified`: 統一事件查詢層封裝 cache key 生成格式: `evt:{domain}:{sorted_cids_hash}`)、L1/L2 layered cache對齊 `core/cache.py` LayeredCache 模式)、rate limit bucket 配置對齊 `configured_rate_limit()` 模式)。domain 包含 `history``materials``rejects``holds``jobs``upstream_history`
- `query-tool-safety-hardening`: 修復 query-tool SQL 注入風險——`_build_in_filter()` `_build_in_clause()` 全面改用 `QueryBuilder.add_in_condition()`消除 `read_sql_df(sql, {})` params 模式加入 route-level rate limit對齊 `configured_rate_limit()` 模式resolve 10/min, history 20/min, association 20/min response cacheL2 Redis, 60s TTL)。
### Modified Capabilities
- `cache-indexed-query-acceleration`: mid-section-defect genealogy 查詢從 Python BFS 多輪 + HM_LOTMOVEOUT 全掃描改為 CONNECT BY 單輪 + 索引查詢
- `oracle-query-fragment-governance`: `_build_in_filter()` / `_build_in_clause()` 廢棄統一收斂到 `QueryBuilder.add_in_condition()`新增 `sql/lineage/` 目錄遵循既有 SQLLoader 慣例
## Impact
- **Affected code**:
- 新建: `src/mes_dashboard/services/lineage_engine.py`, `src/mes_dashboard/sql/lineage/split_ancestors.sql`, `src/mes_dashboard/sql/lineage/merge_sources.sql`
- 重構: `src/mes_dashboard/services/mid_section_defect_service.py` (1194L), `src/mes_dashboard/services/query_tool_service.py` (1329L), `src/mes_dashboard/routes/query_tool_routes.py`
- 廢棄: `src/mes_dashboard/sql/mid_section_defect/genealogy_records.sql`, `src/mes_dashboard/sql/mid_section_defect/split_chain.sql` ( lineage 模組取代標記 deprecated 保留一版)
- 修改: `src/mes_dashboard/sql/query_tool/lot_split_merge_history.sql` (加時間窗 + row limit)
- **Runtime/deploy**: 無新依賴仍為 Flask/Gunicorn + Oracle + RedisDB query pattern 改變但 connection pool 設定不變
- **APIs/pages**: `/query-tool` `/mid-section-defect` 既有 API contract 向下相容——URL輸入輸出格式HTTP status code 均不變純內部實作替換向下相容的擴展query-tool API 新增 rate limit header`Retry-After`對齊 `rate_limit.py` 既有實作query-tool split-merge history 新增可選 `full_history` query param預設 false = fast mode不傳時行為與舊版等價)。
- **Performance**: 見下方 Verification 章節的量化驗收基準
- **Security**: query-tool IN clause SQL injection 風險消除所有 `_build_in_filter()` / `_build_in_clause()` 呼叫點改為 `QueryBuilder.add_in_condition()`
- **Testing**: 需新增 LineageEngine 單元測試並建立 golden test 比對 BFS vs CONNECT BY 結果一致性既有 mid-section-defect query-tool 測試需更新 mock 路徑
## Verification
效能驗收基準——所有指標須在以下條件下量測
**測試資料規模**:
- LOT 血緣樹: 目標 seed lot 具備 3 split depth、≥50 ancestor nodes至少 1 merge path
- mid-section-defect: 使用 TMTT detection 產出 10 seed lots 的日期範圍查詢
- query-tool: resolve 結果 20 lots work order 查詢
**驗收指標**冷查詢 = cache miss熱查詢 = L2 Redis hit:
| 指標 | 現況 (P95) | 目標 (P95) | 條件 |
|------|-----------|-----------|------|
| mid-section-defect genealogy | 30-120s | 8s | CONNECT BY 單輪,≥50 ancestor nodes |
| mid-section-defect genealogy | 3-5s (L2 hit) | 1s | Redis cache hit |
| query-tool lot_split_merge_history fast mode | 無上限 >120s timeout | ≤5s | 時間窗 6 個月 + FETCH FIRST 500 ROWS |
| query-tool lot_split_merge_history full mode | 同上 | ≤60s | 無時間窗,走 `read_sql_df_slow` 120s timeout |
| LineageEngine.resolve_split_ancestors | N/A (新模組) | ≤3s | ≥50 ancestor nodes, CONNECT BY |
| DB connection 佔用時間 | 3-16 round-trips × 0.5-2s each | 單次 ≤3s | 單一 CONNECT BY 查詢 |
**安全驗收**:
- `_build_in_filter()``_build_in_clause()` 零引用grep 確認)
- 所有含使用者輸入的查詢resolve_by_lot_id, resolve_by_serial_number, resolve_by_work_order 等)必須使用 `QueryBuilder` bind params不可字串拼接。純靜態 SQL無使用者輸入允許空 params
**結果一致性驗收**:
- Golden test: 選取 ≥5 個已知血緣結構的 LOT比對 BFS vs CONNECT BY 輸出的 `child_to_parent``cid_to_name` 結果集合完全一致
## Non-Goals
- 前端 UI 改動不在此變更範圍內(前端分段載入和漸進式 UX 列入後續 `trace-progressive-ui` 變更)。
- 不新增任何 API endpoint——既有 API contract 向下相容(僅新增可選 query param `full_history` 作為擴展)。新增 endpoint 由後續 `trace-progressive-ui` 負責。
- 不改動 DB schema、不建立 materialized view、不使用 PARALLEL hints——所有最佳化在應用層SQL 改寫 + Python 重構 + Redis cache完成。
- 不改動其他頁面wip-detail、lot-detail 等)的查詢邏輯——`LineageEngine` 設計為可擴展,但本變更僅接入兩個目標頁面。
- 不使用 Oracle PARALLEL hints在 connection pool 環境下行為不可預測,不做為最佳化手段)。
## Dependencies
- 無前置依賴。本變更可獨立實施。
- 後續 `trace-progressive-ui` 依賴本變更完成後的 `LineageEngine``EventFetcher` 模組。
## Risks
| 風險 | 緩解 |
|------|------|
| CONNECT BY 遇超大血緣樹(>10000 ancestors效能退化 | `LEVEL <= 20` 上限 + `NOCYCLE` 防循環;與目前 BFS `bfs_round > 20` 等效。若 Oracle 19c 執行計劃不佳SQL 檔案內含 recursive `WITH` 替代寫法可快速切換 |
| 血緣結果與 BFS 版本不一致regression | 建立 golden test用 ≥5 個已知 LOT 比對 BFS vs CONNECT BY 輸出CI gate 確保結果集合完全一致 |
| 重構範圍橫跨兩個大 service2500+ 行) | 分階段:先重構 mid-section-defect有 cache+lock 保護regression 風險較低),再做 query-tool |
| `genealogy_records.sql` 廢棄後遺漏引用 | grep 全域搜索確認無其他引用點SQL file 標記 deprecated 保留一個版本後刪除 |
| query-tool 新增 rate limit 影響使用者體驗 | 預設值寬鬆resolve 10/min, history 20/min與 mid-section-defect 既有 rate limit 對齊,回應包含 `Retry-After` header |
| `QueryBuilder` 取代 `_build_in_filter()` 時漏改呼叫點 | grep 搜索 `_build_in_filter``_build_in_clause` 所有引用,逐一替換並確認 0 殘留引用 |

View File

@@ -0,0 +1,18 @@
## ADDED Requirements
### Requirement: Mid-section defect genealogy SHALL use CONNECT BY instead of Python BFS
The mid-section-defect genealogy resolution SHALL use `LineageEngine.resolve_full_genealogy()` (CONNECT BY NOCYCLE) instead of the existing `_bfs_split_chain()` Python BFS implementation.
#### Scenario: Genealogy cold query performance
- **WHEN** mid-section-defect analysis executes genealogy resolution with cache miss
- **THEN** `LineageEngine.resolve_split_ancestors()` SHALL be called (single CONNECT BY query)
- **THEN** response time SHALL be ≤8s (P95) for ≥50 ancestor nodes
- **THEN** Python BFS `_bfs_split_chain()` SHALL NOT be called
#### Scenario: Genealogy hot query performance
- **WHEN** mid-section-defect analysis executes genealogy resolution with L2 Redis cache hit
- **THEN** response time SHALL be ≤1s (P95)
#### Scenario: Golden test result equivalence
- **WHEN** golden test runs with ≥5 known LOTs
- **THEN** CONNECT BY output (`child_to_parent`, `cid_to_name`) SHALL be identical to BFS output for the same inputs

View File

@@ -0,0 +1,20 @@
## ADDED Requirements
### Requirement: EventFetcher SHALL provide unified cached event querying across domains
`EventFetcher` SHALL encapsulate batch event queries with L1/L2 layered cache and rate limit bucket configuration, supporting domains: `history`, `materials`, `rejects`, `holds`, `jobs`, `upstream_history`.
#### Scenario: Cache miss for event domain query
- **WHEN** `EventFetcher` is called for a domain with container IDs and no cache exists
- **THEN** the domain query SHALL execute against Oracle via `read_sql_df()`
- **THEN** the result SHALL be stored in L2 Redis cache with key format `evt:{domain}:{sorted_cids_hash}`
- **THEN** L1 memory cache SHALL also be populated (aligned with `core/cache.py` LayeredCache pattern)
#### Scenario: Cache hit for event domain query
- **WHEN** `EventFetcher` is called for a domain and L2 Redis cache contains a valid entry
- **THEN** the cached result SHALL be returned without executing Oracle query
- **THEN** DB connection pool SHALL NOT be consumed
#### Scenario: Rate limit bucket per domain
- **WHEN** `EventFetcher` is used from a route handler
- **THEN** each domain SHALL have a configurable rate limit bucket aligned with `configured_rate_limit()` pattern
- **THEN** rate limit configuration SHALL be overridable via environment variables

View File

@@ -0,0 +1,57 @@
## ADDED Requirements
### Requirement: LineageEngine SHALL provide unified split ancestor resolution via CONNECT BY NOCYCLE
`LineageEngine.resolve_split_ancestors()` SHALL accept a list of container IDs and return the complete split ancestry graph using a single Oracle `CONNECT BY NOCYCLE` query on `DW_MES_CONTAINER.SPLITFROMID`.
#### Scenario: Normal split chain resolution
- **WHEN** `resolve_split_ancestors()` is called with a list of container IDs
- **THEN** a single SQL query using `CONNECT BY NOCYCLE` SHALL be executed against `DW_MES_CONTAINER`
- **THEN** the result SHALL include a `child_to_parent` mapping and a `cid_to_name` mapping for all discovered ancestor nodes
- **THEN** the traversal depth SHALL be limited to `LEVEL <= 20` (equivalent to existing BFS `bfs_round > 20` guard)
#### Scenario: Large input batch exceeding Oracle IN clause limit
- **WHEN** the input `container_ids` list exceeds `ORACLE_IN_BATCH_SIZE` (1000)
- **THEN** `QueryBuilder.add_in_condition()` SHALL batch the IDs and combine results
- **THEN** all bind parameters SHALL use `QueryBuilder.params` (no string concatenation)
#### Scenario: Cyclic split references in data
- **WHEN** `DW_MES_CONTAINER.SPLITFROMID` contains cyclic references
- **THEN** `NOCYCLE` SHALL prevent infinite traversal
- **THEN** the query SHALL return all non-cyclic ancestors up to `LEVEL <= 20`
#### Scenario: CONNECT BY performance regression
- **WHEN** Oracle 19c execution plan for `CONNECT BY NOCYCLE` performs worse than expected
- **THEN** the SQL file SHALL contain a commented-out recursive `WITH` (recursive subquery factoring) alternative that can be swapped in without code changes
### Requirement: LineageEngine SHALL provide unified merge source resolution
`LineageEngine.resolve_merge_sources()` SHALL accept a list of container IDs and return merge source mappings from `DW_MES_PJ_COMBINEDASSYLOTS`.
#### Scenario: Merge source lookup
- **WHEN** `resolve_merge_sources()` is called with container IDs
- **THEN** the result SHALL include `{cid: [merge_source_cid, ...]}` for all containers that have merge sources
- **THEN** all queries SHALL use `QueryBuilder` bind params
### Requirement: LineageEngine SHALL provide combined genealogy resolution
`LineageEngine.resolve_full_genealogy()` SHALL combine split ancestors and merge sources into a complete genealogy graph.
#### Scenario: Full genealogy for a set of seed lots
- **WHEN** `resolve_full_genealogy()` is called with seed container IDs
- **THEN** split ancestors SHALL be resolved first via `resolve_split_ancestors()`
- **THEN** merge sources SHALL be resolved for all discovered ancestor nodes
- **THEN** the combined result SHALL be equivalent to the existing `_resolve_full_genealogy()` output in `mid_section_defect_service.py`
### Requirement: LineageEngine functions SHALL be profile-agnostic
All `LineageEngine` public functions SHALL accept `container_ids: List[str]` and return dictionary structures without binding to any specific page logic.
#### Scenario: Reuse from different pages
- **WHEN** a new page (e.g., wip-detail) needs lineage resolution
- **THEN** it SHALL be able to call `LineageEngine` functions directly without modification
- **THEN** no page-specific logic (profile, TMTT detection, etc.) SHALL exist in `LineageEngine`
### Requirement: LineageEngine SQL files SHALL reside in `sql/lineage/` directory
New SQL files SHALL follow the existing `SQLLoader` convention under `src/mes_dashboard/sql/lineage/`.
#### Scenario: SQL file organization
- **WHEN** `LineageEngine` executes queries
- **THEN** `split_ancestors.sql` and `merge_sources.sql` SHALL be loaded via `SQLLoader.load_with_params("lineage/split_ancestors", ...)`
- **THEN** the SQL files SHALL NOT reference `HM_LOTMOVEOUT` (48M row table no longer needed for genealogy)

View File

@@ -0,0 +1,23 @@
## ADDED Requirements
### Requirement: Lineage SQL fragments SHALL be centralized in `sql/lineage/` directory
Split ancestor and merge source SQL queries SHALL be defined in `sql/lineage/` and shared across services via `SQLLoader`.
#### Scenario: Mid-section-defect lineage query
- **WHEN** `mid_section_defect_service.py` needs split ancestry or merge source data
- **THEN** it SHALL call `LineageEngine` which loads SQL from `sql/lineage/split_ancestors.sql` and `sql/lineage/merge_sources.sql`
- **THEN** it SHALL NOT use `sql/mid_section_defect/split_chain.sql` or `sql/mid_section_defect/genealogy_records.sql`
#### Scenario: Deprecated SQL file handling
- **WHEN** `sql/mid_section_defect/genealogy_records.sql` and `sql/mid_section_defect/split_chain.sql` are deprecated
- **THEN** the files SHALL be marked with a deprecated comment at the top
- **THEN** grep SHALL confirm zero `SQLLoader.load` references to these files
- **THEN** the files SHALL be retained for one version before deletion
### Requirement: All user-input SQL queries SHALL use QueryBuilder bind params
`_build_in_filter()` and `_build_in_clause()` in `query_tool_service.py` SHALL be fully replaced by `QueryBuilder.add_in_condition()`.
#### Scenario: Complete migration to QueryBuilder
- **WHEN** the refactoring is complete
- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results
- **THEN** all queries involving user-supplied values SHALL use `QueryBuilder.params`

View File

@@ -0,0 +1,57 @@
## ADDED Requirements
### Requirement: query-tool resolve functions SHALL use QueryBuilder bind params for all user input
All `resolve_lots()` family functions (`_resolve_by_lot_id`, `_resolve_by_serial_number`, `_resolve_by_work_order`) SHALL use `QueryBuilder.add_in_condition()` with bind parameters instead of `_build_in_filter()` string concatenation.
#### Scenario: Lot resolve with user-supplied values
- **WHEN** a resolve function receives user-supplied lot IDs, serial numbers, or work order names
- **THEN** the SQL query SHALL use `:p0, :p1, ...` bind parameters via `QueryBuilder`
- **THEN** `read_sql_df()` SHALL receive `builder.params` (never an empty `{}` dict for queries with user input)
- **THEN** `_build_in_filter()` and `_build_in_clause()` SHALL NOT be called
#### Scenario: Pure static SQL without user input
- **WHEN** a query contains no user-supplied values (e.g., static lookups)
- **THEN** empty params `{}` is acceptable
- **THEN** no `_build_in_filter()` SHALL be used
#### Scenario: Zero residual references to deprecated functions
- **WHEN** the refactoring is complete
- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results across the entire codebase
### Requirement: query-tool routes SHALL apply rate limiting
All query-tool API endpoints SHALL apply per-client rate limiting using the existing `configured_rate_limit` mechanism.
#### Scenario: Resolve endpoint rate limit exceeded
- **WHEN** a client sends more than 10 requests to query-tool resolve endpoints within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
- **THEN** the resolve service function SHALL NOT be called
#### Scenario: History endpoint rate limit exceeded
- **WHEN** a client sends more than 20 requests to query-tool history endpoints within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
#### Scenario: Association endpoint rate limit exceeded
- **WHEN** a client sends more than 20 requests to query-tool association endpoints within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
### Requirement: query-tool routes SHALL apply response caching
High-cost query-tool endpoints SHALL cache responses in L2 Redis.
#### Scenario: Resolve result caching
- **WHEN** a resolve request succeeds
- **THEN** the response SHALL be cached in L2 Redis with TTL = 60s
- **THEN** subsequent identical requests within TTL SHALL return cached result without Oracle query
### Requirement: lot_split_merge_history SHALL support fast and full query modes
The `lot_split_merge_history.sql` query SHALL support two modes to balance traceability completeness vs performance.
#### Scenario: Fast mode (default)
- **WHEN** `full_history` query parameter is absent or `false`
- **THEN** the SQL SHALL include `TXNDATE >= ADD_MONTHS(SYSDATE, -6)` time window and `FETCH FIRST 500 ROWS ONLY`
- **THEN** query response time SHALL be ≤5s (P95)
#### Scenario: Full mode
- **WHEN** `full_history=true` query parameter is provided
- **THEN** the SQL SHALL NOT include time window restriction
- **THEN** the query SHALL use `read_sql_df_slow` (120s timeout)
- **THEN** query response time SHALL be ≤60s (P95)

View File

@@ -0,0 +1,57 @@
## Phase 1: LineageEngine 模組建立
- [x] 1.1 建立 `src/mes_dashboard/sql/lineage/split_ancestors.sql`CONNECT BY NOCYCLE含 recursive WITH 註解替代方案)
- [x] 1.2 建立 `src/mes_dashboard/sql/lineage/merge_sources.sql`(從 `mid_section_defect/merge_lookup.sql` 遷移,改用 `{{ FINISHED_NAME_FILTER }}` 結構參數)
- [x] 1.3 建立 `src/mes_dashboard/services/lineage_engine.py``resolve_split_ancestors()``resolve_merge_sources()``resolve_full_genealogy()` 三個公用函數,使用 `QueryBuilder` bind params + `ORACLE_IN_BATCH_SIZE=1000` 分批
- [x] 1.4 LineageEngine 單元測試mock `read_sql_df` 驗證 batch 分割、dict 回傳結構、LEVEL <= 20 防護
## Phase 2: mid-section-defect 切換到 LineageEngine
- [x] 2.1 在 `mid_section_defect_service.py` 中以 `LineageEngine.resolve_split_ancestors()` 取代 `_bfs_split_chain()`
- [x] 2.2 以 `LineageEngine.resolve_merge_sources()` 取代 `_fetch_merge_sources()`
- [x] 2.3 以 `LineageEngine.resolve_full_genealogy()` 取代 `_resolve_full_genealogy()`
- [x] 2.4 Golden test選取 ≥5 個已知血緣結構 LOT比對 BFS vs CONNECT BY 輸出的 `child_to_parent``cid_to_name` 結果集合完全一致
- [x] 2.5 標記 `sql/mid_section_defect/genealogy_records.sql``sql/mid_section_defect/split_chain.sql` 為 deprecated檔案頂部加 `-- DEPRECATED: replaced by sql/lineage/split_ancestors.sql`
## Phase 3: query-tool SQL injection 修復
- [x] 3.1 建立 `sql/query_tool/lot_resolve_id.sql``lot_resolve_serial.sql``lot_resolve_work_order.sql` SQL 檔案(從 inline SQL 遷移到 SQLLoader 管理)
- [x] 3.2 修復 `_resolve_by_lot_id()``_build_in_filter()``QueryBuilder.add_in_condition()` + `SQLLoader.load_with_params()` + `read_sql_df(sql, builder.params)`
- [x] 3.3 修復 `_resolve_by_serial_number()`:同上模式
- [x] 3.4 修復 `_resolve_by_work_order()`:同上模式
- [x] 3.5 修復 `get_lot_history()` 內部 IN 子句:改用 `QueryBuilder`
- [x] 3.6 修復 lot-associations 查詢路徑(`get_lot_materials()` / `get_lot_rejects()` / `get_lot_holds()` / `get_lot_splits()` / `get_lot_jobs()`)中涉及使用者輸入的 IN 子句:改用 `QueryBuilder`
- [x] 3.7 修復 `lot_split_merge_history` 查詢:改用 `QueryBuilder`
- [x] 3.8 刪除 `_build_in_filter()``_build_in_clause()` 函數
- [x] 3.9 驗證:`grep -r "_build_in_filter\|_build_in_clause" src/` 回傳 0 結果
- [x] 3.10 更新既有 query-tool 路由測試的 mock 路徑
## Phase 4: query-tool rate limit + cache
- [x] 4.1 在 `query_tool_routes.py``/resolve` 加入 `configured_rate_limit(bucket='query-tool-resolve', default_max_attempts=10, default_window_seconds=60)`
- [x] 4.2 為 `/lot-history` 加入 `configured_rate_limit(bucket='query-tool-history', default_max_attempts=20, default_window_seconds=60)`
- [x] 4.3 為 `/lot-associations` 加入 `configured_rate_limit(bucket='query-tool-association', default_max_attempts=20, default_window_seconds=60)`
- [x] 4.4 為 `/adjacent-lots` 加入 `configured_rate_limit(bucket='query-tool-adjacent', default_max_attempts=20, default_window_seconds=60)`
- [x] 4.5 為 `/equipment-period` 加入 `configured_rate_limit(bucket='query-tool-equipment', default_max_attempts=5, default_window_seconds=60)`
- [x] 4.6 為 `/export-csv` 加入 `configured_rate_limit(bucket='query-tool-export', default_max_attempts=3, default_window_seconds=60)`
- [x] 4.7 為 resolve 結果加入 L2 Redis cachekey=`qt:resolve:{input_type}:{values_hash}`, TTL=60s
## Phase 5: lot_split_merge_history fast/full 雙模式
- [x] 5.1 修改 `sql/query_tool/lot_split_merge_history.sql`:加入 `{{ TIME_WINDOW }}``{{ ROW_LIMIT }}` 結構參數
- [x] 5.2 在 `query_tool_service.py` 中根據 `full_history` 參數選擇 SQL variantfast: `AND h.TXNDATE >= ADD_MONTHS(SYSDATE, -6)` + `FETCH FIRST 500 ROWS ONLY`full: 無限制 + `read_sql_df_slow`
- [x] 5.3 在 `query_tool_routes.py``/api/query-tool/lot-associations?type=splits` 路徑加入 `full_history` query param 解析,並傳遞到 split-merge-history 查詢
- [x] 5.4 路由測試:驗證 fast mode預設和 full mode`full_history=true`)的行為差異
## Phase 6: EventFetcher 模組建立
- [x] 6.1 建立 `src/mes_dashboard/services/event_fetcher.py``fetch_events(container_ids, domain)` + cache key 生成 + rate limit config
- [x] 6.2 遷移 `mid_section_defect_service.py``_fetch_upstream_history()``EventFetcher.fetch_events(cids, "upstream_history")`
- [x] 6.3 遷移 query-tool event fetch paths 到 `EventFetcher``get_lot_history``get_lot_associations` 的 DB 查詢部分)
- [x] 6.4 EventFetcher 單元測試mock DB 驗證 cache key 格式、rate limit config、domain 分支
## Phase 7: 清理與驗證
- [x] 7.1 確認 `genealogy_records.sql``split_chain.sql` 無活躍引用(`grep -r` 確認),保留 deprecated 標記
- [x] 7.2 確認所有含使用者輸入的查詢使用 `QueryBuilder` bind paramsgrep `read_sql_df` 呼叫點逐一確認)
- [x] 7.3 執行完整 query-tool 和 mid-section-defect 路由測試,確認無 regression

View File

@@ -0,0 +1,2 @@
schema: spec-driven
status: proposal

View File

@@ -0,0 +1,446 @@
## Context
`unified-lineage-engine` 完成後,後端追溯管線從 30-120 秒降至 3-8 秒。但目前的 UX 模式仍是黑盒等待——mid-section-defect 的 `/analysis` GET 一次回傳全部結果KPI + charts + trend + genealogy_statusquery-tool 雖有手動順序resolve → history → association但 lineage 查詢仍在批次載入。
既有前端架構:
- mid-section-defect: `App.vue``Promise.all([apiGet('/analysis'), loadDetail(1)])` 並行呼叫,`loading.querying` 單一布林控制整頁 loading state
- query-tool: `useQueryToolData.js` composable 管理 `loading.resolving / .history / .association / .equipment`,各自獨立但無分段進度
- 共用: `useAutoRefresh` (jittered interval + abort signal), `usePaginationState`, `apiGet/apiPost` (timeout + abort), `useQueryState` (URL sync)
- API 模式: `apiGet/apiPost` 支援 `signal: AbortSignal` + `timeout`,錯誤物件含 `error.retryAfterSeconds`
## Goals / Non-Goals
**Goals:**
- 新增 `/api/trace/*` 三段式 APIseed-resolve → lineage → events通過 `profile` 參數區分頁面行為
- 建立 `useTraceProgress` composable 封裝三段式 sequential fetch + reactive state
- mid-section-defect 漸進渲染: seed lots 先出 → 血緣 → KPI/charts fade-in
- query-tool lineage tab 改為 on-demand點擊單一 lot 後才查血緣)
- 保持 `/api/mid-section-defect/analysis` GET endpoint 向下相容
- 刪除 pre-Vite dead code `static/js/query-tool.js`
**Non-Goals:**
- 不實作 SSE / WebSocketgunicorn sync workers 限制)
- 不新增 Celery/RQ 任務隊列
- 不改動追溯計算邏輯(由 `unified-lineage-engine` 負責)
- 不改動 defect attribution 演算法
- 不改動 equipment-period 查詢
## Decisions
### D1: trace_routes.py Blueprint 架構
**選擇**: 單一 Blueprint `trace_bp`,三個 route handler + profile dispatch
**替代方案**: 每個 profile 獨立 Blueprint`trace_msd_bp`, `trace_qt_bp`
**理由**:
- 三個 endpoint 的 request/response 結構統一,僅內部呼叫邏輯依 profile 分支
- 獨立 Blueprint 會重複 rate limit / cache / error handling boilerplate
- profile 驗證集中在一處(`_validate_profile()`),新增 profile 只需加 if 分支
**路由設計**:
```python
trace_bp = Blueprint('trace', __name__, url_prefix='/api/trace')
@trace_bp.route('/seed-resolve', methods=['POST'])
@configured_rate_limit(bucket='trace-seed', default_max_attempts=10, default_window_seconds=60)
def seed_resolve():
body = request.get_json()
profile = body.get('profile')
params = body.get('params', {})
# profile dispatch → _seed_resolve_query_tool(params) or _seed_resolve_msd(params)
# return jsonify({ "stage": "seed-resolve", "seeds": [...], "seed_count": N, "cache_key": "trace:{hash}" })
@trace_bp.route('/lineage', methods=['POST'])
@configured_rate_limit(bucket='trace-lineage', default_max_attempts=10, default_window_seconds=60)
def lineage():
body = request.get_json()
container_ids = body.get('container_ids', [])
# call LineageEngine.resolve_full_genealogy(container_ids)
# return jsonify({ "stage": "lineage", "ancestors": {...}, "merges": {...}, "total_nodes": N })
@trace_bp.route('/events', methods=['POST'])
@configured_rate_limit(bucket='trace-events', default_max_attempts=15, default_window_seconds=60)
def events():
body = request.get_json()
container_ids = body.get('container_ids', [])
domains = body.get('domains', [])
profile = body.get('profile')
# call EventFetcher for each domain
# if profile == 'mid_section_defect': run aggregation
# return jsonify({ "stage": "events", "results": {...}, "aggregation": {...} | null })
```
**Profile dispatch 內部函數**:
```
_seed_resolve_query_tool(params) → 呼叫 query_tool_service 既有 resolve 邏輯
_seed_resolve_msd(params) → 呼叫 mid_section_defect_service TMTT 偵測邏輯
_aggregate_msd(events_data) → mid-section-defect 專屬 aggregation (KPI, charts, trend)
```
**Cache 策略**:
- seed-resolve: `trace:seed:{profile}:{params_hash}`, TTL=300s
- lineage: `trace:lineage:{sorted_cids_hash}`, TTL=300sprofile-agnostic因為 lineage 不依賴 profile
- events: `trace:evt:{profile}:{domains_hash}:{sorted_cids_hash}`, TTL=300s
- 使用 `LayeredCache` L2 Redis對齊 `core/cache.py` 既有模式)
- cache key hash: `hashlib.md5(sorted(values).encode()).hexdigest()[:12]`
**錯誤處理統一模式**:
```python
def _make_stage_error(stage, code, message, status=400):
return jsonify({"error": message, "code": code}), status
# Timeout 處理: 每個 stage 內部用 read_sql_df() 的 55s call_timeout
# 若超時: return _make_stage_error(stage, f"{STAGE}_TIMEOUT", "...", 504)
```
### D2: useTraceProgress composable 設計
**選擇**: 新建 `frontend/src/shared-composables/useTraceProgress.js`,封裝 sequential fetch + reactive stage state
**替代方案**: 直接在各頁面 App.vue 內實作分段 fetch
**理由**:
- 兩個頁面共用相同的三段式 fetch 邏輯
- 將 stage 狀態管理抽離,頁面只需關注渲染邏輯
- 對齊既有 `shared-composables/` 目錄結構
**Composable 簽名**:
```javascript
export function useTraceProgress({ profile, buildParams }) {
// --- Reactive State ---
const current_stage = ref(null) // 'seed-resolve' | 'lineage' | 'events' | null
const completed_stages = ref([]) // ['seed-resolve', 'lineage']
const stage_results = reactive({
seed: null, // { seeds: [], seed_count: N, cache_key: '...' }
lineage: null, // { ancestors: {...}, merges: {...}, total_nodes: N }
events: null, // { results: {...}, aggregation: {...} }
})
const stage_errors = reactive({
seed: null, // { code: '...', message: '...' }
lineage: null,
events: null,
})
const is_running = ref(false)
// --- Methods ---
async function execute(params) // 執行三段式 fetch
function reset() // 清空所有 state
function abort() // 中止進行中的 fetch
return {
current_stage,
completed_stages,
stage_results,
stage_errors,
is_running,
execute,
reset,
abort,
}
}
```
**Sequential fetch 邏輯**:
```javascript
async function execute(params) {
reset()
is_running.value = true
const abortCtrl = new AbortController()
try {
// Stage 1: seed-resolve
current_stage.value = 'seed-resolve'
const seedResult = await apiPost('/api/trace/seed-resolve', {
profile,
params,
}, { timeout: 60000, signal: abortCtrl.signal })
stage_results.seed = seedResult.data
completed_stages.value.push('seed-resolve')
if (!seedResult.data?.seeds?.length) return // 無 seed不繼續
// Stage 2: lineage
current_stage.value = 'lineage'
const cids = seedResult.data.seeds.map(s => s.container_id)
const lineageResult = await apiPost('/api/trace/lineage', {
profile,
container_ids: cids,
cache_key: seedResult.data.cache_key,
}, { timeout: 60000, signal: abortCtrl.signal })
stage_results.lineage = lineageResult.data
completed_stages.value.push('lineage')
// Stage 3: events
current_stage.value = 'events'
const allCids = _collectAllCids(cids, lineageResult.data)
const eventsResult = await apiPost('/api/trace/events', {
profile,
container_ids: allCids,
domains: _getDomainsForProfile(profile),
cache_key: seedResult.data.cache_key,
}, { timeout: 60000, signal: abortCtrl.signal })
stage_results.events = eventsResult.data
completed_stages.value.push('events')
} catch (err) {
if (err?.name === 'AbortError') return
// 記錄到當前 stage 的 error state
const stage = current_stage.value
if (stage) stage_errors[_stageKey(stage)] = { code: err.errorCode, message: err.message }
} finally {
current_stage.value = null
is_running.value = false
}
}
```
**設計重點**:
- `stage_results` 為 reactive object每個 stage 完成後立即賦值,觸發依賴該 stage 的 UI 更新
- 錯誤不拋出到頁面——記錄在 `stage_errors` 中,已完成的 stage 結果保留
- `abort()` 方法供 `useAutoRefresh` 在新一輪 refresh 前中止上一輪
- `profile` 為建構時注入(不可變),`params` 為執行時傳入(每次查詢可變)
- `cache_key` 在 stage 間傳遞,用於 logging correlation
### D3: mid-section-defect 漸進渲染策略
**選擇**: 分段渲染 + skeleton placeholders + CSS fade-in transition
**替代方案**: 保持一次性渲染(等全部 stage 完成)
**理由**:
- seed stage ≤3s 可先顯示 seed lots 數量和基本資訊
- lineage + events 完成後再填入 KPI/charts使用者感受到進度
- skeleton placeholders 避免 layout shiftchart container 預留固定高度)
**App.vue 查詢流程改造**:
```javascript
// Before (current)
async function loadAnalysis() {
loading.querying = true
const [summaryResult] = await Promise.all([
apiGet('/api/mid-section-defect/analysis', { params, timeout: 120000, signal }),
loadDetail(1, signal),
])
analysisData.value = summaryResult.data // 一次全部更新
loading.querying = false
}
// After (progressive)
const trace = useTraceProgress({ profile: 'mid_section_defect' })
async function loadAnalysis() {
const params = buildFilterParams()
// 分段 fetchseed → lineage → events+aggregation
await trace.execute(params)
// Detail 仍用舊 endpoint 分頁(不走分段 API
await loadDetail(1)
}
```
**渲染層對應**:
```
trace.completed_stages 包含 'seed-resolve'
→ 顯示 seed lots 數量 badge + 基本 filter feedback
→ KPI cards / charts / trend 顯示 skeleton
trace.completed_stages 包含 'lineage'
→ 顯示 genealogy_statusancestor 數量)
→ KPI/charts 仍為 skeleton
trace.completed_stages 包含 'events'
→ trace.stage_results.events.aggregation 不為 null
→ KPI cards 以 fade-in 填入數值
→ Pareto charts 以 fade-in 渲染
→ Trend chart 以 fade-in 渲染
```
**Skeleton Placeholder 規格**:
- KpiCards: 6 個固定高度 card frame`min-height: 100px`),灰色脈動動畫
- ParetoChart: 6 個固定高度 chart frame`min-height: 300px`),灰色脈動動畫
- TrendChart: 1 個固定高度 frame`min-height: 300px`
- fade-in: CSS transition `opacity 0→1, 300ms ease-in`
**Auto-refresh 整合**:
- `useAutoRefresh.onRefresh``trace.abort()` + `trace.execute(committedFilters)`
- 保持現行 5 分鐘 jittered interval
**Detail 分頁不變**:
- `/api/mid-section-defect/analysis/detail` GET endpoint 保持不變
- 不走分段 APIdetail 是分頁查詢,與 trace pipeline 獨立)
### D4: query-tool on-demand lineage 策略
**選擇**: per-lot on-demand fetch使用者點擊 lot card 才查血緣
**替代方案**: batch-load all lots lineage at resolve time
**理由**:
- resolve 結果可能有 20+ lots批次查全部 lineage 增加不必要的 DB 負擔
- 大部分使用者只關注特定幾個 lot 的 lineage
- per-lot fetch 控制在 ≤3s使用者體驗可接受
**useQueryToolData.js 改造**:
```javascript
// 新增 lineage state
const lineageCache = reactive({}) // { [containerId]: { ancestors, merges, loading, error } }
async function loadLotLineage(containerId) {
if (lineageCache[containerId]?.ancestors) return // 已快取
lineageCache[containerId] = { ancestors: null, merges: null, loading: true, error: null }
try {
const result = await apiPost('/api/trace/lineage', {
profile: 'query_tool',
container_ids: [containerId],
}, { timeout: 60000 })
lineageCache[containerId] = {
ancestors: result.data.ancestors,
merges: result.data.merges,
loading: false,
error: null,
}
} catch (err) {
lineageCache[containerId] = {
ancestors: null,
merges: null,
loading: false,
error: err.message,
}
}
}
```
**UI 行為**:
- lot 列表中每個 lot 有展開按鈕(或 accordion
- 點擊展開 → 呼叫 `loadLotLineage(containerId)` → 顯示 loading → 顯示 lineage tree
- 已展開的 lot 再次點擊收合(不重新 fetch
- `lineageCache` 在新一輪 `resolveLots()` 時清空
**query-tool 主流程保持不變**:
- resolve → lot-history → lot-associations 的既有流程不改
- lineage 是新增的 on-demand 功能,不取代既有功能
- query-tool 暫不使用 `useTraceProgress`(因為它的流程是使用者驅動的互動式,非自動 sequential
### D5: 進度指示器元件設計
**選擇**: 共用 `TraceProgressBar.vue` 元件props 驅動
**替代方案**: 各頁面各自實作進度顯示
**理由**:
- 兩個頁面顯示相同的 stage 進度seed → lineage → events
- 統一視覺語言
**元件設計**:
```javascript
// frontend/src/shared-composables/TraceProgressBar.vue
// (放在 shared-composables 目錄,雖然是 .vue 但與 composable 搭配使用)
props: {
current_stage: String | null, // 'seed-resolve' | 'lineage' | 'events'
completed_stages: Array, // ['seed-resolve', 'lineage']
stage_errors: Object, // { seed: null, lineage: { code, message } }
}
// 三個 step indicator:
// [●] Seed → [●] Lineage → [○] Events
// ↑ 完成(green) ↑ 進行中(blue pulse) ↑ 待處理(gray)
// ↑ 錯誤(red)
```
**Stage 顯示名稱**:
| Stage ID | 中文顯示 | 英文顯示 |
|----------|---------|---------|
| seed-resolve | 批次解析 | Resolving |
| lineage | 血緣追溯 | Lineage |
| events | 事件查詢 | Events |
**取代 loading spinner**:
- mid-section-defect: `loading.querying` 原本控制單一 spinner → 改為顯示 `TraceProgressBar`
- 進度指示器放在 filter bar 下方、結果區域上方
### D6: `/analysis` GET endpoint 向下相容橋接
**選擇**: 保留原 handler內部改為呼叫分段管線後合併結果
**替代方案**: 直接改原 handler 不經過分段管線
**理由**:
- 分段管線LineageEngine + EventFetcher`unified-lineage-engine` 完成後已是標準路徑
- 保留原 handler 確保非 portal-shell 路由 fallback 仍可用
- golden test 比對確保結果等價
**橋接邏輯**:
```python
# mid_section_defect_routes.py — /analysis handler 內部改造
@mid_section_defect_bp.route('/analysis', methods=['GET'])
@configured_rate_limit(bucket='msd-analysis', ...)
def api_analysis():
# 現行: result = mid_section_defect_service.query_analysis(start_date, end_date, loss_reasons)
# 改為: 呼叫 service 層的管線函數service 內部已使用 LineageEngine + EventFetcher
# response format 完全不變
result = mid_section_defect_service.query_analysis(start_date, end_date, loss_reasons)
return jsonify({"success": True, "data": result})
```
**實際上 `/analysis` handler 不需要改**——`unified-lineage-engine` Phase 1 已將 service 內部改為使用 `LineageEngine`。本變更只需確認 `/analysis` 回傳結果與重構前完全一致golden test 驗證),不需額外的橋接程式碼。
**Golden test 策略**:
- 選取 ≥3 組已知查詢參數(不同日期範圍、不同 loss_reasons 組合)
- 比對重構前後 `/analysis` JSON response 結構和數值
- 允許浮點數 tolerancedefect_rate 等百分比欄位 ±0.01%
### D7: Legacy static JS 清理
**選擇**: 直接刪除 `src/mes_dashboard/static/js/query-tool.js`
**理由**:
- 此檔案 3056L / 126KB是 pre-Vite 時代的靜態 JS
- `query_tool.html` template 使用 `frontend_asset('query-tool.js')` 載入 Vite 建置產物,非此靜態檔案
- Vite config 確認 entry point: `'query-tool': resolve(__dirname, 'src/query-tool/main.js')`
- `frontend_asset()` 解析 Vite manifest不會指向 `static/js/`
- grep 確認無其他引用
**驗證步驟**:
1. `grep -r "static/js/query-tool.js" src/ frontend/ templates/` → 0 結果
2. 確認 `frontend_asset('query-tool.js')` 正確解析到 Vite manifest 中的 hashed filename
3. 確認 `frontend/src/query-tool/main.js` 為 active entryVite config `input` 對應)
### D8: 實作順序
**Phase 1**: 後端 trace_routes.py無前端改動
1. 建立 `trace_routes.py` + 三個 route handler
2.`app.py` 註冊 `trace_bp` Blueprint
3. Profile dispatch functions呼叫既有 service 邏輯)
4. Rate limit + cache 配置
5. 錯誤碼 + HTTP status 對齊 spec
6. API contract 測試request/response schema 驗證)
**Phase 2**: 前端共用元件
1. 建立 `useTraceProgress.js` composable
2. 建立 `TraceProgressBar.vue` 進度指示器
3. 單元測試mock API calls驗證 stage 狀態轉換)
**Phase 3**: mid-section-defect 漸進渲染
1. `App.vue` 查詢流程改為 `useTraceProgress`
2. 加入 skeleton placeholders + fade-in transitions
3.`TraceProgressBar` 取代 loading spinner
4. 驗證 auto-refresh 整合
5. Golden test: `/analysis` 回傳結果不變
**Phase 4**: query-tool on-demand lineage
1. `useQueryToolData.js` 新增 `lineageCache` + `loadLotLineage()`
2. lot 列表加入 lineage 展開 UI
3. 驗證既有 resolve → history → association 流程不受影響
**Phase 5**: Legacy cleanup
1. 刪除 `src/mes_dashboard/static/js/query-tool.js`
2. grep 確認零引用
3. 確認 `frontend_asset()` 解析正常
## Risks / Trade-offs
| Risk | Mitigation |
|------|-----------|
| 分段 API 增加前端複雜度3 次 fetch + 狀態管理) | 封裝在 `useTraceProgress` composable頁面只需 `execute(params)` + watch `stage_results` |
| `/analysis` golden test 因浮點精度失敗 | 允許 defect_rate 等百分比 ±0.01% tolerance整數欄位嚴格比對 |
| mid-section-defect skeleton → chart 渲染閃爍 | 固定高度 placeholder + fade-in 300ms transitionchart container 不允許 height auto |
| `useTraceProgress` abort 與 `useAutoRefresh` 衝突 | auto-refresh 觸發前先呼叫 `trace.abort()`,確保上一輪 fetch 完整中止 |
| query-tool lineage per-lot fetch 對高頻展開造成 DB 壓力 | lineageCache 防止重複 fetch + trace-lineage rate limit (10/60s) 保護 |
| `static/js/query-tool.js` 刪除影響未知路徑 | grep 全域確認 0 引用 + `frontend_asset()` 確認 Vite manifest 解析正確 |
| cache_key 傳遞中斷(前端忘記傳 cache_key | cache_key 為選填,僅用於 logging correlation缺少不影響功能 |
## Open Questions
- `useTraceProgress` 是否需要支援 retry某段失敗後重試該段而非整體重新執行暫不支援——失敗後使用者重新按查詢按鈕即可。
- mid-section-defect 的 aggregation 邏輯KPI、charts、trend 計算)是放在 `/api/trace/events` 的 mid_section_defect profile 分支內,還是由前端從 raw events 自行計算?**決定: 放在後端 `/api/trace/events` 的 aggregation field**——前端不應承擔 defect attribution 計算責任,且計算邏輯已在 service 層成熟。
- `TraceProgressBar.vue` 放在 `shared-composables/` 還是獨立的 `shared-components/` 目錄?暫放 `shared-composables/`(與 composable 搭配使用),若未來 shared 元件增多再考慮拆分。

View File

@@ -0,0 +1,148 @@
## Why
`unified-lineage-engine` 完成後,後端追溯管線從 30-120 秒降至 3-8 秒但對於大範圍查詢日期跨度長、LOT 數量多)仍可能需要 5-15 秒。目前的 UX 模式是「使用者點擊查詢 → 等待黑盒 → 全部結果一次出現」,即使後端已加速,使用者仍感受不到進度,只有一個 loading spinner。
兩個頁面的前端載入模式存在差異:
- **mid-section-defect**: 一次 API call (`/analysis`) 拿全部結果KPI + charts + detail後端做完全部 4 個 stage 才回傳。
- **query-tool**: Vue 3 版本(`frontend/src/query-tool/`已有手動順序resolve → history → association但部分流程仍可改善漸進載入體驗。
需要統一兩個頁面的前端查詢體驗,實現「分段載入 + 進度可見」的 UX 模式,讓使用者看到追溯的漸進結果而非等待黑盒。
**邊界聲明**:本變更負責新增分段 API endpoints`/api/trace/*`)和前端漸進式載入 UX。後端追溯核心邏輯`LineageEngine``EventFetcher`)由前置的 `unified-lineage-engine` 變更提供,本變更僅作為 API routing layer 呼叫這些模組。
## What Changes
### 後端:新增分段 API endpoints
新增 `trace_routes.py` Blueprint`/api/trace/`),將追溯管線的每個 stage 獨立暴露為 endpoint。通過 `profile` 參數區分頁面行為:
**POST `/api/trace/seed-resolve`**
- Request: `{ "profile": "query_tool" | "mid_section_defect", "params": { ... } }`
- `query_tool` params: `{ "resolve_type": "lot_id" | "serial_number" | "work_order", "values": [...] }`
- `mid_section_defect` params: `{ "date_range": [...], "workcenter": "...", ... }` (TMTT detection 參數)
- Response: `{ "stage": "seed-resolve", "seeds": [{ "container_id": "...", "container_name": "...", "lot_id": "..." }], "seed_count": N, "cache_key": "trace:{hash}" }`
- Error: `{ "error": "...", "code": "SEED_RESOLVE_EMPTY" | "SEED_RESOLVE_TIMEOUT" | "INVALID_PROFILE" }`
- Rate limit: `configured_rate_limit(bucket="trace-seed", default_max_attempts=10, default_window_seconds=60)`
- Cache: L2 Redis, key = `trace:seed:{profile}:{params_hash}`, TTL = 300s
**POST `/api/trace/lineage`**
- Request: `{ "profile": "query_tool" | "mid_section_defect", "container_ids": [...], "cache_key": "trace:{hash}" }`
- Response: `{ "stage": "lineage", "ancestors": { "{cid}": ["{ancestor_cid}", ...] }, "merges": { "{cid}": ["{merge_source_cid}", ...] }, "total_nodes": N }`
- Error: `{ "error": "...", "code": "LINEAGE_TIMEOUT" | "LINEAGE_TOO_LARGE" }`
- Rate limit: `configured_rate_limit(bucket="trace-lineage", default_max_attempts=10, default_window_seconds=60)`
- Cache: L2 Redis, key = `trace:lineage:{sorted_cids_hash}`, TTL = 300s
- 冪等性: 相同 `container_ids` 集合(排序後 hash回傳 cache 結果
**POST `/api/trace/events`**
- Request: `{ "profile": "query_tool" | "mid_section_defect", "container_ids": [...], "domains": ["history", "materials", ...], "cache_key": "trace:{hash}" }`
- `mid_section_defect` 額外支援 `"domains": ["upstream_history"]` 和自動串接 aggregation
- Response: `{ "stage": "events", "results": { "{domain}": { "data": [...], "count": N } }, "aggregation": { ... } | null }`
- Error: `{ "error": "...", "code": "EVENTS_TIMEOUT" | "EVENTS_PARTIAL_FAILURE" }`
- `EVENTS_PARTIAL_FAILURE`: 部分 domain 查詢失敗時仍回傳已成功的結果,`failed_domains` 列出失敗項
- Rate limit: `configured_rate_limit(bucket="trace-events", default_max_attempts=15, default_window_seconds=60)`
- Cache: L2 Redis, key = `trace:evt:{profile}:{domains_hash}:{sorted_cids_hash}`, TTL = 300s
**所有 endpoints 共通規則**:
- HTTP status: 200 (success), 400 (invalid params/profile), 429 (rate limited), 504 (stage timeout >10s)
- Rate limit headers: `Retry-After`(對齊 `rate_limit.py` 既有實作,回應 body 含 `retry_after_seconds` 欄位)
- `cache_key` 為選填欄位,前端可傳入前一 stage 回傳的 cache_key 作為追溯鏈標識(用於 logging correlation不影響 cache 命中邏輯
- 每個 stage 獨立可呼叫——前端可按需組合,不要求嚴格順序(但 lineage 需要 seed 結果的 container_idsevents 需要 lineage 結果的 container_ids
### 舊 endpoint 兼容
- `/api/mid-section-defect/analysis` 保留內部改為呼叫分段管線seed-resolve → lineage → events+aggregation後合併結果回傳。行為等價API contract 不變。
- `/api/query-tool/*` 保留不變,前端可視進度逐步遷移到新 API。
### 前端:漸進式載入
- 新增 `frontend/src/shared-composables/useTraceProgress.js` composable封裝
- 三段式 sequential fetchseed → lineage → events
- 每段完成後更新 reactive state`current_stage`, `completed_stages`, `stage_results`
- 錯誤處理: 每段獨立,某段失敗不阻斷已完成的結果顯示
- profile 參數注入
- **mid-section-defect** (`App.vue`): 查詢流程改為分段 fetch + 漸進渲染:
- 查詢後先顯示 seed lots 列表skeleton UI → 填入 seed 結果)
- 血緣樹結構逐步展開
- KPI/圖表以 skeleton placeholders + fade-in 動畫漸進填入,避免 layout shift
- 明細表格仍使用 detail endpoint 分頁
- **query-tool** (`useQueryToolData.js`): lineage tab 改為 on-demand 展開(使用者點擊 lot 後才查血緣),主要強化漸進載入體驗。
- 兩個頁面新增進度指示器元件,顯示目前正在執行的 stageseed → lineage → events → aggregation和已完成的 stage。
### Legacy 檔案處理
- **廢棄**: `src/mes_dashboard/static/js/query-tool.js`3056L, 126KB——這是 pre-Vite 時代的靜態 JS 檔案,目前已無任何 template 載入(`query_tool.html` 使用 `frontend_asset('query-tool.js')` 載入 Vite 建置產物,非此靜態檔案)。此檔案為 dead code可安全刪除。
- **保留**: `frontend/src/query-tool/main.js`3139L——這是 Vue 3 版本的 Vite entry pointVite config 確認為 `'query-tool': resolve(__dirname, 'src/query-tool/main.js')`。此檔案持續維護。
- **保留**: `src/mes_dashboard/templates/query_tool.html`——Jinja2 模板line 1264 `{% set query_tool_js = frontend_asset('query-tool.js') %}` 載入 Vite 建置產物。目前 portal-shell route 已生效(`/portal-shell/query-tool` 走 Vue 3此模板為 non-portal-shell 路由的 fallback暫不刪除。
## Capabilities
### New Capabilities
- `trace-staged-api`: 統一的分段追溯 API 層(`/api/trace/seed-resolve``/api/trace/lineage``/api/trace/events`)。通過 `profile` 參數配置頁面行為。每段獨立可 cacheL2 Redis、可 rate limit`configured_rate_limit()`前端可按需組合。API contract 定義於本提案 What Changes 章節。
- `progressive-trace-ux`: 兩個頁面的漸進式載入 UX。`useTraceProgress` composable 封裝三段式 sequential fetch + reactive state。包含
- 進度指示器元件(顯示 seed → lineage → events → aggregation 各階段狀態)
- mid-section-defect: seed lots 先出 → 血緣結構 → KPI/圖表漸進填入skeleton + fade-in
- query-tool: lineage tab 改為 on-demand 展開(使用者點擊 lot 後才查血緣)
### Modified Capabilities
- `trace-staged-api` 取代 mid-section-defect 現有的單一 `/analysis` endpoint 邏輯(保留舊 endpoint 作為兼容,內部改為呼叫分段管線 + 合併結果,行為等價)。
- query-tool 現有的 `useQueryToolData.js` composable 改為使用分段 API。
## Impact
- **Affected code**:
- 新建: `src/mes_dashboard/routes/trace_routes.py`, `frontend/src/shared-composables/useTraceProgress.js`
- 重構: `frontend/src/mid-section-defect/App.vue`(查詢流程改為分段 fetch
- 重構: `frontend/src/query-tool/composables/useQueryToolData.js`lineage 改分段)
- 修改: `src/mes_dashboard/routes/mid_section_defect_routes.py``/analysis` 內部改用分段管線)
- 刪除: `src/mes_dashboard/static/js/query-tool.js`pre-Vite dead code, 3056L, 126KB, 無任何引用)
- **Runtime/deploy**: 無新依賴。新增 3 個 API endpoints`/api/trace/*`),原有 endpoints 保持兼容。
- **APIs/pages**: 新增 `/api/trace/seed-resolve``/api/trace/lineage``/api/trace/events` 三個 endpointscontract 定義見 What Changes 章節)。原有 `/api/mid-section-defect/analysis``/api/query-tool/*` 保持兼容但 `/analysis` 內部實作改為呼叫分段管線。
- **UX**: 查詢體驗從「黑盒等待」變為「漸進可見」。mid-section-defect 使用者可在血緣解析階段就看到 seed lots 和初步資料。
## Verification
**前端漸進載入驗收**:
| 指標 | 現況 | 目標 | 條件 |
|------|------|------|------|
| mid-section-defect 首次可見內容 (seed lots) | 全部完成後一次顯示30-120s, unified-lineage-engine 後 3-8s | seed stage 完成即顯示≤3s | ≥10 seed lots 查詢 |
| mid-section-defect KPI/chart 完整顯示 | 同上 | lineage + events 完成後顯示≤8s | skeleton → fade-in, 無 layout shift |
| query-tool lineage tab | 一次載入全部 lot 的 lineage | 點擊單一 lot 後載入該 lot lineage≤3s | on-demand, ≥20 lots resolved |
| 進度指示器 | 無loading spinner | 每個 stage 切換時更新進度文字 | seed → lineage → events 三階段可見 |
**API contract 驗收**:
- 每個 `/api/trace/*` endpoint 回傳 JSON 結構符合 What Changes 章節定義的 schema
- 400 (invalid params) / 429 (rate limited) / 504 (timeout) status code 正確回傳
- Rate limit header `Retry-After` 存在(對齊既有 `rate_limit.py` 實作)
- `/api/mid-section-defect/analysis` 兼容性: 回傳結果與重構前完全一致golden test 比對)
**Legacy cleanup 驗收**:
- `src/mes_dashboard/static/js/query-tool.js` 已刪除
- grep 確認無任何程式碼引用 `static/js/query-tool.js`
- `query_tool.html``frontend_asset('query-tool.js')` 仍正常解析到 Vite 建置產物
## Dependencies
- **前置條件**: `unified-lineage-engine` 變更必須先完成。本變更依賴 `LineageEngine``EventFetcher` 作為分段 API 的後端實作。
## Non-Goals
- 不實作 SSE (Server-Sent Events) 或 WebSocket 即時推送——考慮到 gunicorn sync workers 的限制,使用分段 API + 前端 sequential fetch 模式。
- 不改動後端追溯邏輯——分段 API 純粹是將 `LineageEngine` / `EventFetcher` 各 stage 獨立暴露為 HTTP endpoint不改變計算邏輯。
- 不新增任務隊列Celery/RQ——維持同步 request-response 模式,各 stage 控制在 <10s 回應時間內
- 不改動 mid-section-defect defect attribution 演算法
- 不改動 query-tool equipment-period 查詢已有 `read_sql_df_slow` 120s timeout 處理)。
- 不改動 DB schema不建立 materialized view——所有最佳化在應用層完成
## Risks
| 風險 | 緩解 |
|------|------|
| 分段 API 增加前端複雜度多次 fetch + 狀態管理 | 封裝為 `useTraceProgress` composable頁面只需提供 profile + params內部處理 sequential fetch + error + state |
| 前後端分段 contract 不匹配 | API contract 完整定義於本提案 What Changes 章節 request/response schemaerror codescache key 格式CI 契約測試驗證 |
| `/analysis` endpoint 需保持兼容 | 保留舊 endpoint內部改為呼叫分段管線 + 合併結果golden test 比對重構前後輸出一致 |
| 刪除 `static/js/query-tool.js` 影響功能 | 已確認此檔案為 pre-Vite dead code`query_tool.html` 使用 `frontend_asset('query-tool.js')` 載入 Vite 建置產物非此靜態檔案grep 確認無其他引用 |
| mid-section-defect 分段渲染導致 chart 閃爍 | 使用 skeleton placeholders + fade-in 動畫避免 layout shiftchart container 預留固定高度 |
| `cache_key` 被濫用於跨 stage 繞過 rate limit | cache_key 僅用於 logging correlation不影響 cache 命中或 rate limit 邏輯每個 stage 獨立計算 cache key |

View File

@@ -0,0 +1,25 @@
## MODIFIED Requirements
### Requirement: Staged trace API endpoints SHALL apply rate limiting
The `/api/trace/seed-resolve`, `/api/trace/lineage`, and `/api/trace/events` endpoints SHALL apply per-client rate limiting using the existing `configured_rate_limit` mechanism.
#### Scenario: Seed-resolve rate limit exceeded
- **WHEN** a client sends more than 10 requests to `/api/trace/seed-resolve` within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
#### Scenario: Lineage rate limit exceeded
- **WHEN** a client sends more than 10 requests to `/api/trace/lineage` within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
#### Scenario: Events rate limit exceeded
- **WHEN** a client sends more than 15 requests to `/api/trace/events` within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
### Requirement: Mid-section defect analysis endpoint SHALL internally use staged pipeline
The existing `/api/mid-section-defect/analysis` endpoint SHALL internally delegate to the staged trace pipeline while maintaining full backward compatibility.
#### Scenario: Analysis endpoint backward compatibility
- **WHEN** a client calls `GET /api/mid-section-defect/analysis` with existing query parameters
- **THEN** the response JSON structure SHALL be identical to pre-refactoring output
- **THEN** existing rate limiting (6/min analysis, 15/min detail, 3/min export) SHALL remain unchanged
- **THEN** existing distributed lock behavior SHALL remain unchanged

View File

@@ -0,0 +1,64 @@
## ADDED Requirements
### Requirement: useTraceProgress composable SHALL orchestrate staged fetching with reactive state
`useTraceProgress` SHALL provide a shared composable for sequential stage fetching with per-stage reactive state updates.
#### Scenario: Normal three-stage fetch sequence
- **WHEN** `useTraceProgress` is invoked with profile and params
- **THEN** it SHALL execute seed-resolve → lineage → events sequentially
- **THEN** after each stage completes, `current_stage` and `completed_stages` reactive refs SHALL update immediately
- **THEN** `stage_results` SHALL accumulate results from completed stages
#### Scenario: Stage failure does not block completed results
- **WHEN** the lineage stage fails after seed-resolve has completed
- **THEN** seed-resolve results SHALL remain visible and accessible
- **THEN** the error SHALL be captured in stage-specific error state
- **THEN** subsequent stages (events) SHALL NOT execute
### Requirement: mid-section-defect SHALL render progressively as stages complete
The mid-section-defect page SHALL display partial results as each trace stage completes.
#### Scenario: Seed lots visible before lineage completes
- **WHEN** seed-resolve stage completes (≤3s for ≥10 seed lots)
- **THEN** the seed lots list SHALL be rendered immediately
- **THEN** lineage and events sections SHALL show skeleton placeholders
#### Scenario: KPI/charts visible after events complete
- **WHEN** lineage and events stages complete
- **THEN** KPI cards and charts SHALL render with fade-in animation
- **THEN** no layout shift SHALL occur (skeleton placeholders SHALL have matching dimensions)
#### Scenario: Detail table pagination unchanged
- **WHEN** the user requests detail data
- **THEN** the existing detail endpoint with pagination SHALL be used (not the staged API)
### Requirement: query-tool lineage tab SHALL load on-demand
The query-tool lineage tab SHALL load lineage data for individual lots on user interaction, not batch-load all lots.
#### Scenario: User clicks a lot to view lineage
- **WHEN** the user clicks a lot card to expand lineage information
- **THEN** lineage SHALL be fetched via `/api/trace/lineage` for that single lot's container IDs
- **THEN** response time SHALL be ≤3s for the individual lot
#### Scenario: Multiple lots expanded
- **WHEN** the user expands lineage for multiple lots
- **THEN** each lot's lineage SHALL be fetched independently (not batch)
- **THEN** already-fetched lineage data SHALL be preserved (not re-fetched)
### Requirement: Both pages SHALL display a stage progress indicator
Both mid-section-defect and query-tool SHALL display a progress indicator showing the current trace stage.
#### Scenario: Progress indicator during staged fetch
- **WHEN** a trace query is in progress
- **THEN** a progress indicator SHALL display the current stage (seed → lineage → events)
- **THEN** completed stages SHALL be visually distinct from pending/active stages
- **THEN** the indicator SHALL replace the existing single loading spinner
### Requirement: Legacy static query-tool.js SHALL be removed
The pre-Vite static file `src/mes_dashboard/static/js/query-tool.js` (3056L, 126KB) SHALL be deleted as dead code.
#### Scenario: Dead code removal verification
- **WHEN** `static/js/query-tool.js` is deleted
- **THEN** grep for `static/js/query-tool.js` SHALL return zero results across the codebase
- **THEN** `query_tool.html` template SHALL continue to function via `frontend_asset('query-tool.js')` which resolves to the Vite-built bundle
- **THEN** `frontend/src/query-tool/main.js` (Vue 3 Vite entry) SHALL remain unaffected

View File

@@ -0,0 +1,89 @@
## ADDED Requirements
### Requirement: Staged trace API SHALL expose seed-resolve endpoint
`POST /api/trace/seed-resolve` SHALL resolve seed lots based on the provided profile and parameters.
#### Scenario: query_tool profile seed resolve
- **WHEN** request body contains `{ "profile": "query_tool", "params": { "resolve_type": "lot_id", "values": [...] } }`
- **THEN** the endpoint SHALL call existing lot resolve logic and return `{ "stage": "seed-resolve", "seeds": [...], "seed_count": N, "cache_key": "trace:{hash}" }`
- **THEN** each seed object SHALL contain `container_id`, `container_name`, and `lot_id`
#### Scenario: mid_section_defect profile seed resolve
- **WHEN** request body contains `{ "profile": "mid_section_defect", "params": { "date_range": [...], "workcenter": "..." } }`
- **THEN** the endpoint SHALL call TMTT detection logic and return seed lots in the same response format
#### Scenario: Empty seed result
- **WHEN** seed resolution finds no matching lots
- **THEN** the endpoint SHALL return HTTP 200 with `{ "stage": "seed-resolve", "seeds": [], "seed_count": 0, "cache_key": "trace:{hash}" }`
- **THEN** the error code `SEED_RESOLVE_EMPTY` SHALL NOT be used for empty results (reserved for resolution failures)
#### Scenario: Invalid profile
- **WHEN** request body contains an unrecognized `profile` value
- **THEN** the endpoint SHALL return HTTP 400 with `{ "error": "...", "code": "INVALID_PROFILE" }`
### Requirement: Staged trace API SHALL expose lineage endpoint
`POST /api/trace/lineage` SHALL resolve lineage graph for provided container IDs using `LineageEngine`.
#### Scenario: Normal lineage resolution
- **WHEN** request body contains `{ "profile": "query_tool", "container_ids": [...] }`
- **THEN** the endpoint SHALL call `LineageEngine.resolve_full_genealogy()` and return `{ "stage": "lineage", "ancestors": {...}, "merges": {...}, "total_nodes": N }`
#### Scenario: Lineage result caching with idempotency
- **WHEN** two requests with the same `container_ids` set (regardless of order) arrive
- **THEN** the cache key SHALL be computed as `trace:lineage:{sorted_cids_hash}`
- **THEN** the second request SHALL return cached result from L2 Redis (TTL = 300s)
#### Scenario: Lineage timeout
- **WHEN** lineage resolution exceeds 10 seconds
- **THEN** the endpoint SHALL return HTTP 504 with `{ "error": "...", "code": "LINEAGE_TIMEOUT" }`
### Requirement: Staged trace API SHALL expose events endpoint
`POST /api/trace/events` SHALL query events for specified domains using `EventFetcher`.
#### Scenario: Normal events query
- **WHEN** request body contains `{ "profile": "query_tool", "container_ids": [...], "domains": ["history", "materials"] }`
- **THEN** the endpoint SHALL return `{ "stage": "events", "results": { "history": { "data": [...], "count": N }, "materials": { "data": [...], "count": N } }, "aggregation": null }`
#### Scenario: mid_section_defect profile with aggregation
- **WHEN** request body contains `{ "profile": "mid_section_defect", "container_ids": [...], "domains": ["upstream_history"] }`
- **THEN** the endpoint SHALL automatically run aggregation logic after event fetching
- **THEN** the response `aggregation` field SHALL contain the aggregated results (not null)
#### Scenario: Partial domain failure
- **WHEN** one domain query fails while others succeed
- **THEN** the endpoint SHALL return HTTP 200 with `{ "error": "...", "code": "EVENTS_PARTIAL_FAILURE" }`
- **THEN** the response SHALL include successfully fetched domains in `results` and list failed domains in `failed_domains`
### Requirement: All staged trace endpoints SHALL apply rate limiting and caching
Every `/api/trace/*` endpoint SHALL use `configured_rate_limit()` and L2 Redis caching.
#### Scenario: Rate limit exceeded on any trace endpoint
- **WHEN** a client exceeds the configured request budget for a trace endpoint
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
- **THEN** the body SHALL contain `{ "error": "...", "meta": { "retry_after_seconds": N } }`
#### Scenario: Cache hit on trace endpoint
- **WHEN** a request matches a cached result in L2 Redis (TTL = 300s)
- **THEN** the cached result SHALL be returned without executing backend logic
- **THEN** Oracle DB connection pool SHALL NOT be consumed
### Requirement: cache_key parameter SHALL be used for logging correlation only
The optional `cache_key` field in request bodies SHALL be used solely for logging and tracing correlation.
#### Scenario: cache_key provided in request
- **WHEN** a request includes `cache_key` from a previous stage response
- **THEN** the value SHALL be logged for correlation purposes
- **THEN** the value SHALL NOT influence cache lookup or rate limiting logic
#### Scenario: cache_key omitted in request
- **WHEN** a request omits the `cache_key` field
- **THEN** the endpoint SHALL function normally without any degradation
### Requirement: Existing `GET /api/mid-section-defect/analysis` SHALL remain compatible
The existing analysis endpoint (GET method) SHALL internally delegate to the staged pipeline while maintaining identical external behavior.
#### Scenario: Legacy analysis endpoint invocation
- **WHEN** a client calls `GET /api/mid-section-defect/analysis` with existing query parameters
- **THEN** the endpoint SHALL internally execute seed-resolve → lineage → events + aggregation
- **THEN** the response format SHALL be identical to the pre-refactoring output
- **THEN** a golden test SHALL verify output equivalence

View File

@@ -0,0 +1,41 @@
## Phase 1: 後端 trace_routes.py Blueprint
- [x] 1.1 建立 `src/mes_dashboard/routes/trace_routes.py``trace_bp` Blueprint`url_prefix='/api/trace'`
- [x] 1.2 實作 `POST /api/trace/seed-resolve` handlerrequest body 驗證、profile dispatch`_seed_resolve_query_tool` / `_seed_resolve_msd`、response format
- [x] 1.3 實作 `POST /api/trace/lineage` handler呼叫 `LineageEngine.resolve_full_genealogy()`、response format、504 timeout 處理
- [x] 1.4 實作 `POST /api/trace/events` handler呼叫 `EventFetcher.fetch_events()`、mid_section_defect profile 自動 aggregation、`EVENTS_PARTIAL_FAILURE` 處理
- [x] 1.5 為三個 endpoint 加入 `configured_rate_limit()`seed: 10/60s, lineage: 10/60s, events: 15/60s
- [x] 1.6 為三個 endpoint 加入 L2 Redis cacheseed: `trace:seed:{profile}:{params_hash}`, lineage: `trace:lineage:{sorted_cids_hash}`, events: `trace:evt:{profile}:{domains_hash}:{sorted_cids_hash}`TTL=300s
- [x] 1.7 在 `src/mes_dashboard/routes/__init__.py` 匯入並註冊 `trace_bp` Blueprint維持專案統一的 route 註冊入口)
- [x] 1.8 API contract 測試:驗證 200/400/429/504 status code、`Retry-After` header、error code 格式、snake_case field names
## Phase 2: 前端共用元件
- [x] 2.1 建立 `frontend/src/shared-composables/useTraceProgress.js`reactive state`current_stage`, `completed_stages`, `stage_results`, `stage_errors`, `is_running`+ `execute()` / `reset()` / `abort()` methods
- [x] 2.2 實作 sequential fetch 邏輯seed-resolve → lineage → events每段完成後立即更新 reactive state錯誤記錄到 stage_errors 不拋出
- [x] 2.3 建立 `frontend/src/shared-composables/TraceProgressBar.vue`三段式進度指示器props: `current_stage`, `completed_stages`, `stage_errors`),完成=green、進行中=blue pulse、待處理=gray、錯誤=red
## Phase 3: mid-section-defect 漸進渲染
- [x] 3.1 在 `frontend/src/mid-section-defect/App.vue` 中引入 `useTraceProgress({ profile: 'mid_section_defect' })`
- [x] 3.2 改造 `loadAnalysis()` 流程:從 `apiGet('/analysis')` 單次呼叫改為 `trace.execute(params)` 分段 fetch
- [x] 3.3 加入 skeleton placeholdersKpiCards6 cards, min-height 100px、ParetoChart6 charts, min-height 300px、TrendChartmin-height 300px灰色脈動動畫
- [x] 3.4 加入 fade-in transitionstage_results.events 完成後 KPI/charts 以 `opacity 0→1, 300ms ease-in` 填入
- [x] 3.5 用 `TraceProgressBar` 取代 filter bar 下方的 loading spinner
- [x] 3.6 整合 `useAutoRefresh``onRefresh``trace.abort()` + `trace.execute(committedFilters)`
- [x] 3.7 驗證 detail 分頁不受影響(仍使用 `/api/mid-section-defect/analysis/detail` GET endpoint
- [x] 3.8 Golden test`/api/mid-section-defect/analysis` GET endpoint 回傳結果與重構前完全一致(浮點 tolerance ±0.01%
## Phase 4: query-tool on-demand lineage
- [x] 4.1 在 `useQueryToolData.js` 新增 `lineageCache` reactive object + `loadLotLineage(containerId)` 函數
- [x] 4.2 `loadLotLineage` 呼叫 `POST /api/trace/lineage``profile: 'query_tool'`, `container_ids: [containerId]`),結果存入 `lineageCache`
- [x] 4.3 在 lot 列表 UI 新增 lineage 展開按鈕accordion pattern點擊觸發 `loadLotLineage`,已快取的不重新 fetch
- [x] 4.4 `resolveLots()` 時清空 `lineageCache`(新一輪查詢)
- [x] 4.5 驗證既有 resolve → lot-history → lot-associations 流程不受影響
## Phase 5: Legacy cleanup
- [x] 5.1 刪除 `src/mes_dashboard/static/js/query-tool.js`3056L, 126KB pre-Vite dead code
- [x] 5.2 `grep -r "static/js/query-tool.js" src/ frontend/ templates/` 確認 0 結果
- [x] 5.3 確認 `frontend_asset('query-tool.js')` 正確解析到 Vite manifest 中的 hashed filename

View File

@@ -24,3 +24,20 @@ The system SHALL continue to maintain full-table cache behavior for `resource` a
- **WHEN** cache update runs for `resource` or `wip`
- **THEN** the updater MUST retain full-table snapshot semantics and MUST NOT switch these domains to partial-only cache mode
### Requirement: Mid-section defect genealogy SHALL use CONNECT BY instead of Python BFS
The mid-section-defect genealogy resolution SHALL use `LineageEngine.resolve_full_genealogy()` (CONNECT BY NOCYCLE) instead of the existing `_bfs_split_chain()` Python BFS implementation.
#### Scenario: Genealogy cold query performance
- **WHEN** mid-section-defect analysis executes genealogy resolution with cache miss
- **THEN** `LineageEngine.resolve_split_ancestors()` SHALL be called (single CONNECT BY query)
- **THEN** response time SHALL be ≤8s (P95) for ≥50 ancestor nodes
- **THEN** Python BFS `_bfs_split_chain()` SHALL NOT be called
#### Scenario: Genealogy hot query performance
- **WHEN** mid-section-defect analysis executes genealogy resolution with L2 Redis cache hit
- **THEN** response time SHALL be ≤1s (P95)
#### Scenario: Golden test result equivalence
- **WHEN** golden test runs with ≥5 known LOTs
- **THEN** CONNECT BY output (`child_to_parent`, `cid_to_name`) SHALL be identical to BFS output for the same inputs

View File

@@ -0,0 +1,24 @@
# event-fetcher-unified Specification
## Purpose
TBD - created by archiving change unified-lineage-engine. Update Purpose after archive.
## Requirements
### Requirement: EventFetcher SHALL provide unified cached event querying across domains
`EventFetcher` SHALL encapsulate batch event queries with L1/L2 layered cache and rate limit bucket configuration, supporting domains: `history`, `materials`, `rejects`, `holds`, `jobs`, `upstream_history`.
#### Scenario: Cache miss for event domain query
- **WHEN** `EventFetcher` is called for a domain with container IDs and no cache exists
- **THEN** the domain query SHALL execute against Oracle via `read_sql_df()`
- **THEN** the result SHALL be stored in L2 Redis cache with key format `evt:{domain}:{sorted_cids_hash}`
- **THEN** L1 memory cache SHALL also be populated (aligned with `core/cache.py` LayeredCache pattern)
#### Scenario: Cache hit for event domain query
- **WHEN** `EventFetcher` is called for a domain and L2 Redis cache contains a valid entry
- **THEN** the cached result SHALL be returned without executing Oracle query
- **THEN** DB connection pool SHALL NOT be consumed
#### Scenario: Rate limit bucket per domain
- **WHEN** `EventFetcher` is used from a route handler
- **THEN** each domain SHALL have a configurable rate limit bucket aligned with `configured_rate_limit()` pattern
- **THEN** rate limit configuration SHALL be overridable via environment variables

View File

@@ -0,0 +1,61 @@
# lineage-engine-core Specification
## Purpose
TBD - created by archiving change unified-lineage-engine. Update Purpose after archive.
## Requirements
### Requirement: LineageEngine SHALL provide unified split ancestor resolution via CONNECT BY NOCYCLE
`LineageEngine.resolve_split_ancestors()` SHALL accept a list of container IDs and return the complete split ancestry graph using a single Oracle `CONNECT BY NOCYCLE` query on `DW_MES_CONTAINER.SPLITFROMID`.
#### Scenario: Normal split chain resolution
- **WHEN** `resolve_split_ancestors()` is called with a list of container IDs
- **THEN** a single SQL query using `CONNECT BY NOCYCLE` SHALL be executed against `DW_MES_CONTAINER`
- **THEN** the result SHALL include a `child_to_parent` mapping and a `cid_to_name` mapping for all discovered ancestor nodes
- **THEN** the traversal depth SHALL be limited to `LEVEL <= 20` (equivalent to existing BFS `bfs_round > 20` guard)
#### Scenario: Large input batch exceeding Oracle IN clause limit
- **WHEN** the input `container_ids` list exceeds `ORACLE_IN_BATCH_SIZE` (1000)
- **THEN** `QueryBuilder.add_in_condition()` SHALL batch the IDs and combine results
- **THEN** all bind parameters SHALL use `QueryBuilder.params` (no string concatenation)
#### Scenario: Cyclic split references in data
- **WHEN** `DW_MES_CONTAINER.SPLITFROMID` contains cyclic references
- **THEN** `NOCYCLE` SHALL prevent infinite traversal
- **THEN** the query SHALL return all non-cyclic ancestors up to `LEVEL <= 20`
#### Scenario: CONNECT BY performance regression
- **WHEN** Oracle 19c execution plan for `CONNECT BY NOCYCLE` performs worse than expected
- **THEN** the SQL file SHALL contain a commented-out recursive `WITH` (recursive subquery factoring) alternative that can be swapped in without code changes
### Requirement: LineageEngine SHALL provide unified merge source resolution
`LineageEngine.resolve_merge_sources()` SHALL accept a list of container IDs and return merge source mappings from `DW_MES_PJ_COMBINEDASSYLOTS`.
#### Scenario: Merge source lookup
- **WHEN** `resolve_merge_sources()` is called with container IDs
- **THEN** the result SHALL include `{cid: [merge_source_cid, ...]}` for all containers that have merge sources
- **THEN** all queries SHALL use `QueryBuilder` bind params
### Requirement: LineageEngine SHALL provide combined genealogy resolution
`LineageEngine.resolve_full_genealogy()` SHALL combine split ancestors and merge sources into a complete genealogy graph.
#### Scenario: Full genealogy for a set of seed lots
- **WHEN** `resolve_full_genealogy()` is called with seed container IDs
- **THEN** split ancestors SHALL be resolved first via `resolve_split_ancestors()`
- **THEN** merge sources SHALL be resolved for all discovered ancestor nodes
- **THEN** the combined result SHALL be equivalent to the existing `_resolve_full_genealogy()` output in `mid_section_defect_service.py`
### Requirement: LineageEngine functions SHALL be profile-agnostic
All `LineageEngine` public functions SHALL accept `container_ids: List[str]` and return dictionary structures without binding to any specific page logic.
#### Scenario: Reuse from different pages
- **WHEN** a new page (e.g., wip-detail) needs lineage resolution
- **THEN** it SHALL be able to call `LineageEngine` functions directly without modification
- **THEN** no page-specific logic (profile, TMTT detection, etc.) SHALL exist in `LineageEngine`
### Requirement: LineageEngine SQL files SHALL reside in `sql/lineage/` directory
New SQL files SHALL follow the existing `SQLLoader` convention under `src/mes_dashboard/sql/lineage/`.
#### Scenario: SQL file organization
- **WHEN** `LineageEngine` executes queries
- **THEN** `split_ancestors.sql` and `merge_sources.sql` SHALL be loaded via `SQLLoader.load_with_params("lineage/split_ancestors", ...)`
- **THEN** the SQL files SHALL NOT reference `HM_LOTMOVEOUT` (48M row table no longer needed for genealogy)

View File

@@ -17,3 +17,25 @@ Services consuming shared Oracle query fragments SHALL preserve existing selecte
- **WHEN** cache services execute queries via shared fragments
- **THEN** resulting payload structure MUST remain compatible with existing aggregation and API contracts
### Requirement: Lineage SQL fragments SHALL be centralized in `sql/lineage/` directory
Split ancestor and merge source SQL queries SHALL be defined in `sql/lineage/` and shared across services via `SQLLoader`.
#### Scenario: Mid-section-defect lineage query
- **WHEN** `mid_section_defect_service.py` needs split ancestry or merge source data
- **THEN** it SHALL call `LineageEngine` which loads SQL from `sql/lineage/split_ancestors.sql` and `sql/lineage/merge_sources.sql`
- **THEN** it SHALL NOT use `sql/mid_section_defect/split_chain.sql` or `sql/mid_section_defect/genealogy_records.sql`
#### Scenario: Deprecated SQL file handling
- **WHEN** `sql/mid_section_defect/genealogy_records.sql` and `sql/mid_section_defect/split_chain.sql` are deprecated
- **THEN** the files SHALL be marked with a deprecated comment at the top
- **THEN** grep SHALL confirm zero `SQLLoader.load` references to these files
- **THEN** the files SHALL be retained for one version before deletion
### Requirement: All user-input SQL queries SHALL use QueryBuilder bind params
`_build_in_filter()` and `_build_in_clause()` in `query_tool_service.py` SHALL be fully replaced by `QueryBuilder.add_in_condition()`.
#### Scenario: Complete migration to QueryBuilder
- **WHEN** the refactoring is complete
- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results
- **THEN** all queries involving user-supplied values SHALL use `QueryBuilder.params`

View File

@@ -0,0 +1,61 @@
# query-tool-safety-hardening Specification
## Purpose
TBD - created by archiving change unified-lineage-engine. Update Purpose after archive.
## Requirements
### Requirement: query-tool resolve functions SHALL use QueryBuilder bind params for all user input
All `resolve_lots()` family functions (`_resolve_by_lot_id`, `_resolve_by_serial_number`, `_resolve_by_work_order`) SHALL use `QueryBuilder.add_in_condition()` with bind parameters instead of `_build_in_filter()` string concatenation.
#### Scenario: Lot resolve with user-supplied values
- **WHEN** a resolve function receives user-supplied lot IDs, serial numbers, or work order names
- **THEN** the SQL query SHALL use `:p0, :p1, ...` bind parameters via `QueryBuilder`
- **THEN** `read_sql_df()` SHALL receive `builder.params` (never an empty `{}` dict for queries with user input)
- **THEN** `_build_in_filter()` and `_build_in_clause()` SHALL NOT be called
#### Scenario: Pure static SQL without user input
- **WHEN** a query contains no user-supplied values (e.g., static lookups)
- **THEN** empty params `{}` is acceptable
- **THEN** no `_build_in_filter()` SHALL be used
#### Scenario: Zero residual references to deprecated functions
- **WHEN** the refactoring is complete
- **THEN** grep for `_build_in_filter` and `_build_in_clause` SHALL return zero results across the entire codebase
### Requirement: query-tool routes SHALL apply rate limiting
All query-tool API endpoints SHALL apply per-client rate limiting using the existing `configured_rate_limit` mechanism.
#### Scenario: Resolve endpoint rate limit exceeded
- **WHEN** a client sends more than 10 requests to query-tool resolve endpoints within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
- **THEN** the resolve service function SHALL NOT be called
#### Scenario: History endpoint rate limit exceeded
- **WHEN** a client sends more than 20 requests to query-tool history endpoints within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
#### Scenario: Association endpoint rate limit exceeded
- **WHEN** a client sends more than 20 requests to query-tool association endpoints within 60 seconds
- **THEN** the endpoint SHALL return HTTP 429 with a `Retry-After` header
### Requirement: query-tool routes SHALL apply response caching
High-cost query-tool endpoints SHALL cache responses in L2 Redis.
#### Scenario: Resolve result caching
- **WHEN** a resolve request succeeds
- **THEN** the response SHALL be cached in L2 Redis with TTL = 60s
- **THEN** subsequent identical requests within TTL SHALL return cached result without Oracle query
### Requirement: lot_split_merge_history SHALL support fast and full query modes
The `lot_split_merge_history.sql` query SHALL support two modes to balance traceability completeness vs performance.
#### Scenario: Fast mode (default)
- **WHEN** `full_history` query parameter is absent or `false`
- **THEN** the SQL SHALL include `TXNDATE >= ADD_MONTHS(SYSDATE, -6)` time window and `FETCH FIRST 500 ROWS ONLY`
- **THEN** query response time SHALL be ≤5s (P95)
#### Scenario: Full mode
- **WHEN** `full_history=true` query parameter is provided
- **THEN** the SQL SHALL NOT include time window restriction
- **THEN** the query SHALL use `read_sql_df_slow` (120s timeout)
- **THEN** query response time SHALL be ≤60s (P95)

View File

@@ -19,6 +19,7 @@ from .query_tool_routes import query_tool_bp
from .tmtt_defect_routes import tmtt_defect_bp
from .qc_gate_routes import qc_gate_bp
from .mid_section_defect_routes import mid_section_defect_bp
from .trace_routes import trace_bp
def register_routes(app) -> None:
@@ -36,6 +37,7 @@ def register_routes(app) -> None:
app.register_blueprint(tmtt_defect_bp)
app.register_blueprint(qc_gate_bp)
app.register_blueprint(mid_section_defect_bp)
app.register_blueprint(trace_bp)
__all__ = [
'wip_bp',
@@ -53,5 +55,6 @@ __all__ = [
'tmtt_defect_bp',
'qc_gate_bp',
'mid_section_defect_bp',
'trace_bp',
'register_routes',
]

View File

@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
"""Query Tool API routes.
"""Query Tool API routes.
Contains Flask Blueprint for batch tracing and equipment period query endpoints:
- LOT resolution (LOT ID / Serial Number / Work Order → CONTAINERID)
@@ -7,12 +7,16 @@ Contains Flask Blueprint for batch tracing and equipment period query endpoints:
- LOT associations (materials, rejects, holds, jobs)
- Equipment period queries (status hours, lots, materials, rejects, jobs)
- CSV export functionality
"""
from flask import Blueprint, jsonify, request, Response, render_template
from mes_dashboard.core.modernization_policy import maybe_redirect_to_canonical_shell
from mes_dashboard.services.query_tool_service import (
"""
import hashlib
from flask import Blueprint, jsonify, request, Response, render_template
from mes_dashboard.core.cache import cache_get, cache_set
from mes_dashboard.core.modernization_policy import maybe_redirect_to_canonical_shell
from mes_dashboard.core.rate_limit import configured_rate_limit
from mes_dashboard.services.query_tool_service import (
resolve_lots,
get_lot_history,
get_adjacent_lots,
@@ -33,8 +37,51 @@ from mes_dashboard.services.query_tool_service import (
validate_equipment_input,
)
# Create Blueprint
query_tool_bp = Blueprint('query_tool', __name__)
# Create Blueprint
query_tool_bp = Blueprint('query_tool', __name__)
_QUERY_TOOL_RESOLVE_RATE_LIMIT = configured_rate_limit(
bucket="query-tool-resolve",
max_attempts_env="QT_RESOLVE_RATE_MAX_REQUESTS",
window_seconds_env="QT_RESOLVE_RATE_WINDOW_SECONDS",
default_max_attempts=10,
default_window_seconds=60,
)
_QUERY_TOOL_HISTORY_RATE_LIMIT = configured_rate_limit(
bucket="query-tool-history",
max_attempts_env="QT_HISTORY_RATE_MAX_REQUESTS",
window_seconds_env="QT_HISTORY_RATE_WINDOW_SECONDS",
default_max_attempts=20,
default_window_seconds=60,
)
_QUERY_TOOL_ASSOC_RATE_LIMIT = configured_rate_limit(
bucket="query-tool-association",
max_attempts_env="QT_ASSOC_RATE_MAX_REQUESTS",
window_seconds_env="QT_ASSOC_RATE_WINDOW_SECONDS",
default_max_attempts=20,
default_window_seconds=60,
)
_QUERY_TOOL_ADJACENT_RATE_LIMIT = configured_rate_limit(
bucket="query-tool-adjacent",
max_attempts_env="QT_ADJACENT_RATE_MAX_REQUESTS",
window_seconds_env="QT_ADJACENT_RATE_WINDOW_SECONDS",
default_max_attempts=20,
default_window_seconds=60,
)
_QUERY_TOOL_EQUIPMENT_RATE_LIMIT = configured_rate_limit(
bucket="query-tool-equipment",
max_attempts_env="QT_EQUIP_RATE_MAX_REQUESTS",
window_seconds_env="QT_EQUIP_RATE_WINDOW_SECONDS",
default_max_attempts=5,
default_window_seconds=60,
)
_QUERY_TOOL_EXPORT_RATE_LIMIT = configured_rate_limit(
bucket="query-tool-export",
max_attempts_env="QT_EXPORT_RATE_MAX_REQUESTS",
window_seconds_env="QT_EXPORT_RATE_WINDOW_SECONDS",
default_max_attempts=3,
default_window_seconds=60,
)
# ============================================================
@@ -54,8 +101,9 @@ def query_tool_page():
# LOT Resolution API
# ============================================================
@query_tool_bp.route('/api/query-tool/resolve', methods=['POST'])
def resolve_lot_input():
@query_tool_bp.route('/api/query-tool/resolve', methods=['POST'])
@_QUERY_TOOL_RESOLVE_RATE_LIMIT
def resolve_lot_input():
"""Resolve user input to CONTAINERID list.
Expects JSON body:
@@ -86,24 +134,43 @@ def resolve_lot_input():
return jsonify({'error': f'不支援的查詢類型: {input_type}'}), 400
# Validate values
validation_error = validate_lot_input(input_type, values)
if validation_error:
return jsonify({'error': validation_error}), 400
result = resolve_lots(input_type, values)
if 'error' in result:
return jsonify(result), 400
return jsonify(result)
validation_error = validate_lot_input(input_type, values)
if validation_error:
return jsonify({'error': validation_error}), 400
cache_values = [
v.strip()
for v in values
if isinstance(v, str) and v.strip()
]
cache_key = None
if cache_values:
values_hash = hashlib.md5(
"|".join(sorted(cache_values)).encode("utf-8")
).hexdigest()[:16]
cache_key = f"qt:resolve:{input_type}:{values_hash}"
cached = cache_get(cache_key)
if cached is not None:
return jsonify(cached)
result = resolve_lots(input_type, values)
if 'error' in result:
return jsonify(result), 400
if cache_key is not None:
cache_set(cache_key, result, ttl=60)
return jsonify(result)
# ============================================================
# LOT History API
# ============================================================
@query_tool_bp.route('/api/query-tool/lot-history', methods=['GET'])
def query_lot_history():
@query_tool_bp.route('/api/query-tool/lot-history', methods=['GET'])
@_QUERY_TOOL_HISTORY_RATE_LIMIT
def query_lot_history():
"""Query production history for a LOT.
Query params:
@@ -137,8 +204,9 @@ def query_lot_history():
# Adjacent Lots API
# ============================================================
@query_tool_bp.route('/api/query-tool/adjacent-lots', methods=['GET'])
def query_adjacent_lots():
@query_tool_bp.route('/api/query-tool/adjacent-lots', methods=['GET'])
@_QUERY_TOOL_ADJACENT_RATE_LIMIT
def query_adjacent_lots():
"""Query adjacent lots (前後批) for a specific equipment.
Finds lots before/after target on same equipment until different PJ_TYPE,
@@ -170,16 +238,18 @@ def query_adjacent_lots():
# LOT Associations API
# ============================================================
@query_tool_bp.route('/api/query-tool/lot-associations', methods=['GET'])
def query_lot_associations():
@query_tool_bp.route('/api/query-tool/lot-associations', methods=['GET'])
@_QUERY_TOOL_ASSOC_RATE_LIMIT
def query_lot_associations():
"""Query association data for a LOT.
Query params:
container_id: CONTAINERID (16-char hex)
type: Association type ('materials', 'rejects', 'holds', 'jobs')
equipment_id: Equipment ID (required for 'jobs' type)
time_start: Start time (required for 'jobs' type)
time_end: End time (required for 'jobs' type)
time_start: Start time (required for 'jobs' type)
time_end: End time (required for 'jobs' type)
full_history: Optional boolean for 'splits' type (default false)
Returns association records based on type.
"""
@@ -199,8 +269,9 @@ def query_lot_associations():
result = get_lot_rejects(container_id)
elif assoc_type == 'holds':
result = get_lot_holds(container_id)
elif assoc_type == 'splits':
result = get_lot_splits(container_id)
elif assoc_type == 'splits':
full_history = request.args.get('full_history', 'false').lower() == 'true'
result = get_lot_splits(container_id, full_history=full_history)
elif assoc_type == 'jobs':
equipment_id = request.args.get('equipment_id')
time_start = request.args.get('time_start')
@@ -221,8 +292,9 @@ def query_lot_associations():
# Equipment Period Query API
# ============================================================
@query_tool_bp.route('/api/query-tool/equipment-period', methods=['POST'])
def query_equipment_period():
@query_tool_bp.route('/api/query-tool/equipment-period', methods=['POST'])
@_QUERY_TOOL_EQUIPMENT_RATE_LIMIT
def query_equipment_period():
"""Query equipment data for a time period.
Expects JSON body:
@@ -362,8 +434,9 @@ def get_workcenter_groups_list():
# CSV Export API
# ============================================================
@query_tool_bp.route('/api/query-tool/export-csv', methods=['POST'])
def export_csv():
@query_tool_bp.route('/api/query-tool/export-csv', methods=['POST'])
@_QUERY_TOOL_EXPORT_RATE_LIMIT
def export_csv():
"""Export query results as CSV.
Expects JSON body:

View File

@@ -0,0 +1,478 @@
# -*- coding: utf-8 -*-
"""Staged trace API routes.
Provides three stage endpoints for progressive trace execution:
- seed-resolve
- lineage
- events
"""
from __future__ import annotations
import hashlib
import json
import logging
import time
from typing import Any, Dict, List, Optional
from flask import Blueprint, jsonify, request
from mes_dashboard.core.cache import cache_get, cache_set
from mes_dashboard.core.rate_limit import configured_rate_limit
from mes_dashboard.core.response import error_response
from mes_dashboard.services.event_fetcher import EventFetcher
from mes_dashboard.services.lineage_engine import LineageEngine
from mes_dashboard.services.mid_section_defect_service import (
build_trace_aggregation_from_events,
parse_loss_reasons_param,
resolve_trace_seed_lots,
)
from mes_dashboard.services.query_tool_service import resolve_lots
logger = logging.getLogger("mes_dashboard.trace_routes")
trace_bp = Blueprint("trace", __name__, url_prefix="/api/trace")
TRACE_STAGE_TIMEOUT_SECONDS = 10.0
TRACE_CACHE_TTL_SECONDS = 300
PROFILE_QUERY_TOOL = "query_tool"
PROFILE_MID_SECTION_DEFECT = "mid_section_defect"
SUPPORTED_PROFILES = {PROFILE_QUERY_TOOL, PROFILE_MID_SECTION_DEFECT}
QUERY_TOOL_RESOLVE_TYPES = {"lot_id", "serial_number", "work_order"}
SUPPORTED_EVENT_DOMAINS = {
"history",
"materials",
"rejects",
"holds",
"jobs",
"upstream_history",
}
_TRACE_SEED_RATE_LIMIT = configured_rate_limit(
bucket="trace-seed",
max_attempts_env="TRACE_SEED_RATE_LIMIT_MAX_REQUESTS",
window_seconds_env="TRACE_SEED_RATE_LIMIT_WINDOW_SECONDS",
default_max_attempts=10,
default_window_seconds=60,
)
_TRACE_LINEAGE_RATE_LIMIT = configured_rate_limit(
bucket="trace-lineage",
max_attempts_env="TRACE_LINEAGE_RATE_LIMIT_MAX_REQUESTS",
window_seconds_env="TRACE_LINEAGE_RATE_LIMIT_WINDOW_SECONDS",
default_max_attempts=10,
default_window_seconds=60,
)
_TRACE_EVENTS_RATE_LIMIT = configured_rate_limit(
bucket="trace-events",
max_attempts_env="TRACE_EVENTS_RATE_LIMIT_MAX_REQUESTS",
window_seconds_env="TRACE_EVENTS_RATE_LIMIT_WINDOW_SECONDS",
default_max_attempts=15,
default_window_seconds=60,
)
def _json_body() -> Optional[Dict[str, Any]]:
payload = request.get_json(silent=True)
if isinstance(payload, dict):
return payload
return None
def _normalize_strings(values: Any) -> List[str]:
if not isinstance(values, list):
return []
normalized: List[str] = []
seen = set()
for value in values:
if not isinstance(value, str):
continue
text = value.strip()
if not text or text in seen:
continue
seen.add(text)
normalized.append(text)
return normalized
def _short_hash(parts: List[str]) -> str:
digest = hashlib.md5("|".join(parts).encode("utf-8")).hexdigest()
return digest[:12]
def _hash_payload(payload: Any) -> str:
dumped = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str, separators=(",", ":"))
return hashlib.md5(dumped.encode("utf-8")).hexdigest()[:12]
def _seed_cache_key(profile: str, params: Dict[str, Any]) -> str:
return f"trace:seed:{profile}:{_hash_payload(params)}"
def _lineage_cache_key(container_ids: List[str]) -> str:
return f"trace:lineage:{_short_hash(sorted(container_ids))}"
def _events_cache_key(profile: str, domains: List[str], container_ids: List[str]) -> str:
domains_hash = _short_hash(sorted(domains))
cid_hash = _short_hash(sorted(container_ids))
return f"trace:evt:{profile}:{domains_hash}:{cid_hash}"
def _error(code: str, message: str, status_code: int = 400):
return error_response(code, message, status_code=status_code)
def _timeout(stage: str):
return _error(f"{stage.upper().replace('-', '_')}_TIMEOUT", f"{stage} stage exceeded timeout budget", 504)
def _is_timeout_exception(exc: Exception) -> bool:
text = str(exc).lower()
timeout_fragments = (
"timeout",
"timed out",
"ora-01013",
"dpi-1067",
"cancelled",
)
return any(fragment in text for fragment in timeout_fragments)
def _validate_profile(profile: Any) -> Optional[str]:
if not isinstance(profile, str):
return None
value = profile.strip()
if value in SUPPORTED_PROFILES:
return value
return None
def _extract_date_range(params: Dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
date_range = params.get("date_range")
if isinstance(date_range, list) and len(date_range) == 2:
start_date = str(date_range[0] or "").strip()
end_date = str(date_range[1] or "").strip()
if start_date and end_date:
return start_date, end_date
start_date = str(params.get("start_date") or "").strip()
end_date = str(params.get("end_date") or "").strip()
if start_date and end_date:
return start_date, end_date
return None, None
def _seed_resolve_query_tool(params: Dict[str, Any]) -> tuple[Optional[Dict[str, Any]], Optional[tuple[str, str, int]]]:
resolve_type = str(params.get("resolve_type") or params.get("input_type") or "").strip()
if resolve_type not in QUERY_TOOL_RESOLVE_TYPES:
return None, ("INVALID_PARAMS", "resolve_type must be lot_id/serial_number/work_order", 400)
values = _normalize_strings(params.get("values", []))
if not values:
return None, ("INVALID_PARAMS", "values must contain at least one query value", 400)
resolved = resolve_lots(resolve_type, values)
if not isinstance(resolved, dict):
return None, ("SEED_RESOLVE_FAILED", "seed resolve returned unexpected payload", 500)
if "error" in resolved:
return None, ("SEED_RESOLVE_FAILED", str(resolved.get("error") or "seed resolve failed"), 400)
seeds = []
seen = set()
for row in resolved.get("data", []):
if not isinstance(row, dict):
continue
container_id = str(row.get("container_id") or row.get("CONTAINERID") or "").strip()
if not container_id or container_id in seen:
continue
seen.add(container_id)
lot_id = str(
row.get("lot_id")
or row.get("CONTAINERNAME")
or row.get("input_value")
or container_id
).strip()
seeds.append({
"container_id": container_id,
"container_name": lot_id,
"lot_id": lot_id,
})
return {"seeds": seeds, "seed_count": len(seeds)}, None
def _seed_resolve_mid_section_defect(
params: Dict[str, Any],
) -> tuple[Optional[Dict[str, Any]], Optional[tuple[str, str, int]]]:
start_date, end_date = _extract_date_range(params)
if not start_date or not end_date:
return None, ("INVALID_PARAMS", "start_date/end_date (or date_range) is required", 400)
result = resolve_trace_seed_lots(start_date, end_date)
if result is None:
return None, ("SEED_RESOLVE_FAILED", "seed resolve service unavailable", 503)
if "error" in result:
return None, ("SEED_RESOLVE_FAILED", str(result["error"]), 400)
return result, None
def _build_lineage_response(container_ids: List[str], ancestors_raw: Dict[str, Any]) -> Dict[str, Any]:
normalized_ancestors: Dict[str, List[str]] = {}
all_nodes = set(container_ids)
for seed in container_ids:
raw_values = ancestors_raw.get(seed, set())
values = raw_values if isinstance(raw_values, (set, list, tuple)) else []
normalized_list = sorted({
str(item).strip()
for item in values
if isinstance(item, str) and str(item).strip()
})
normalized_ancestors[seed] = normalized_list
all_nodes.update(normalized_list)
return {
"stage": "lineage",
"ancestors": normalized_ancestors,
"merges": {},
"total_nodes": len(all_nodes),
}
def _flatten_domain_records(events_by_cid: Dict[str, List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
rows: List[Dict[str, Any]] = []
for records in events_by_cid.values():
if not isinstance(records, list):
continue
for row in records:
if isinstance(row, dict):
rows.append(row)
return rows
def _parse_lineage_payload(payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
lineage = payload.get("lineage")
if isinstance(lineage, dict):
ancestors = lineage.get("ancestors")
if isinstance(ancestors, dict):
return ancestors
direct_ancestors = payload.get("ancestors")
if isinstance(direct_ancestors, dict):
return direct_ancestors
return None
def _build_msd_aggregation(
payload: Dict[str, Any],
domain_results: Dict[str, Dict[str, List[Dict[str, Any]]]],
) -> tuple[Optional[Dict[str, Any]], Optional[tuple[str, str, int]]]:
params = payload.get("params")
if not isinstance(params, dict):
return None, ("INVALID_PARAMS", "params is required for mid_section_defect profile", 400)
start_date, end_date = _extract_date_range(params)
if not start_date or not end_date:
return None, ("INVALID_PARAMS", "start_date/end_date is required in params", 400)
raw_loss_reasons = params.get("loss_reasons")
loss_reasons = parse_loss_reasons_param(raw_loss_reasons)
lineage_ancestors = _parse_lineage_payload(payload)
seed_container_ids = _normalize_strings(payload.get("seed_container_ids", []))
if not seed_container_ids and isinstance(lineage_ancestors, dict):
seed_container_ids = _normalize_strings(list(lineage_ancestors.keys()))
upstream_events = domain_results.get("upstream_history", {})
aggregation = build_trace_aggregation_from_events(
start_date,
end_date,
loss_reasons=loss_reasons,
seed_container_ids=seed_container_ids,
lineage_ancestors=lineage_ancestors,
upstream_events_by_cid=upstream_events,
)
if aggregation is None:
return None, ("EVENTS_AGGREGATION_FAILED", "aggregation service unavailable", 503)
if "error" in aggregation:
return None, ("EVENTS_AGGREGATION_FAILED", str(aggregation["error"]), 400)
return aggregation, None
@trace_bp.route("/seed-resolve", methods=["POST"])
@_TRACE_SEED_RATE_LIMIT
def seed_resolve():
payload = _json_body()
if payload is None:
return _error("INVALID_PARAMS", "request body must be JSON object", 400)
profile = _validate_profile(payload.get("profile"))
if profile is None:
return _error("INVALID_PROFILE", "unsupported profile", 400)
params = payload.get("params")
if not isinstance(params, dict):
return _error("INVALID_PARAMS", "params must be an object", 400)
seed_cache_key = _seed_cache_key(profile, params)
cached = cache_get(seed_cache_key)
if cached is not None:
return jsonify(cached)
request_cache_key = payload.get("cache_key")
logger.info(
"trace seed-resolve profile=%s correlation_cache_key=%s",
profile,
request_cache_key,
)
started = time.monotonic()
if profile == PROFILE_QUERY_TOOL:
resolved, route_error = _seed_resolve_query_tool(params)
else:
resolved, route_error = _seed_resolve_mid_section_defect(params)
elapsed = time.monotonic() - started
if elapsed > TRACE_STAGE_TIMEOUT_SECONDS:
return _timeout("seed_resolve")
if route_error is not None:
code, message, status = route_error
return _error(code, message, status)
response = {
"stage": "seed-resolve",
"seeds": resolved.get("seeds", []),
"seed_count": int(resolved.get("seed_count", 0)),
"cache_key": seed_cache_key,
}
cache_set(seed_cache_key, response, ttl=TRACE_CACHE_TTL_SECONDS)
return jsonify(response)
@trace_bp.route("/lineage", methods=["POST"])
@_TRACE_LINEAGE_RATE_LIMIT
def lineage():
payload = _json_body()
if payload is None:
return _error("INVALID_PARAMS", "request body must be JSON object", 400)
profile = _validate_profile(payload.get("profile"))
if profile is None:
return _error("INVALID_PROFILE", "unsupported profile", 400)
container_ids = _normalize_strings(payload.get("container_ids", []))
if not container_ids:
return _error("INVALID_PARAMS", "container_ids must contain at least one id", 400)
lineage_cache_key = _lineage_cache_key(container_ids)
cached = cache_get(lineage_cache_key)
if cached is not None:
return jsonify(cached)
logger.info(
"trace lineage profile=%s count=%s correlation_cache_key=%s",
profile,
len(container_ids),
payload.get("cache_key"),
)
started = time.monotonic()
try:
ancestors_raw = LineageEngine.resolve_full_genealogy(container_ids)
except Exception as exc:
if _is_timeout_exception(exc):
return _error("LINEAGE_TIMEOUT", "lineage query timed out", 504)
logger.error("lineage stage failed: %s", exc, exc_info=True)
return _error("LINEAGE_FAILED", "lineage stage failed", 500)
elapsed = time.monotonic() - started
if elapsed > TRACE_STAGE_TIMEOUT_SECONDS:
return _error("LINEAGE_TIMEOUT", "lineage query timed out", 504)
response = _build_lineage_response(container_ids, ancestors_raw)
cache_set(lineage_cache_key, response, ttl=TRACE_CACHE_TTL_SECONDS)
return jsonify(response)
@trace_bp.route("/events", methods=["POST"])
@_TRACE_EVENTS_RATE_LIMIT
def events():
payload = _json_body()
if payload is None:
return _error("INVALID_PARAMS", "request body must be JSON object", 400)
profile = _validate_profile(payload.get("profile"))
if profile is None:
return _error("INVALID_PROFILE", "unsupported profile", 400)
container_ids = _normalize_strings(payload.get("container_ids", []))
if not container_ids:
return _error("INVALID_PARAMS", "container_ids must contain at least one id", 400)
domains = _normalize_strings(payload.get("domains", []))
if not domains:
return _error("INVALID_PARAMS", "domains must contain at least one domain", 400)
invalid_domains = sorted(set(domains) - SUPPORTED_EVENT_DOMAINS)
if invalid_domains:
return _error(
"INVALID_PARAMS",
f"unsupported domains: {','.join(invalid_domains)}",
400,
)
events_cache_key = _events_cache_key(profile, domains, container_ids)
cached = cache_get(events_cache_key)
if cached is not None:
return jsonify(cached)
logger.info(
"trace events profile=%s domains=%s cid_count=%s correlation_cache_key=%s",
profile,
",".join(domains),
len(container_ids),
payload.get("cache_key"),
)
started = time.monotonic()
results: Dict[str, Dict[str, Any]] = {}
raw_domain_results: Dict[str, Dict[str, List[Dict[str, Any]]]] = {}
failed_domains: List[str] = []
for domain in domains:
try:
events_by_cid = EventFetcher.fetch_events(container_ids, domain)
raw_domain_results[domain] = events_by_cid
rows = _flatten_domain_records(events_by_cid)
results[domain] = {"data": rows, "count": len(rows)}
except Exception as exc:
logger.error("events stage domain failed domain=%s: %s", domain, exc, exc_info=True)
failed_domains.append(domain)
elapsed = time.monotonic() - started
if elapsed > TRACE_STAGE_TIMEOUT_SECONDS:
return _error("EVENTS_TIMEOUT", "events stage timed out", 504)
aggregation = None
if profile == PROFILE_MID_SECTION_DEFECT:
aggregation, agg_error = _build_msd_aggregation(payload, raw_domain_results)
if agg_error is not None:
code, message, status = agg_error
return _error(code, message, status)
response: Dict[str, Any] = {
"stage": "events",
"results": results,
"aggregation": aggregation,
}
if failed_domains:
response["error"] = "one or more domains failed"
response["code"] = "EVENTS_PARTIAL_FAILURE"
response["failed_domains"] = sorted(failed_domains)
cache_set(events_cache_key, response, ttl=TRACE_CACHE_TTL_SECONDS)
return jsonify(response)

View File

@@ -0,0 +1,237 @@
# -*- coding: utf-8 -*-
"""Unified event query fetcher with cache and domain-level policy metadata."""
from __future__ import annotations
import hashlib
import logging
import os
from collections import defaultdict
from typing import Any, Dict, List
from mes_dashboard.core.cache import cache_get, cache_set
from mes_dashboard.core.database import read_sql_df
from mes_dashboard.sql import QueryBuilder, SQLLoader
logger = logging.getLogger("mes_dashboard.event_fetcher")
ORACLE_IN_BATCH_SIZE = 1000
_DOMAIN_SPECS: Dict[str, Dict[str, Any]] = {
"history": {
"filter_column": "h.CONTAINERID",
"cache_ttl": 300,
"bucket": "event-history",
"max_env": "EVT_HISTORY_RATE_MAX_REQUESTS",
"window_env": "EVT_HISTORY_RATE_WINDOW_SECONDS",
"default_max": 20,
"default_window": 60,
},
"materials": {
"filter_column": "CONTAINERID",
"cache_ttl": 300,
"bucket": "event-materials",
"max_env": "EVT_MATERIALS_RATE_MAX_REQUESTS",
"window_env": "EVT_MATERIALS_RATE_WINDOW_SECONDS",
"default_max": 20,
"default_window": 60,
},
"rejects": {
"filter_column": "CONTAINERID",
"cache_ttl": 300,
"bucket": "event-rejects",
"max_env": "EVT_REJECTS_RATE_MAX_REQUESTS",
"window_env": "EVT_REJECTS_RATE_WINDOW_SECONDS",
"default_max": 20,
"default_window": 60,
},
"holds": {
"filter_column": "CONTAINERID",
"cache_ttl": 180,
"bucket": "event-holds",
"max_env": "EVT_HOLDS_RATE_MAX_REQUESTS",
"window_env": "EVT_HOLDS_RATE_WINDOW_SECONDS",
"default_max": 20,
"default_window": 60,
},
"jobs": {
"filter_column": "j.CONTAINERIDS",
"match_mode": "contains",
"cache_ttl": 180,
"bucket": "event-jobs",
"max_env": "EVT_JOBS_RATE_MAX_REQUESTS",
"window_env": "EVT_JOBS_RATE_WINDOW_SECONDS",
"default_max": 20,
"default_window": 60,
},
"upstream_history": {
"filter_column": "h.CONTAINERID",
"cache_ttl": 300,
"bucket": "event-upstream",
"max_env": "EVT_UPSTREAM_RATE_MAX_REQUESTS",
"window_env": "EVT_UPSTREAM_RATE_WINDOW_SECONDS",
"default_max": 20,
"default_window": 60,
},
}
def _env_int(name: str, default: int) -> int:
raw = os.getenv(name)
if raw is None:
return int(default)
try:
value = int(raw)
except (TypeError, ValueError):
return int(default)
return max(value, 1)
def _normalize_ids(container_ids: List[str]) -> List[str]:
if not container_ids:
return []
seen = set()
normalized: List[str] = []
for cid in container_ids:
if not isinstance(cid, str):
continue
value = cid.strip()
if not value or value in seen:
continue
seen.add(value)
normalized.append(value)
return normalized
class EventFetcher:
"""Fetches container-scoped event records with cache and batching."""
@staticmethod
def _cache_key(domain: str, container_ids: List[str]) -> str:
normalized = sorted(_normalize_ids(container_ids))
digest = hashlib.md5("|".join(normalized).encode("utf-8")).hexdigest()[:12]
return f"evt:{domain}:{digest}"
@staticmethod
def _get_rate_limit_config(domain: str) -> Dict[str, int | str]:
spec = _DOMAIN_SPECS.get(domain)
if spec is None:
raise ValueError(f"Unsupported event domain: {domain}")
return {
"bucket": spec["bucket"],
"max_attempts": _env_int(spec["max_env"], spec["default_max"]),
"window_seconds": _env_int(spec["window_env"], spec["default_window"]),
}
@staticmethod
def _build_domain_sql(domain: str, condition_sql: str) -> str:
if domain == "upstream_history":
return SQLLoader.load_with_params(
"mid_section_defect/upstream_history",
ANCESTOR_FILTER=condition_sql,
)
if domain == "history":
sql = SQLLoader.load("query_tool/lot_history")
sql = sql.replace("h.CONTAINERID = :container_id", condition_sql)
return sql.replace("{{ WORKCENTER_FILTER }}", "")
if domain == "materials":
sql = SQLLoader.load("query_tool/lot_materials")
return sql.replace("CONTAINERID = :container_id", condition_sql)
if domain == "rejects":
sql = SQLLoader.load("query_tool/lot_rejects")
return sql.replace("CONTAINERID = :container_id", condition_sql)
if domain == "holds":
sql = SQLLoader.load("query_tool/lot_holds")
return sql.replace("CONTAINERID = :container_id", condition_sql)
if domain == "jobs":
return f"""
SELECT
j.JOBID,
j.RESOURCEID,
j.RESOURCENAME,
j.JOBSTATUS,
j.JOBMODELNAME,
j.JOBORDERNAME,
j.CREATEDATE,
j.COMPLETEDATE,
j.CAUSECODENAME,
j.REPAIRCODENAME,
j.SYMPTOMCODENAME,
j.CONTAINERIDS,
j.CONTAINERNAMES,
NULL AS CONTAINERID
FROM DWH.DW_MES_JOB j
WHERE {condition_sql}
ORDER BY j.CREATEDATE DESC
"""
raise ValueError(f"Unsupported event domain: {domain}")
@staticmethod
def fetch_events(
container_ids: List[str],
domain: str,
) -> Dict[str, List[Dict[str, Any]]]:
"""Fetch event records grouped by CONTAINERID."""
if domain not in _DOMAIN_SPECS:
raise ValueError(f"Unsupported event domain: {domain}")
normalized_ids = _normalize_ids(container_ids)
if not normalized_ids:
return {}
cache_key = EventFetcher._cache_key(domain, normalized_ids)
cached = cache_get(cache_key)
if cached is not None:
return cached
grouped: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
spec = _DOMAIN_SPECS[domain]
filter_column = spec["filter_column"]
match_mode = spec.get("match_mode", "in")
for i in range(0, len(normalized_ids), ORACLE_IN_BATCH_SIZE):
batch = normalized_ids[i:i + ORACLE_IN_BATCH_SIZE]
builder = QueryBuilder()
if match_mode == "contains":
builder.add_or_like_conditions(filter_column, batch, position="both")
else:
builder.add_in_condition(filter_column, batch)
sql = EventFetcher._build_domain_sql(domain, builder.get_conditions_sql())
df = read_sql_df(sql, builder.params)
if df is None or df.empty:
continue
for _, row in df.iterrows():
if domain == "jobs":
record = row.to_dict()
containers = record.get("CONTAINERIDS")
if not isinstance(containers, str) or not containers:
continue
for cid in batch:
if cid in containers:
enriched = dict(record)
enriched["CONTAINERID"] = cid
grouped[cid].append(enriched)
continue
cid = row.get("CONTAINERID")
if not isinstance(cid, str) or not cid:
continue
grouped[cid].append(row.to_dict())
result = dict(grouped)
cache_set(cache_key, result, ttl=_DOMAIN_SPECS[domain]["cache_ttl"])
logger.info(
"EventFetcher fetched domain=%s queried_cids=%s hit_cids=%s",
domain,
len(normalized_ids),
len(result),
)
return result

View File

@@ -0,0 +1,221 @@
# -*- coding: utf-8 -*-
"""Unified LOT lineage resolution helpers."""
from __future__ import annotations
import logging
from collections import defaultdict
from typing import Any, Dict, List, Optional, Set
from mes_dashboard.core.database import read_sql_df
from mes_dashboard.sql import QueryBuilder, SQLLoader
logger = logging.getLogger("mes_dashboard.lineage_engine")
ORACLE_IN_BATCH_SIZE = 1000
MAX_SPLIT_DEPTH = 20
def _normalize_list(values: List[str]) -> List[str]:
"""Normalize string list while preserving input order."""
if not values:
return []
seen = set()
normalized: List[str] = []
for value in values:
if not isinstance(value, str):
continue
text = value.strip()
if not text or text in seen:
continue
seen.add(text)
normalized.append(text)
return normalized
def _safe_str(value: Any) -> Optional[str]:
"""Convert value to non-empty string if possible."""
if not isinstance(value, str):
return None
value = value.strip()
return value if value else None
class LineageEngine:
"""Unified split/merge genealogy resolver."""
@staticmethod
def resolve_split_ancestors(
container_ids: List[str],
initial_names: Optional[Dict[str, str]] = None,
) -> Dict[str, Dict[str, str]]:
"""Resolve split lineage with CONNECT BY NOCYCLE.
Returns:
{
"child_to_parent": {child_cid: parent_cid},
"cid_to_name": {cid: container_name},
}
"""
normalized_cids = _normalize_list(container_ids)
child_to_parent: Dict[str, str] = {}
cid_to_name: Dict[str, str] = {
cid: name
for cid, name in (initial_names or {}).items()
if _safe_str(cid) and _safe_str(name)
}
if not normalized_cids:
return {"child_to_parent": child_to_parent, "cid_to_name": cid_to_name}
for i in range(0, len(normalized_cids), ORACLE_IN_BATCH_SIZE):
batch = normalized_cids[i:i + ORACLE_IN_BATCH_SIZE]
builder = QueryBuilder()
builder.add_in_condition("c.CONTAINERID", batch)
sql = SQLLoader.load_with_params(
"lineage/split_ancestors",
CID_FILTER=builder.get_conditions_sql(),
)
df = read_sql_df(sql, builder.params)
if df is None or df.empty:
continue
for _, row in df.iterrows():
cid = _safe_str(row.get("CONTAINERID"))
if not cid:
continue
name = _safe_str(row.get("CONTAINERNAME"))
if name:
cid_to_name[cid] = name
depth_raw = row.get("SPLIT_DEPTH")
depth = int(depth_raw) if depth_raw is not None else 0
if depth > MAX_SPLIT_DEPTH:
continue
parent = _safe_str(row.get("SPLITFROMID"))
if parent and parent != cid:
child_to_parent.setdefault(cid, parent)
logger.info(
"Split ancestor resolution completed: seed=%s, edges=%s, names=%s",
len(normalized_cids),
len(child_to_parent),
len(cid_to_name),
)
return {"child_to_parent": child_to_parent, "cid_to_name": cid_to_name}
@staticmethod
def resolve_merge_sources(
container_names: List[str],
) -> Dict[str, List[str]]:
"""Resolve merge source lots from FINISHEDNAME."""
normalized_names = _normalize_list(container_names)
if not normalized_names:
return {}
result: Dict[str, Set[str]] = defaultdict(set)
for i in range(0, len(normalized_names), ORACLE_IN_BATCH_SIZE):
batch = normalized_names[i:i + ORACLE_IN_BATCH_SIZE]
builder = QueryBuilder()
builder.add_in_condition("ca.FINISHEDNAME", batch)
sql = SQLLoader.load_with_params(
"lineage/merge_sources",
FINISHED_NAME_FILTER=builder.get_conditions_sql(),
)
df = read_sql_df(sql, builder.params)
if df is None or df.empty:
continue
for _, row in df.iterrows():
finished_name = _safe_str(row.get("FINISHEDNAME"))
source_cid = _safe_str(row.get("SOURCE_CID"))
if not finished_name or not source_cid:
continue
result[finished_name].add(source_cid)
mapped = {k: sorted(v) for k, v in result.items()}
logger.info(
"Merge source resolution completed: finished_names=%s, mapped=%s",
len(normalized_names),
len(mapped),
)
return mapped
@staticmethod
def resolve_full_genealogy(
container_ids: List[str],
initial_names: Optional[Dict[str, str]] = None,
) -> Dict[str, Set[str]]:
"""Resolve combined split + merge genealogy graph.
Returns:
{seed_cid: set(ancestor_cids)}
"""
seed_cids = _normalize_list(container_ids)
if not seed_cids:
return {}
split_result = LineageEngine.resolve_split_ancestors(seed_cids, initial_names)
child_to_parent = split_result["child_to_parent"]
cid_to_name = split_result["cid_to_name"]
ancestors: Dict[str, Set[str]] = {}
for seed in seed_cids:
visited: Set[str] = set()
current = seed
depth = 0
while current in child_to_parent and depth < MAX_SPLIT_DEPTH:
depth += 1
parent = child_to_parent[current]
if parent in visited:
break
visited.add(parent)
current = parent
ancestors[seed] = visited
all_names = [name for name in cid_to_name.values() if _safe_str(name)]
merge_source_map = LineageEngine.resolve_merge_sources(all_names)
if not merge_source_map:
return ancestors
merge_source_cids_all: Set[str] = set()
for seed in seed_cids:
self_and_ancestors = ancestors[seed] | {seed}
for cid in list(self_and_ancestors):
name = cid_to_name.get(cid)
if not name:
continue
for source_cid in merge_source_map.get(name, []):
if source_cid == cid or source_cid in self_and_ancestors:
continue
ancestors[seed].add(source_cid)
merge_source_cids_all.add(source_cid)
seen = set(seed_cids) | set(child_to_parent.keys()) | set(child_to_parent.values())
new_merge_cids = list(merge_source_cids_all - seen)
if not new_merge_cids:
return ancestors
merge_split_result = LineageEngine.resolve_split_ancestors(new_merge_cids)
merge_child_to_parent = merge_split_result["child_to_parent"]
for seed in seed_cids:
for merge_cid in list(ancestors[seed] & merge_source_cids_all):
current = merge_cid
depth = 0
while current in merge_child_to_parent and depth < MAX_SPLIT_DEPTH:
depth += 1
parent = merge_child_to_parent[current]
if parent in ancestors[seed]:
break
ancestors[seed].add(parent)
current = parent
return ancestors

View File

@@ -33,15 +33,16 @@ import pandas as pd
from mes_dashboard.core.database import read_sql_df
from mes_dashboard.core.cache import cache_get, cache_set, make_cache_key
from mes_dashboard.core.redis_client import try_acquire_lock, release_lock
from mes_dashboard.sql import SQLLoader, QueryBuilder
from mes_dashboard.sql import SQLLoader
from mes_dashboard.services.event_fetcher import EventFetcher
from mes_dashboard.services.lineage_engine import LineageEngine
logger = logging.getLogger('mes_dashboard.mid_section_defect')
# Constants
MAX_QUERY_DAYS = 180
CACHE_TTL_TMTT = 300 # 5 min for TMTT detection data
CACHE_TTL_LOSS_REASONS = 86400 # 24h for loss reason list (daily sync)
ORACLE_IN_BATCH_SIZE = 1000 # Oracle IN clause limit
CACHE_TTL_LOSS_REASONS = 86400 # 24h for loss reason list (daily sync)
# Distributed lock settings for query_analysis cold-cache path
ANALYSIS_LOCK_TTL_SECONDS = 120
@@ -82,11 +83,11 @@ CSV_COLUMNS = [
# Public API
# ============================================================
def query_analysis(
start_date: str,
end_date: str,
loss_reasons: Optional[List[str]] = None,
) -> Optional[Dict[str, Any]]:
def query_analysis(
start_date: str,
end_date: str,
loss_reasons: Optional[List[str]] = None,
) -> Optional[Dict[str, Any]]:
"""Main entry point for mid-section defect traceability analysis.
Args:
@@ -217,12 +218,161 @@ def query_analysis(
finally:
if lock_acquired:
release_lock(lock_name)
def query_analysis_detail(
start_date: str,
end_date: str,
loss_reasons: Optional[List[str]] = None,
def parse_loss_reasons_param(loss_reasons: Any) -> Optional[List[str]]:
"""Normalize loss reason input from API payloads.
Accepts comma-separated strings or list-like inputs.
Returns None when no valid value is provided.
"""
if loss_reasons is None:
return None
values: List[str]
if isinstance(loss_reasons, str):
values = [item.strip() for item in loss_reasons.split(',') if item.strip()]
elif isinstance(loss_reasons, (list, tuple, set)):
values = []
for item in loss_reasons:
if not isinstance(item, str):
continue
text = item.strip()
if text:
values.append(text)
else:
return None
if not values:
return None
deduped: List[str] = []
seen = set()
for value in values:
if value in seen:
continue
seen.add(value)
deduped.append(value)
return deduped or None
def resolve_trace_seed_lots(
start_date: str,
end_date: str,
) -> Optional[Dict[str, Any]]:
"""Resolve seed lots for staged mid-section trace API."""
error = _validate_date_range(start_date, end_date)
if error:
return {'error': error}
tmtt_df = _fetch_tmtt_data(start_date, end_date)
if tmtt_df is None:
return None
if tmtt_df.empty:
return {'seeds': [], 'seed_count': 0}
seeds = []
unique_rows = tmtt_df.drop_duplicates(subset=['CONTAINERID'])
for _, row in unique_rows.iterrows():
cid = _safe_str(row.get('CONTAINERID'))
if not cid:
continue
lot_id = _safe_str(row.get('CONTAINERNAME')) or cid
seeds.append({
'container_id': cid,
'container_name': lot_id,
'lot_id': lot_id,
})
seeds.sort(key=lambda item: (item.get('lot_id', ''), item.get('container_id', '')))
return {
'seeds': seeds,
'seed_count': len(seeds),
}
def build_trace_aggregation_from_events(
start_date: str,
end_date: str,
*,
loss_reasons: Optional[List[str]] = None,
seed_container_ids: Optional[List[str]] = None,
lineage_ancestors: Optional[Dict[str, Any]] = None,
upstream_events_by_cid: Optional[Dict[str, List[Dict[str, Any]]]] = None,
) -> Optional[Dict[str, Any]]:
"""Build mid-section summary payload from staged events data."""
error = _validate_date_range(start_date, end_date)
if error:
return {'error': error}
normalized_loss_reasons = parse_loss_reasons_param(loss_reasons)
tmtt_df = _fetch_tmtt_data(start_date, end_date)
if tmtt_df is None:
return None
if tmtt_df.empty:
empty_result = _empty_result()
return {
'kpi': empty_result['kpi'],
'charts': empty_result['charts'],
'daily_trend': empty_result['daily_trend'],
'available_loss_reasons': empty_result['available_loss_reasons'],
'genealogy_status': empty_result['genealogy_status'],
'detail_total_count': 0,
}
available_loss_reasons = sorted(
tmtt_df.loc[tmtt_df['REJECTQTY'] > 0, 'LOSSREASONNAME']
.dropna().unique().tolist()
)
if normalized_loss_reasons:
filtered_df = tmtt_df[
(tmtt_df['LOSSREASONNAME'].isin(normalized_loss_reasons))
| (tmtt_df['REJECTQTY'] == 0)
| (tmtt_df['LOSSREASONNAME'].isna())
].copy()
else:
filtered_df = tmtt_df
tmtt_data = _build_tmtt_lookup(filtered_df)
normalized_ancestors = _normalize_lineage_ancestors(
lineage_ancestors,
seed_container_ids=seed_container_ids,
fallback_seed_ids=list(tmtt_data.keys()),
)
normalized_upstream = _normalize_upstream_event_records(upstream_events_by_cid or {})
attribution = _attribute_defects(
tmtt_data,
normalized_ancestors,
normalized_upstream,
normalized_loss_reasons,
)
detail = _build_detail_table(filtered_df, normalized_ancestors, normalized_upstream)
seed_ids = [
cid for cid in (seed_container_ids or list(tmtt_data.keys()))
if isinstance(cid, str) and cid.strip()
]
genealogy_status = 'ready'
if seed_ids and lineage_ancestors is None:
genealogy_status = 'error'
return {
'kpi': _build_kpi(filtered_df, attribution, normalized_loss_reasons),
'charts': _build_all_charts(attribution, tmtt_data),
'daily_trend': _build_daily_trend(filtered_df, normalized_loss_reasons),
'available_loss_reasons': available_loss_reasons,
'genealogy_status': genealogy_status,
'detail_total_count': len(detail),
}
def query_analysis_detail(
start_date: str,
end_date: str,
loss_reasons: Optional[List[str]] = None,
page: int = 1,
page_size: int = 200,
) -> Optional[Dict[str, Any]]:
@@ -428,193 +578,14 @@ def _fetch_tmtt_data(start_date: str, end_date: str) -> Optional[pd.DataFrame]:
# Query 2: LOT Genealogy
# ============================================================
def _resolve_full_genealogy(
tmtt_cids: List[str],
tmtt_names: Dict[str, str],
) -> Dict[str, Set[str]]:
"""Resolve full genealogy for TMTT lots via SPLITFROMID + COMBINEDASSYLOTS.
Step 1: BFS upward through DW_MES_CONTAINER.SPLITFROMID
Step 2: Merge expansion via DW_MES_PJ_COMBINEDASSYLOTS
Step 3: BFS on merge source CIDs (one more round)
Args:
tmtt_cids: TMTT lot CONTAINERIDs
tmtt_names: {cid: containername} from TMTT detection data
Returns:
{tmtt_cid: set(all ancestor CIDs)}
"""
# ---- Step 1: Split chain BFS upward ----
child_to_parent, cid_to_name = _bfs_split_chain(tmtt_cids, tmtt_names)
# Build initial ancestor sets per TMTT lot (walk up split chain)
ancestors: Dict[str, Set[str]] = {}
for tmtt_cid in tmtt_cids:
visited: Set[str] = set()
current = tmtt_cid
while current in child_to_parent:
parent = child_to_parent[current]
if parent in visited:
break # cycle protection
visited.add(parent)
current = parent
ancestors[tmtt_cid] = visited
# ---- Step 2: Merge expansion via COMBINEDASSYLOTS ----
all_names = set(cid_to_name.values())
if not all_names:
_log_genealogy_summary(ancestors, tmtt_cids, 0)
return ancestors
merge_source_map = _fetch_merge_sources(list(all_names))
if not merge_source_map:
_log_genealogy_summary(ancestors, tmtt_cids, 0)
return ancestors
# Reverse map: name → set of CIDs with that name
name_to_cids: Dict[str, Set[str]] = defaultdict(set)
for cid, name in cid_to_name.items():
name_to_cids[name].add(cid)
# Expand ancestors with merge sources
merge_source_cids_all: Set[str] = set()
for tmtt_cid in tmtt_cids:
self_and_ancestors = ancestors[tmtt_cid] | {tmtt_cid}
for cid in list(self_and_ancestors):
name = cid_to_name.get(cid)
if name and name in merge_source_map:
for src_cid in merge_source_map[name]:
if src_cid != cid and src_cid not in self_and_ancestors:
ancestors[tmtt_cid].add(src_cid)
merge_source_cids_all.add(src_cid)
# ---- Step 3: BFS on merge source CIDs ----
seen = set(tmtt_cids) | set(child_to_parent.values()) | set(child_to_parent.keys())
new_merge_cids = list(merge_source_cids_all - seen)
if new_merge_cids:
merge_c2p, _ = _bfs_split_chain(new_merge_cids, {})
child_to_parent.update(merge_c2p)
# Walk up merge sources' split chains for each TMTT lot
for tmtt_cid in tmtt_cids:
for merge_cid in list(ancestors[tmtt_cid] & merge_source_cids_all):
current = merge_cid
while current in merge_c2p:
parent = merge_c2p[current]
if parent in ancestors[tmtt_cid]:
break
ancestors[tmtt_cid].add(parent)
current = parent
_log_genealogy_summary(ancestors, tmtt_cids, len(merge_source_cids_all))
return ancestors
def _bfs_split_chain(
start_cids: List[str],
initial_names: Dict[str, str],
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""BFS upward through DW_MES_CONTAINER.SPLITFROMID.
Args:
start_cids: Starting CONTAINERIDs
initial_names: Pre-known {cid: containername} mappings
Returns:
child_to_parent: {child_cid: parent_cid} for all split edges
cid_to_name: {cid: containername} for all encountered CIDs
"""
child_to_parent: Dict[str, str] = {}
cid_to_name: Dict[str, str] = dict(initial_names)
seen: Set[str] = set(start_cids)
frontier = list(start_cids)
bfs_round = 0
while frontier:
bfs_round += 1
batch_results: List[Dict[str, Any]] = []
for i in range(0, len(frontier), ORACLE_IN_BATCH_SIZE):
batch = frontier[i:i + ORACLE_IN_BATCH_SIZE]
builder = QueryBuilder()
builder.add_in_condition("c.CONTAINERID", batch)
sql = SQLLoader.load_with_params(
"mid_section_defect/split_chain",
CID_FILTER=builder.get_conditions_sql(),
)
try:
df = read_sql_df(sql, builder.params)
if df is not None and not df.empty:
batch_results.extend(df.to_dict('records'))
except Exception as exc:
logger.warning(f"Split chain BFS round {bfs_round} batch failed: {exc}")
new_parents: Set[str] = set()
for row in batch_results:
cid = row['CONTAINERID']
split_from = row.get('SPLITFROMID')
name = row.get('CONTAINERNAME')
if isinstance(name, str) and name:
cid_to_name[cid] = name
if isinstance(split_from, str) and split_from and cid != split_from:
child_to_parent[cid] = split_from
if split_from not in seen:
new_parents.add(split_from)
seen.add(split_from)
frontier = list(new_parents)
if bfs_round > 20:
logger.warning("Split chain BFS exceeded 20 rounds, stopping")
break
logger.info(
f"Split chain BFS: {bfs_round} rounds, "
f"{len(child_to_parent)} split edges, "
f"{len(cid_to_name)} names collected"
)
return child_to_parent, cid_to_name
def _fetch_merge_sources(
finished_names: List[str],
) -> Dict[str, List[str]]:
"""Find source lots merged into finished lots via COMBINEDASSYLOTS.
Args:
finished_names: CONTAINERNAMEs to look up as FINISHEDNAME
Returns:
{finished_name: [source_cid, ...]}
"""
result: Dict[str, List[str]] = {}
for i in range(0, len(finished_names), ORACLE_IN_BATCH_SIZE):
batch = finished_names[i:i + ORACLE_IN_BATCH_SIZE]
builder = QueryBuilder()
builder.add_in_condition("ca.FINISHEDNAME", batch)
sql = SQLLoader.load_with_params(
"mid_section_defect/merge_lookup",
FINISHED_NAME_FILTER=builder.get_conditions_sql(),
)
try:
df = read_sql_df(sql, builder.params)
if df is not None and not df.empty:
for _, row in df.iterrows():
fn = row['FINISHEDNAME']
src = row['SOURCE_CID']
if isinstance(fn, str) and fn and isinstance(src, str) and src:
result.setdefault(fn, []).append(src)
except Exception as exc:
logger.warning(f"Merge lookup batch failed: {exc}")
if result:
total_sources = sum(len(v) for v in result.values())
logger.info(
f"Merge lookup: {len(result)} finished names → {total_sources} source CIDs"
)
return result
def _resolve_full_genealogy(
tmtt_cids: List[str],
tmtt_names: Dict[str, str],
) -> Dict[str, Set[str]]:
"""Resolve full genealogy for TMTT lots via shared LineageEngine."""
ancestors = LineageEngine.resolve_full_genealogy(tmtt_cids, tmtt_names)
_log_genealogy_summary(ancestors, tmtt_cids, 0)
return ancestors
def _log_genealogy_summary(
@@ -646,60 +617,81 @@ def _fetch_upstream_history(
Returns:
{containerid: [{'workcenter_group': ..., 'equipment_name': ..., ...}, ...]}
"""
if not all_cids:
return {}
unique_cids = list(set(all_cids))
all_rows = []
# Batch query in chunks of ORACLE_IN_BATCH_SIZE
for i in range(0, len(unique_cids), ORACLE_IN_BATCH_SIZE):
batch = unique_cids[i:i + ORACLE_IN_BATCH_SIZE]
builder = QueryBuilder()
builder.add_in_condition("h.CONTAINERID", batch)
conditions_sql = builder.get_conditions_sql()
params = builder.params
sql = SQLLoader.load_with_params(
"mid_section_defect/upstream_history",
ANCESTOR_FILTER=conditions_sql,
)
try:
df = read_sql_df(sql, params)
if df is not None and not df.empty:
all_rows.append(df)
except Exception as exc:
logger.error(
f"Upstream history batch {i//ORACLE_IN_BATCH_SIZE + 1} failed: {exc}",
exc_info=True,
)
if not all_rows:
return {}
combined = pd.concat(all_rows, ignore_index=True)
result: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
for _, row in combined.iterrows():
cid = row['CONTAINERID']
group_name = _safe_str(row.get('WORKCENTER_GROUP'))
if not group_name:
group_name = '(未知)'
result[cid].append({
'workcenter_group': group_name,
'equipment_id': _safe_str(row.get('EQUIPMENTID')),
'equipment_name': _safe_str(row.get('EQUIPMENTNAME')),
'spec_name': _safe_str(row.get('SPECNAME')),
'track_in_time': _safe_str(row.get('TRACKINTIMESTAMP')),
})
if not all_cids:
return {}
unique_cids = list(set(all_cids))
events_by_cid = EventFetcher.fetch_events(unique_cids, "upstream_history")
result = _normalize_upstream_event_records(events_by_cid)
logger.info(
f"Upstream history: {len(result)} lots with classified records, "
f"from {len(unique_cids)} queried CIDs"
)
return dict(result)
return dict(result)
def _normalize_lineage_ancestors(
lineage_ancestors: Optional[Dict[str, Any]],
*,
seed_container_ids: Optional[List[str]] = None,
fallback_seed_ids: Optional[List[str]] = None,
) -> Dict[str, Set[str]]:
"""Normalize lineage payload to {seed_cid: set(ancestor_cid)}."""
ancestors: Dict[str, Set[str]] = {}
if isinstance(lineage_ancestors, dict):
for seed, raw_values in lineage_ancestors.items():
seed_cid = _safe_str(seed)
if not seed_cid:
continue
values = raw_values if isinstance(raw_values, (list, tuple, set)) else []
normalized_values: Set[str] = set()
for value in values:
ancestor_cid = _safe_str(value)
if ancestor_cid and ancestor_cid != seed_cid:
normalized_values.add(ancestor_cid)
ancestors[seed_cid] = normalized_values
candidate_seeds = []
for seed in (seed_container_ids or []):
seed_cid = _safe_str(seed)
if seed_cid:
candidate_seeds.append(seed_cid)
if not candidate_seeds:
for seed in (fallback_seed_ids or []):
seed_cid = _safe_str(seed)
if seed_cid:
candidate_seeds.append(seed_cid)
for seed_cid in candidate_seeds:
ancestors.setdefault(seed_cid, set())
return ancestors
def _normalize_upstream_event_records(
events_by_cid: Dict[str, List[Dict[str, Any]]],
) -> Dict[str, List[Dict[str, Any]]]:
"""Normalize EventFetcher upstream payload into attribution-ready records."""
result: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
for cid, events in events_by_cid.items():
cid_value = _safe_str(cid)
if not cid_value:
continue
for event in events:
group_name = _safe_str(event.get('WORKCENTER_GROUP'))
if not group_name:
group_name = '(未知)'
result[cid_value].append({
'workcenter_group': group_name,
'equipment_id': _safe_str(event.get('EQUIPMENTID')),
'equipment_name': _safe_str(event.get('EQUIPMENTNAME')),
'spec_name': _safe_str(event.get('SPECNAME')),
'track_in_time': _safe_str(event.get('TRACKINTIMESTAMP')),
})
return dict(result)
# ============================================================

View File

@@ -18,14 +18,15 @@ Architecture:
import csv
import io
import logging
from datetime import datetime, timedelta
from decimal import Decimal
from typing import Any, Dict, List, Optional, Generator
from datetime import datetime, timedelta
from decimal import Decimal
from typing import Any, Dict, List, Optional, Generator
import pandas as pd
from mes_dashboard.core.database import read_sql_df
from mes_dashboard.sql import SQLLoader
from mes_dashboard.sql import QueryBuilder, SQLLoader
from mes_dashboard.services.event_fetcher import EventFetcher
try:
from mes_dashboard.core.database import read_sql_df_slow
@@ -122,59 +123,7 @@ def validate_equipment_input(equipment_ids: List[str]) -> Optional[str]:
return None
# ============================================================
# Helper Functions
# ============================================================
def _build_in_clause(values: List[str], max_chunk_size: int = BATCH_SIZE) -> List[str]:
"""Build SQL IN clause lists for values.
Oracle has a limit of ~1000 items per IN clause, so we chunk if needed.
Args:
values: List of values.
max_chunk_size: Maximum items per IN clause.
Returns:
List of SQL IN clause strings (e.g., "'val1', 'val2', 'val3'").
"""
if not values:
return []
# Escape single quotes
escaped = [v.replace("'", "''") for v in values]
# Chunk into groups
chunks = []
for i in range(0, len(escaped), max_chunk_size):
chunk = escaped[i:i + max_chunk_size]
chunks.append("'" + "', '".join(chunk) + "'")
return chunks
def _build_in_filter(values: List[str], column: str) -> str:
"""Build SQL IN filter clause.
Args:
values: List of values.
column: Column name.
Returns:
SQL condition string.
"""
chunks = _build_in_clause(values)
if not chunks:
return "1=0"
if len(chunks) == 1:
return f"{column} IN ({chunks[0]})"
conditions = [f"{column} IN ({chunk})" for chunk in chunks]
return "(" + " OR ".join(conditions) + ")"
def _df_to_records(df: pd.DataFrame) -> List[Dict[str, Any]]:
def _df_to_records(df: pd.DataFrame) -> List[Dict[str, Any]]:
"""Convert DataFrame to list of records with proper type handling.
Args:
@@ -250,7 +199,7 @@ def resolve_lots(input_type: str, values: List[str]) -> Dict[str, Any]:
return {'error': f'解析失敗: {str(exc)}'}
def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]:
def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]:
"""Resolve LOT IDs (CONTAINERNAME) to CONTAINERID.
Args:
@@ -259,23 +208,14 @@ def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]:
Returns:
Resolution result dict.
"""
in_filter = _build_in_filter(lot_ids, 'CONTAINERNAME')
sql = SQLLoader.load("query_tool/lot_resolve_id")
sql = sql.replace("{{ CONTAINER_NAMES }}", in_filter.replace("CONTAINERNAME IN (", "").rstrip(")"))
# Direct IN clause construction
sql = f"""
SELECT
CONTAINERID,
CONTAINERNAME,
MFGORDERNAME,
SPECNAME,
QTY
FROM DWH.DW_MES_CONTAINER
WHERE {in_filter}
"""
df = read_sql_df(sql, {})
builder = QueryBuilder()
builder.add_in_condition("CONTAINERNAME", lot_ids)
sql = SQLLoader.load_with_params(
"query_tool/lot_resolve_id",
CONTAINER_FILTER=builder.get_conditions_sql(),
)
df = read_sql_df(sql, builder.params)
data = _df_to_records(df)
# Map results
@@ -305,7 +245,7 @@ def _resolve_by_lot_id(lot_ids: List[str]) -> Dict[str, Any]:
}
def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]:
def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]:
"""Resolve serial numbers (FINISHEDNAME) to CONTAINERID.
Note: One serial number may map to multiple CONTAINERIDs.
@@ -316,21 +256,14 @@ def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]:
Returns:
Resolution result dict.
"""
in_filter = _build_in_filter(serial_numbers, 'p.FINISHEDNAME')
# JOIN with CONTAINER to get LOT ID (CONTAINERNAME)
sql = f"""
SELECT DISTINCT
p.CONTAINERID,
p.FINISHEDNAME,
c.CONTAINERNAME,
c.SPECNAME
FROM DWH.DW_MES_PJ_COMBINEDASSYLOTS p
LEFT JOIN DWH.DW_MES_CONTAINER c ON p.CONTAINERID = c.CONTAINERID
WHERE {in_filter}
"""
df = read_sql_df(sql, {})
builder = QueryBuilder()
builder.add_in_condition("p.FINISHEDNAME", serial_numbers)
sql = SQLLoader.load_with_params(
"query_tool/lot_resolve_serial",
SERIAL_FILTER=builder.get_conditions_sql(),
)
df = read_sql_df(sql, builder.params)
data = _df_to_records(df)
# Group by serial number
@@ -370,7 +303,7 @@ def _resolve_by_serial_number(serial_numbers: List[str]) -> Dict[str, Any]:
}
def _resolve_by_work_order(work_orders: List[str]) -> Dict[str, Any]:
def _resolve_by_work_order(work_orders: List[str]) -> Dict[str, Any]:
"""Resolve work orders (PJ_WORKORDER) to CONTAINERID.
Note: One work order may expand to many CONTAINERIDs (can be 100+).
@@ -381,21 +314,14 @@ def _resolve_by_work_order(work_orders: List[str]) -> Dict[str, Any]:
Returns:
Resolution result dict.
"""
in_filter = _build_in_filter(work_orders, 'h.PJ_WORKORDER')
# JOIN with CONTAINER to get LOT ID (CONTAINERNAME)
sql = f"""
SELECT DISTINCT
h.CONTAINERID,
h.PJ_WORKORDER,
c.CONTAINERNAME,
c.SPECNAME
FROM DWH.DW_MES_LOTWIPHISTORY h
LEFT JOIN DWH.DW_MES_CONTAINER c ON h.CONTAINERID = c.CONTAINERID
WHERE {in_filter}
"""
df = read_sql_df(sql, {})
builder = QueryBuilder()
builder.add_in_condition("h.PJ_WORKORDER", work_orders)
sql = SQLLoader.load_with_params(
"query_tool/lot_resolve_work_order",
WORK_ORDER_FILTER=builder.get_conditions_sql(),
)
df = read_sql_df(sql, builder.params)
data = _df_to_records(df)
# Group by work order
@@ -455,10 +381,10 @@ def _get_workcenters_for_groups(groups: List[str]) -> List[str]:
return get_workcenters_for_groups(groups)
def get_lot_history(
container_id: str,
workcenter_groups: Optional[List[str]] = None
) -> Dict[str, Any]:
def get_lot_history(
container_id: str,
workcenter_groups: Optional[List[str]] = None
) -> Dict[str, Any]:
"""Get production history for a LOT.
Args:
@@ -471,28 +397,27 @@ def get_lot_history(
if not container_id:
return {'error': '請指定 CONTAINERID'}
try:
sql = SQLLoader.load("query_tool/lot_history")
params = {'container_id': container_id}
# Add workcenter filter if groups specified
workcenter_filter = ""
if workcenter_groups:
workcenters = _get_workcenters_for_groups(workcenter_groups)
if workcenters:
workcenter_filter = f"AND {_build_in_filter(workcenters, 'h.WORKCENTERNAME')}"
logger.debug(
f"Filtering by {len(workcenter_groups)} groups "
f"({len(workcenters)} workcenters)"
)
# Replace placeholder in SQL
sql = sql.replace("{{ WORKCENTER_FILTER }}", workcenter_filter)
df = read_sql_df(sql, params)
data = _df_to_records(df)
logger.debug(f"LOT history: {len(data)} records for {container_id}")
try:
events_by_cid = EventFetcher.fetch_events([container_id], "history")
rows = list(events_by_cid.get(container_id, []))
if workcenter_groups:
workcenters = _get_workcenters_for_groups(workcenter_groups)
if workcenters:
workcenter_set = set(workcenters)
rows = [
row
for row in rows
if row.get('WORKCENTERNAME') in workcenter_set
]
logger.debug(
f"Filtering by {len(workcenter_groups)} groups "
f"({len(workcenters)} workcenters)"
)
data = _df_to_records(pd.DataFrame(rows))
logger.debug(f"LOT history: {len(data)} records for {container_id}")
return {
'data': data,
@@ -563,7 +488,7 @@ def get_adjacent_lots(
# LOT Association Functions
# ============================================================
def get_lot_materials(container_id: str) -> Dict[str, Any]:
def get_lot_materials(container_id: str) -> Dict[str, Any]:
"""Get material consumption records for a LOT.
Args:
@@ -575,12 +500,9 @@ def get_lot_materials(container_id: str) -> Dict[str, Any]:
if not container_id:
return {'error': '請指定 CONTAINERID'}
try:
sql = SQLLoader.load("query_tool/lot_materials")
params = {'container_id': container_id}
df = read_sql_df(sql, params)
data = _df_to_records(df)
try:
events_by_cid = EventFetcher.fetch_events([container_id], "materials")
data = _df_to_records(pd.DataFrame(events_by_cid.get(container_id, [])))
logger.debug(f"LOT materials: {len(data)} records for {container_id}")
@@ -595,7 +517,7 @@ def get_lot_materials(container_id: str) -> Dict[str, Any]:
return {'error': f'查詢失敗: {str(exc)}'}
def get_lot_rejects(container_id: str) -> Dict[str, Any]:
def get_lot_rejects(container_id: str) -> Dict[str, Any]:
"""Get reject (defect) records for a LOT.
Args:
@@ -607,12 +529,9 @@ def get_lot_rejects(container_id: str) -> Dict[str, Any]:
if not container_id:
return {'error': '請指定 CONTAINERID'}
try:
sql = SQLLoader.load("query_tool/lot_rejects")
params = {'container_id': container_id}
df = read_sql_df(sql, params)
data = _df_to_records(df)
try:
events_by_cid = EventFetcher.fetch_events([container_id], "rejects")
data = _df_to_records(pd.DataFrame(events_by_cid.get(container_id, [])))
logger.debug(f"LOT rejects: {len(data)} records for {container_id}")
@@ -627,7 +546,7 @@ def get_lot_rejects(container_id: str) -> Dict[str, Any]:
return {'error': f'查詢失敗: {str(exc)}'}
def get_lot_holds(container_id: str) -> Dict[str, Any]:
def get_lot_holds(container_id: str) -> Dict[str, Any]:
"""Get HOLD/RELEASE records for a LOT.
Args:
@@ -639,12 +558,9 @@ def get_lot_holds(container_id: str) -> Dict[str, Any]:
if not container_id:
return {'error': '請指定 CONTAINERID'}
try:
sql = SQLLoader.load("query_tool/lot_holds")
params = {'container_id': container_id}
df = read_sql_df(sql, params)
data = _df_to_records(df)
try:
events_by_cid = EventFetcher.fetch_events([container_id], "holds")
data = _df_to_records(pd.DataFrame(events_by_cid.get(container_id, [])))
logger.debug(f"LOT holds: {len(data)} records for {container_id}")
@@ -659,10 +575,11 @@ def get_lot_holds(container_id: str) -> Dict[str, Any]:
return {'error': f'查詢失敗: {str(exc)}'}
def get_lot_split_merge_history(
work_order: str,
current_container_id: str = None
) -> Dict[str, Any]:
def get_lot_split_merge_history(
work_order: str,
current_container_id: str = None,
full_history: bool = False,
) -> Dict[str, Any]:
"""Get complete split/merge history for a work order (完整拆併批歷史).
Queries DW_MES_HM_LOTMOVEOUT for SplitLot and CombineLot operations
@@ -679,9 +596,11 @@ def get_lot_split_merge_history(
- A00-001-01: Split at production station (製程站點拆分)
- A00-001-01C: Split at TMTT (TMTT 拆分)
Args:
work_order: MFGORDERNAME value (e.g., GA25120713)
current_container_id: Current LOT's CONTAINERID for highlighting
Args:
work_order: MFGORDERNAME value (e.g., GA25120713)
current_container_id: Current LOT's CONTAINERID for highlighting
full_history: If True, query complete history using slow connection.
If False (default), query only last 6 months with row limit.
Returns:
Dict with 'data' (split/merge history records) and 'total', or 'error'.
@@ -690,16 +609,29 @@ def get_lot_split_merge_history(
return {'error': '請指定工單號', 'data': [], 'total': 0}
try:
sql = SQLLoader.load("query_tool/lot_split_merge_history")
params = {'work_order': work_order}
logger.info(f"Starting split/merge history query for MFGORDERNAME={work_order}")
# Use slow query connection with 120s timeout
# Note: DW_MES_HM_LOTMOVEOUT has 48M rows, no index on CONTAINERID/FROMCONTAINERID
# Query by MFGORDERNAME is faster but still needs extra time
df = read_sql_df_slow(sql, params, timeout_seconds=120)
data = _df_to_records(df)
builder = QueryBuilder()
builder.add_in_condition("MFGORDERNAME", [work_order])
fast_time_window = "AND h.TXNDATE >= ADD_MONTHS(SYSDATE, -6)"
fast_row_limit = "FETCH FIRST 500 ROWS ONLY"
sql = SQLLoader.load_with_params(
"query_tool/lot_split_merge_history",
WORK_ORDER_FILTER=builder.get_conditions_sql(),
TIME_WINDOW="" if full_history else fast_time_window,
ROW_LIMIT="" if full_history else fast_row_limit,
)
params = builder.params
mode = "full" if full_history else "fast"
logger.info(
f"Starting split/merge history query for MFGORDERNAME={work_order} mode={mode}"
)
if full_history:
# Full mode uses dedicated slow query timeout path.
df = read_sql_df_slow(sql, params, timeout_seconds=120)
else:
df = read_sql_df(sql, params)
data = _df_to_records(df)
# Process records for display
processed = []
@@ -734,11 +666,12 @@ def get_lot_split_merge_history(
logger.info(f"Split/merge history completed: {len(processed)} records for MFGORDERNAME={work_order}")
return {
'data': processed,
'total': len(processed),
'work_order': work_order,
}
return {
'data': processed,
'total': len(processed),
'work_order': work_order,
'mode': mode,
}
except Exception as exc:
error_str = str(exc)
@@ -784,10 +717,11 @@ def _get_mfg_order_for_lot(container_id: str) -> Optional[str]:
return None
def get_lot_splits(
container_id: str,
include_production_history: bool = True # Uses dedicated slow query connection with 120s timeout
) -> Dict[str, Any]:
def get_lot_splits(
container_id: str,
include_production_history: bool = True,
full_history: bool = False,
) -> Dict[str, Any]:
"""Get combined split/merge data for a LOT (拆併批紀錄).
Data sources:
@@ -798,9 +732,10 @@ def get_lot_splits(
Production history now queries by MFGORDERNAME (indexed) instead of CONTAINERID
for much better performance (~1 second vs 40+ seconds).
Args:
container_id: CONTAINERID (16-char hex)
include_production_history: If True (default), include production history query.
Args:
container_id: CONTAINERID (16-char hex)
include_production_history: If True (default), include production history query.
full_history: If True, query split/merge history without fast-mode limits.
Returns:
Dict with 'production_history', 'serial_numbers', and totals.
@@ -833,10 +768,11 @@ def get_lot_splits(
if mfg_order:
logger.info(f"Querying production history for MFGORDERNAME={mfg_order} (LOT: {container_id})")
history_result = get_lot_split_merge_history(
work_order=mfg_order,
current_container_id=container_id
)
history_result = get_lot_split_merge_history(
work_order=mfg_order,
current_container_id=container_id,
full_history=full_history,
)
logger.info(f"[DEBUG] history_result keys: {list(history_result.keys())}")
logger.info(f"[DEBUG] history_result total: {history_result.get('total', 0)}")
@@ -1005,15 +941,17 @@ def get_equipment_status_hours(
if validation_error:
return {'error': validation_error}
try:
# Build filter on HISTORYID (which maps to RESOURCEID)
equipment_filter = _build_in_filter(equipment_ids, 'r.RESOURCEID')
sql = SQLLoader.load("query_tool/equipment_status_hours")
sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter)
params = {'start_date': start_date, 'end_date': end_date}
df = read_sql_df(sql, params)
try:
builder = QueryBuilder()
builder.add_in_condition("r.RESOURCEID", equipment_ids)
sql = SQLLoader.load_with_params(
"query_tool/equipment_status_hours",
EQUIPMENT_FILTER=builder.get_conditions_sql(),
)
params = {'start_date': start_date, 'end_date': end_date}
params.update(builder.params)
df = read_sql_df(sql, params)
data = _df_to_records(df)
# Calculate totals
@@ -1075,14 +1013,17 @@ def get_equipment_lots(
if validation_error:
return {'error': validation_error}
try:
equipment_filter = _build_in_filter(equipment_ids, 'h.EQUIPMENTID')
sql = SQLLoader.load("query_tool/equipment_lots")
sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter)
params = {'start_date': start_date, 'end_date': end_date}
df = read_sql_df(sql, params)
try:
builder = QueryBuilder()
builder.add_in_condition("h.EQUIPMENTID", equipment_ids)
sql = SQLLoader.load_with_params(
"query_tool/equipment_lots",
EQUIPMENT_FILTER=builder.get_conditions_sql(),
)
params = {'start_date': start_date, 'end_date': end_date}
params.update(builder.params)
df = read_sql_df(sql, params)
data = _df_to_records(df)
logger.info(f"Equipment lots: {len(data)} records")
@@ -1122,14 +1063,17 @@ def get_equipment_materials(
if validation_error:
return {'error': validation_error}
try:
equipment_filter = _build_in_filter(equipment_names, 'EQUIPMENTNAME')
sql = SQLLoader.load("query_tool/equipment_materials")
sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter)
params = {'start_date': start_date, 'end_date': end_date}
df = read_sql_df(sql, params)
try:
builder = QueryBuilder()
builder.add_in_condition("EQUIPMENTNAME", equipment_names)
sql = SQLLoader.load_with_params(
"query_tool/equipment_materials",
EQUIPMENT_FILTER=builder.get_conditions_sql(),
)
params = {'start_date': start_date, 'end_date': end_date}
params.update(builder.params)
df = read_sql_df(sql, params)
data = _df_to_records(df)
logger.info(f"Equipment materials: {len(data)} records")
@@ -1169,14 +1113,17 @@ def get_equipment_rejects(
if validation_error:
return {'error': validation_error}
try:
equipment_filter = _build_in_filter(equipment_names, 'EQUIPMENTNAME')
sql = SQLLoader.load("query_tool/equipment_rejects")
sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter)
params = {'start_date': start_date, 'end_date': end_date}
df = read_sql_df(sql, params)
try:
builder = QueryBuilder()
builder.add_in_condition("EQUIPMENTNAME", equipment_names)
sql = SQLLoader.load_with_params(
"query_tool/equipment_rejects",
EQUIPMENT_FILTER=builder.get_conditions_sql(),
)
params = {'start_date': start_date, 'end_date': end_date}
params.update(builder.params)
df = read_sql_df(sql, params)
data = _df_to_records(df)
logger.info(f"Equipment rejects: {len(data)} records")
@@ -1218,14 +1165,17 @@ def get_equipment_jobs(
if validation_error:
return {'error': validation_error}
try:
equipment_filter = _build_in_filter(equipment_ids, 'RESOURCEID')
sql = SQLLoader.load("query_tool/equipment_jobs")
sql = sql.replace("{{ EQUIPMENT_FILTER }}", equipment_filter)
params = {'start_date': start_date, 'end_date': end_date}
df = read_sql_df(sql, params)
try:
builder = QueryBuilder()
builder.add_in_condition("RESOURCEID", equipment_ids)
sql = SQLLoader.load_with_params(
"query_tool/equipment_jobs",
EQUIPMENT_FILTER=builder.get_conditions_sql(),
)
params = {'start_date': start_date, 'end_date': end_date}
params.update(builder.params)
df = read_sql_df(sql, params)
data = _df_to_records(df)
logger.info(f"Equipment jobs: {len(data)} records")

View File

@@ -0,0 +1,13 @@
-- Unified LineageEngine - Merge Sources
-- Find source lots merged into finished lots from DW_MES_PJ_COMBINEDASSYLOTS.
--
-- Parameters:
-- FINISHED_NAME_FILTER - QueryBuilder-generated condition on ca.FINISHEDNAME
--
SELECT
ca.CONTAINERID AS SOURCE_CID,
ca.CONTAINERNAME AS SOURCE_NAME,
ca.FINISHEDNAME,
ca.LOTID AS FINISHED_CID
FROM DWH.DW_MES_PJ_COMBINEDASSYLOTS ca
WHERE {{ FINISHED_NAME_FILTER }}

View File

@@ -0,0 +1,23 @@
-- Unified LineageEngine - Split Ancestors
-- Resolve split genealogy upward via DW_MES_CONTAINER.SPLITFROMID
--
-- Parameters:
-- CID_FILTER - QueryBuilder-generated condition for START WITH
--
-- Notes:
-- - CONNECT BY NOCYCLE prevents infinite loops on cyclic data.
-- - LEVEL <= 20 matches previous BFS guard.
--
-- Recursive WITH fallback (Oracle recursive subquery factoring):
-- If CONNECT BY execution plan regresses, replace this file's content with
-- sql/lineage/split_ancestors_recursive.sql (kept as reference).
--
SELECT
c.CONTAINERID,
c.SPLITFROMID,
c.CONTAINERNAME,
LEVEL AS SPLIT_DEPTH
FROM DWH.DW_MES_CONTAINER c
START WITH {{ CID_FILTER }}
CONNECT BY NOCYCLE PRIOR c.SPLITFROMID = c.CONTAINERID
AND LEVEL <= 20

View File

@@ -1,5 +1,6 @@
-- Mid-Section Defect Traceability - LOT Genealogy Records (Query 2)
-- Batch query for split/merge records related to work orders
-- DEPRECATED: replaced by sql/lineage/split_ancestors.sql
-- Mid-Section Defect Traceability - LOT Genealogy Records (Query 2)
-- Batch query for split/merge records related to work orders
--
-- Parameters:
-- MFG_ORDER_FILTER - Dynamic IN clause for MFGORDERNAME (built by QueryBuilder)

View File

@@ -1,5 +1,6 @@
-- Mid-Section Defect Traceability - Split Chain (Query 2a)
-- Resolve split ancestors via DW_MES_CONTAINER.SPLITFROMID
-- DEPRECATED: replaced by sql/lineage/split_ancestors.sql
-- Mid-Section Defect Traceability - Split Chain (Query 2a)
-- Resolve split ancestors via DW_MES_CONTAINER.SPLITFROMID
--
-- Parameters:
-- Dynamically built IN clause for CONTAINERIDs

View File

@@ -1,11 +1,11 @@
-- LOT ID to CONTAINERID Resolution
-- Converts user-input LOT ID (CONTAINERNAME) to internal CONTAINERID
--
-- Parameters:
-- :container_names - List of CONTAINERNAME values (bind variable list)
--
-- Note: CONTAINERID is 16-char hex (e.g., '48810380001cba48')
-- CONTAINERNAME is user-visible LOT ID (e.g., 'GA23100020-A00-011')
-- LOT ID to CONTAINERID Resolution
-- Converts user-input LOT ID (CONTAINERNAME) to internal CONTAINERID
--
-- Parameters:
-- CONTAINER_FILTER - QueryBuilder filter on CONTAINERNAME
--
-- Note: CONTAINERID is 16-char hex (e.g., '48810380001cba48')
-- CONTAINERNAME is user-visible LOT ID (e.g., 'GA23100020-A00-011')
SELECT
CONTAINERID,
@@ -13,5 +13,5 @@ SELECT
MFGORDERNAME,
SPECNAME,
QTY
FROM DWH.DW_MES_CONTAINER
WHERE CONTAINERNAME IN ({{ CONTAINER_NAMES }})
FROM DWH.DW_MES_CONTAINER
WHERE {{ CONTAINER_FILTER }}

View File

@@ -0,0 +1,14 @@
-- Serial Number (流水號) to CONTAINERID Resolution
-- Converts finished product serial numbers to CONTAINERID list.
--
-- Parameters:
-- SERIAL_FILTER - QueryBuilder filter on p.FINISHEDNAME
--
SELECT DISTINCT
p.CONTAINERID,
p.FINISHEDNAME,
c.CONTAINERNAME,
c.SPECNAME
FROM DWH.DW_MES_PJ_COMBINEDASSYLOTS p
LEFT JOIN DWH.DW_MES_CONTAINER c ON p.CONTAINERID = c.CONTAINERID
WHERE {{ SERIAL_FILTER }}

View File

@@ -0,0 +1,14 @@
-- GA Work Order to CONTAINERID Resolution
-- Expands work orders to associated CONTAINERIDs.
--
-- Parameters:
-- WORK_ORDER_FILTER - QueryBuilder filter on h.PJ_WORKORDER
--
SELECT DISTINCT
h.CONTAINERID,
h.PJ_WORKORDER,
c.CONTAINERNAME,
c.SPECNAME
FROM DWH.DW_MES_LOTWIPHISTORY h
LEFT JOIN DWH.DW_MES_CONTAINER c ON h.CONTAINERID = c.CONTAINERID
WHERE {{ WORK_ORDER_FILTER }}

View File

@@ -1,13 +1,18 @@
-- LOT Split/Merge History Query (拆併批歷史紀錄)
-- Query by CONTAINERID list from same work order
-- Check both TARGET (CONTAINERID) and SOURCE (FROMCONTAINERID) to find all related records
WITH work_order_lots AS (
SELECT CONTAINERID
FROM DWH.DW_MES_CONTAINER
WHERE MFGORDERNAME = :work_order
)
SELECT
-- LOT Split/Merge History Query (拆併批歷史紀錄)
-- Query by CONTAINERID list from same work order
-- Check both TARGET (CONTAINERID) and SOURCE (FROMCONTAINERID) to find all related records
--
-- Parameters:
-- WORK_ORDER_FILTER - QueryBuilder filter on MFGORDERNAME
-- TIME_WINDOW - Optional time-window filter (default fast mode: 6 months)
-- ROW_LIMIT - Optional row limit (default fast mode: 500)
WITH work_order_lots AS (
SELECT CONTAINERID
FROM DWH.DW_MES_CONTAINER
WHERE {{ WORK_ORDER_FILTER }}
)
SELECT
h.HISTORYMAINLINEID,
h.CDONAME AS OPERATION_TYPE,
h.CONTAINERID AS TARGET_CONTAINERID,
@@ -17,10 +22,11 @@ SELECT
h.QTY AS TARGET_QTY,
h.TXNDATE
FROM DWH.DW_MES_HM_LOTMOVEOUT h
WHERE (
h.CONTAINERID IN (SELECT CONTAINERID FROM work_order_lots)
OR h.FROMCONTAINERID IN (SELECT CONTAINERID FROM work_order_lots)
)
AND h.FROMCONTAINERID IS NOT NULL
ORDER BY h.TXNDATE
FETCH FIRST 100 ROWS ONLY
WHERE (
h.CONTAINERID IN (SELECT CONTAINERID FROM work_order_lots)
OR h.FROMCONTAINERID IN (SELECT CONTAINERID FROM work_order_lots)
)
AND h.FROMCONTAINERID IS NOT NULL
{{ TIME_WINDOW }}
ORDER BY h.TXNDATE
{{ ROW_LIMIT }}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""Unit tests for EventFetcher."""
from __future__ import annotations
from unittest.mock import patch
import pandas as pd
from mes_dashboard.services.event_fetcher import EventFetcher
def test_cache_key_is_stable_for_sorted_ids():
key1 = EventFetcher._cache_key("history", ["CID-B", "CID-A", "CID-A"])
key2 = EventFetcher._cache_key("history", ["CID-A", "CID-B"])
assert key1 == key2
assert key1.startswith("evt:history:")
def test_get_rate_limit_config_supports_env_override(monkeypatch):
monkeypatch.setenv("EVT_HISTORY_RATE_MAX_REQUESTS", "33")
monkeypatch.setenv("EVT_HISTORY_RATE_WINDOW_SECONDS", "77")
config = EventFetcher._get_rate_limit_config("history")
assert config["bucket"] == "event-history"
assert config["max_attempts"] == 33
assert config["window_seconds"] == 77
@patch("mes_dashboard.services.event_fetcher.read_sql_df")
@patch("mes_dashboard.services.event_fetcher.cache_get")
def test_fetch_events_cache_hit_skips_db(mock_cache_get, mock_read_sql_df):
mock_cache_get.return_value = {"CID-1": [{"CONTAINERID": "CID-1"}]}
result = EventFetcher.fetch_events(["CID-1"], "materials")
assert result["CID-1"][0]["CONTAINERID"] == "CID-1"
mock_read_sql_df.assert_not_called()
@patch("mes_dashboard.services.event_fetcher.cache_set")
@patch("mes_dashboard.services.event_fetcher.cache_get", return_value=None)
@patch("mes_dashboard.services.event_fetcher.read_sql_df")
@patch("mes_dashboard.services.event_fetcher.SQLLoader.load_with_params")
def test_fetch_events_upstream_history_branch(
mock_sql_load,
mock_read_sql_df,
_mock_cache_get,
mock_cache_set,
):
mock_sql_load.return_value = "SELECT * FROM UPSTREAM"
mock_read_sql_df.return_value = pd.DataFrame(
[
{"CONTAINERID": "CID-1", "WORKCENTER_GROUP": "DB"},
{"CONTAINERID": "CID-2", "WORKCENTER_GROUP": "WB"},
]
)
result = EventFetcher.fetch_events(["CID-1", "CID-2"], "upstream_history")
assert sorted(result.keys()) == ["CID-1", "CID-2"]
assert mock_sql_load.call_args.args[0] == "mid_section_defect/upstream_history"
_, params = mock_read_sql_df.call_args.args
assert len(params) == 2
mock_cache_set.assert_called_once()
assert mock_cache_set.call_args.args[0].startswith("evt:upstream_history:")
@patch("mes_dashboard.services.event_fetcher.cache_set")
@patch("mes_dashboard.services.event_fetcher.cache_get", return_value=None)
@patch("mes_dashboard.services.event_fetcher.read_sql_df")
@patch("mes_dashboard.services.event_fetcher.SQLLoader.load")
def test_fetch_events_history_branch_replaces_container_filter(
mock_sql_load,
mock_read_sql_df,
_mock_cache_get,
_mock_cache_set,
):
mock_sql_load.return_value = (
"SELECT * FROM t WHERE h.CONTAINERID = :container_id {{ WORKCENTER_FILTER }}"
)
mock_read_sql_df.return_value = pd.DataFrame([])
EventFetcher.fetch_events(["CID-1"], "history")
sql, params = mock_read_sql_df.call_args.args
assert "h.CONTAINERID = :container_id" not in sql
assert "{{ WORKCENTER_FILTER }}" not in sql
assert params == {"p0": "CID-1"}

View File

@@ -0,0 +1,231 @@
# -*- coding: utf-8 -*-
"""Unit tests for LineageEngine."""
from __future__ import annotations
from unittest.mock import patch
import pandas as pd
from mes_dashboard.services.lineage_engine import LineageEngine
@patch("mes_dashboard.services.lineage_engine.read_sql_df")
def test_resolve_split_ancestors_batches_and_enforces_max_depth(mock_read_sql_df):
cids = [f"C{i:04d}" for i in range(1001)]
mock_read_sql_df.side_effect = [
pd.DataFrame(
[
{
"CONTAINERID": "C0000",
"SPLITFROMID": "P0000",
"CONTAINERNAME": "LOT-0000",
"SPLIT_DEPTH": 1,
},
{
"CONTAINERID": "P0000",
"SPLITFROMID": None,
"CONTAINERNAME": "LOT-P0000",
"SPLIT_DEPTH": 2,
},
]
),
pd.DataFrame(
[
{
"CONTAINERID": "C1000",
"SPLITFROMID": "P1000",
"CONTAINERNAME": "LOT-1000",
"SPLIT_DEPTH": 1,
},
{
"CONTAINERID": "C-TOO-DEEP",
"SPLITFROMID": "P-TOO-DEEP",
"CONTAINERNAME": "LOT-DEEP",
"SPLIT_DEPTH": 21,
},
]
),
]
result = LineageEngine.resolve_split_ancestors(cids, {"INIT": "LOT-INIT"})
assert mock_read_sql_df.call_count == 2
first_sql, first_params = mock_read_sql_df.call_args_list[0].args
second_sql, second_params = mock_read_sql_df.call_args_list[1].args
assert "LEVEL <= 20" in first_sql
assert "LEVEL <= 20" in second_sql
assert len(first_params) == 1000
assert len(second_params) == 1
assert result["child_to_parent"]["C0000"] == "P0000"
assert result["child_to_parent"]["C1000"] == "P1000"
assert "C-TOO-DEEP" not in result["child_to_parent"]
assert result["cid_to_name"]["C0000"] == "LOT-0000"
assert result["cid_to_name"]["INIT"] == "LOT-INIT"
@patch("mes_dashboard.services.lineage_engine.read_sql_df")
def test_resolve_merge_sources_batches_and_returns_mapping(mock_read_sql_df):
names = [f"FN{i:04d}" for i in range(1001)]
mock_read_sql_df.side_effect = [
pd.DataFrame(
[
{"FINISHEDNAME": "FN0000", "SOURCE_CID": "SRC-A"},
{"FINISHEDNAME": "FN0000", "SOURCE_CID": "SRC-B"},
]
),
pd.DataFrame(
[
{"FINISHEDNAME": "FN1000", "SOURCE_CID": "SRC-C"},
{"FINISHEDNAME": "FN1000", "SOURCE_CID": "SRC-C"},
{"FINISHEDNAME": None, "SOURCE_CID": "SRC-INVALID"},
]
),
]
result = LineageEngine.resolve_merge_sources(names)
assert mock_read_sql_df.call_count == 2
first_sql, first_params = mock_read_sql_df.call_args_list[0].args
second_sql, second_params = mock_read_sql_df.call_args_list[1].args
assert "{{ FINISHED_NAME_FILTER }}" not in first_sql
assert "{{ FINISHED_NAME_FILTER }}" not in second_sql
assert len(first_params) == 1000
assert len(second_params) == 1
assert result["FN0000"] == ["SRC-A", "SRC-B"]
assert result["FN1000"] == ["SRC-C"]
@patch("mes_dashboard.services.lineage_engine.LineageEngine.resolve_merge_sources")
@patch("mes_dashboard.services.lineage_engine.LineageEngine.resolve_split_ancestors")
def test_resolve_full_genealogy_combines_split_and_merge(
mock_resolve_split_ancestors,
mock_resolve_merge_sources,
):
mock_resolve_split_ancestors.side_effect = [
{
"child_to_parent": {
"A": "B",
"B": "C",
},
"cid_to_name": {
"A": "LOT-A",
"B": "LOT-B",
"C": "LOT-C",
},
},
{
"child_to_parent": {
"M1": "M0",
},
"cid_to_name": {
"M1": "LOT-M1",
"M0": "LOT-M0",
},
},
]
mock_resolve_merge_sources.return_value = {"LOT-B": ["M1"]}
result = LineageEngine.resolve_full_genealogy(["A"], {"A": "LOT-A"})
assert result == {"A": {"B", "C", "M1", "M0"}}
assert mock_resolve_split_ancestors.call_count == 2
mock_resolve_merge_sources.assert_called_once()
@patch("mes_dashboard.services.lineage_engine.read_sql_df")
def test_split_ancestors_matches_legacy_bfs_for_five_known_lots(mock_read_sql_df):
parent_by_cid = {
"L1": "L1P1",
"L1P1": "L1P2",
"L2": "L2P1",
"L3": None,
"L4": "L4P1",
"L4P1": "L4P2",
"L4P2": "L4P3",
"L5": "L5P1",
"L5P1": "L5P2",
"L5P2": "L5P1",
}
name_by_cid = {
"L1": "LOT-1",
"L1P1": "LOT-1-P1",
"L1P2": "LOT-1-P2",
"L2": "LOT-2",
"L2P1": "LOT-2-P1",
"L3": "LOT-3",
"L4": "LOT-4",
"L4P1": "LOT-4-P1",
"L4P2": "LOT-4-P2",
"L4P3": "LOT-4-P3",
"L5": "LOT-5",
"L5P1": "LOT-5-P1",
"L5P2": "LOT-5-P2",
}
seed_lots = ["L1", "L2", "L3", "L4", "L5"]
def _connect_by_rows(start_cids):
rows = []
for seed in start_cids:
current = seed
depth = 1
visited = set()
while current and depth <= 20 and current not in visited:
visited.add(current)
rows.append(
{
"CONTAINERID": current,
"SPLITFROMID": parent_by_cid.get(current),
"CONTAINERNAME": name_by_cid.get(current),
"SPLIT_DEPTH": depth,
}
)
current = parent_by_cid.get(current)
depth += 1
return pd.DataFrame(rows)
def _mock_read_sql(_sql, params):
requested = [value for value in params.values()]
return _connect_by_rows(requested)
mock_read_sql_df.side_effect = _mock_read_sql
connect_by_result = LineageEngine.resolve_split_ancestors(seed_lots)
# Legacy BFS reference implementation from previous mid_section_defect_service.
legacy_child_to_parent = {}
legacy_cid_to_name = {}
frontier = list(seed_lots)
seen = set(seed_lots)
rounds = 0
while frontier:
rounds += 1
batch_rows = []
for cid in frontier:
batch_rows.append(
{
"CONTAINERID": cid,
"SPLITFROMID": parent_by_cid.get(cid),
"CONTAINERNAME": name_by_cid.get(cid),
}
)
new_parents = set()
for row in batch_rows:
cid = row["CONTAINERID"]
split_from = row["SPLITFROMID"]
name = row["CONTAINERNAME"]
if isinstance(name, str) and name:
legacy_cid_to_name[cid] = name
if isinstance(split_from, str) and split_from and split_from != cid:
legacy_child_to_parent[cid] = split_from
if split_from not in seen:
seen.add(split_from)
new_parents.add(split_from)
frontier = list(new_parents)
if rounds > 20:
break
assert connect_by_result["child_to_parent"] == legacy_child_to_parent
assert connect_by_result["cid_to_name"] == legacy_cid_to_name

View File

@@ -8,6 +8,7 @@ from unittest.mock import patch
import pandas as pd
from mes_dashboard.services.mid_section_defect_service import (
build_trace_aggregation_from_events,
query_analysis,
query_analysis_detail,
query_all_loss_reasons,
@@ -126,3 +127,116 @@ def test_query_all_loss_reasons_cache_miss_queries_and_caches_sorted_values(
{'loss_reasons': ['A_REASON', 'B_REASON']},
ttl=86400,
)
@patch('mes_dashboard.services.mid_section_defect_service.cache_set')
@patch('mes_dashboard.services.mid_section_defect_service.cache_get', return_value=None)
@patch('mes_dashboard.services.mid_section_defect_service.release_lock')
@patch('mes_dashboard.services.mid_section_defect_service.try_acquire_lock', return_value=True)
@patch('mes_dashboard.services.mid_section_defect_service._fetch_upstream_history')
@patch('mes_dashboard.services.mid_section_defect_service._resolve_full_genealogy')
@patch('mes_dashboard.services.mid_section_defect_service._fetch_tmtt_data')
def test_trace_aggregation_matches_query_analysis_summary(
mock_fetch_tmtt_data,
mock_resolve_genealogy,
mock_fetch_upstream_history,
_mock_lock,
_mock_release_lock,
_mock_cache_get,
_mock_cache_set,
):
tmtt_df = pd.DataFrame([
{
'CONTAINERID': 'CID-001',
'CONTAINERNAME': 'LOT-001',
'TRACKINQTY': 100,
'REJECTQTY': 5,
'LOSSREASONNAME': 'R1',
'WORKFLOW': 'WF-A',
'PRODUCTLINENAME': 'PKG-A',
'PJ_TYPE': 'TYPE-A',
'TMTT_EQUIPMENTNAME': 'TMTT-01',
'TRACKINTIMESTAMP': '2025-01-10 10:00:00',
'FINISHEDRUNCARD': 'FR-001',
},
{
'CONTAINERID': 'CID-002',
'CONTAINERNAME': 'LOT-002',
'TRACKINQTY': 120,
'REJECTQTY': 6,
'LOSSREASONNAME': 'R2',
'WORKFLOW': 'WF-B',
'PRODUCTLINENAME': 'PKG-B',
'PJ_TYPE': 'TYPE-B',
'TMTT_EQUIPMENTNAME': 'TMTT-02',
'TRACKINTIMESTAMP': '2025-01-11 10:00:00',
'FINISHEDRUNCARD': 'FR-002',
},
])
ancestors = {
'CID-001': {'CID-101'},
'CID-002': set(),
}
upstream_normalized = {
'CID-101': [{
'workcenter_group': '中段',
'equipment_id': 'EQ-01',
'equipment_name': 'EQ-01',
'spec_name': 'SPEC-A',
'track_in_time': '2025-01-09 08:00:00',
}],
'CID-002': [{
'workcenter_group': '中段',
'equipment_id': 'EQ-02',
'equipment_name': 'EQ-02',
'spec_name': 'SPEC-B',
'track_in_time': '2025-01-11 08:00:00',
}],
}
upstream_events = {
'CID-101': [{
'WORKCENTER_GROUP': '中段',
'EQUIPMENTID': 'EQ-01',
'EQUIPMENTNAME': 'EQ-01',
'SPECNAME': 'SPEC-A',
'TRACKINTIMESTAMP': '2025-01-09 08:00:00',
}],
'CID-002': [{
'WORKCENTER_GROUP': '中段',
'EQUIPMENTID': 'EQ-02',
'EQUIPMENTNAME': 'EQ-02',
'SPECNAME': 'SPEC-B',
'TRACKINTIMESTAMP': '2025-01-11 08:00:00',
}],
}
mock_fetch_tmtt_data.return_value = tmtt_df
mock_resolve_genealogy.return_value = ancestors
mock_fetch_upstream_history.return_value = upstream_normalized
summary = query_analysis('2025-01-01', '2025-01-31')
staged_summary = build_trace_aggregation_from_events(
'2025-01-01',
'2025-01-31',
seed_container_ids=['CID-001', 'CID-002'],
lineage_ancestors={
'CID-001': ['CID-101'],
'CID-002': [],
},
upstream_events_by_cid=upstream_events,
)
assert staged_summary['available_loss_reasons'] == summary['available_loss_reasons']
assert staged_summary['genealogy_status'] == summary['genealogy_status']
assert staged_summary['detail_total_count'] == len(summary['detail'])
assert staged_summary['kpi']['total_input'] == summary['kpi']['total_input']
assert staged_summary['kpi']['lot_count'] == summary['kpi']['lot_count']
assert staged_summary['kpi']['total_defect_qty'] == summary['kpi']['total_defect_qty']
assert abs(
staged_summary['kpi']['total_defect_rate'] - summary['kpi']['total_defect_rate']
) <= 0.01
assert staged_summary['daily_trend'] == summary['daily_trend']
assert staged_summary['charts'].keys() == summary['charts'].keys()

View File

@@ -7,25 +7,35 @@ Tests the API endpoints with mocked service dependencies:
- Error handling
"""
import pytest
import json
from unittest.mock import patch, MagicMock
from mes_dashboard import create_app
import pytest
import json
from unittest.mock import patch, MagicMock
from mes_dashboard import create_app
from mes_dashboard.core.cache import NoOpCache
from mes_dashboard.core.rate_limit import reset_rate_limits_for_tests
@pytest.fixture
def app():
"""Create test Flask application."""
app = create_app()
app.config['TESTING'] = True
return app
def app():
"""Create test Flask application."""
app = create_app()
app.config['TESTING'] = True
app.extensions["cache"] = NoOpCache()
return app
@pytest.fixture
def client(app):
"""Create test client."""
return app.test_client()
@pytest.fixture
def client(app):
"""Create test client."""
return app.test_client()
@pytest.fixture(autouse=True)
def _reset_rate_limits():
reset_rate_limits_for_tests()
yield
reset_rate_limits_for_tests()
class TestQueryToolPage:
@@ -129,8 +139,8 @@ class TestResolveEndpoint:
assert data['total'] == 1
assert data['data'][0]['lot_id'] == 'GA23100020-A00-001'
@patch('mes_dashboard.routes.query_tool_routes.resolve_lots')
def test_resolve_not_found(self, mock_resolve, client):
@patch('mes_dashboard.routes.query_tool_routes.resolve_lots')
def test_resolve_not_found(self, mock_resolve, client):
"""Should return not_found list for missing LOT IDs."""
mock_resolve.return_value = {
'data': [],
@@ -148,8 +158,56 @@ class TestResolveEndpoint:
)
assert response.status_code == 200
data = json.loads(response.data)
assert data['total'] == 0
assert 'INVALID-LOT-ID' in data['not_found']
assert data['total'] == 0
assert 'INVALID-LOT-ID' in data['not_found']
@patch('mes_dashboard.routes.query_tool_routes.resolve_lots')
@patch('mes_dashboard.routes.query_tool_routes.cache_get')
def test_resolve_cache_hit_skips_service(self, mock_cache_get, mock_resolve, client):
mock_cache_get.return_value = {
'data': [{'container_id': 'C1', 'input_value': 'LOT-1'}],
'total': 1,
'input_count': 1,
'not_found': [],
}
response = client.post(
'/api/query-tool/resolve',
json={'input_type': 'lot_id', 'values': ['LOT-1']},
)
assert response.status_code == 200
payload = response.get_json()
assert payload['total'] == 1
mock_resolve.assert_not_called()
@patch('mes_dashboard.routes.query_tool_routes.cache_set')
@patch('mes_dashboard.routes.query_tool_routes.cache_get', return_value=None)
@patch('mes_dashboard.routes.query_tool_routes.resolve_lots')
def test_resolve_success_caches_result(
self,
mock_resolve,
_mock_cache_get,
mock_cache_set,
client,
):
mock_resolve.return_value = {
'data': [{'container_id': 'C1', 'input_value': 'LOT-1'}],
'total': 1,
'input_count': 1,
'not_found': [],
}
response = client.post(
'/api/query-tool/resolve',
json={'input_type': 'lot_id', 'values': ['LOT-1']},
)
assert response.status_code == 200
mock_cache_set.assert_called_once()
cache_key = mock_cache_set.call_args.args[0]
assert cache_key.startswith('qt:resolve:lot_id:')
assert mock_cache_set.call_args.kwargs['ttl'] == 60
class TestLotHistoryEndpoint:
@@ -267,7 +325,7 @@ class TestAdjacentLotsEndpoint:
assert '2024-01-15' in call_args[0][1] # target_time
class TestLotAssociationsEndpoint:
class TestLotAssociationsEndpoint:
"""Tests for /api/query-tool/lot-associations endpoint."""
def test_missing_container_id(self, client):
@@ -294,8 +352,8 @@ class TestLotAssociationsEndpoint:
assert 'error' in data
assert '不支援' in data['error'] or 'type' in data['error'].lower()
@patch('mes_dashboard.routes.query_tool_routes.get_lot_materials')
def test_lot_materials_success(self, mock_query, client):
@patch('mes_dashboard.routes.query_tool_routes.get_lot_materials')
def test_lot_materials_success(self, mock_query, client):
"""Should return lot materials on success."""
mock_query.return_value = {
'data': [
@@ -313,8 +371,137 @@ class TestLotAssociationsEndpoint:
)
assert response.status_code == 200
data = json.loads(response.data)
assert 'data' in data
assert data['total'] == 1
assert 'data' in data
assert data['total'] == 1
@patch('mes_dashboard.routes.query_tool_routes.get_lot_splits')
def test_lot_splits_default_fast_mode(self, mock_query, client):
mock_query.return_value = {'data': [], 'total': 0}
response = client.get(
'/api/query-tool/lot-associations?container_id=488103800029578b&type=splits'
)
assert response.status_code == 200
mock_query.assert_called_once_with('488103800029578b', full_history=False)
@patch('mes_dashboard.routes.query_tool_routes.get_lot_splits')
def test_lot_splits_full_history_mode(self, mock_query, client):
mock_query.return_value = {'data': [], 'total': 0}
response = client.get(
'/api/query-tool/lot-associations?'
'container_id=488103800029578b&type=splits&full_history=true'
)
assert response.status_code == 200
mock_query.assert_called_once_with('488103800029578b', full_history=True)
class TestQueryToolRateLimit:
"""Rate-limit behavior for high-cost query-tool endpoints."""
@patch('mes_dashboard.routes.query_tool_routes.resolve_lots')
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 5))
def test_resolve_rate_limited_returns_429(self, _mock_limit, mock_resolve, client):
response = client.post(
'/api/query-tool/resolve',
json={'input_type': 'lot_id', 'values': ['GA23100020-A00-001']},
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '5'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
mock_resolve.assert_not_called()
@patch('mes_dashboard.routes.query_tool_routes.get_lot_history')
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 6))
def test_lot_history_rate_limited_returns_429(self, _mock_limit, mock_history, client):
response = client.get('/api/query-tool/lot-history?container_id=488103800029578b')
assert response.status_code == 429
assert response.headers.get('Retry-After') == '6'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
mock_history.assert_not_called()
@patch('mes_dashboard.routes.query_tool_routes.get_lot_materials')
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 7))
def test_lot_association_rate_limited_returns_429(
self,
_mock_limit,
mock_materials,
client,
):
response = client.get(
'/api/query-tool/lot-associations?container_id=488103800029578b&type=materials'
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '7'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
mock_materials.assert_not_called()
@patch('mes_dashboard.routes.query_tool_routes.get_adjacent_lots')
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 8))
def test_adjacent_lots_rate_limited_returns_429(
self,
_mock_limit,
mock_adjacent,
client,
):
response = client.get(
'/api/query-tool/adjacent-lots?equipment_id=EQ001&target_time=2024-01-15T10:30:00'
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '8'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
mock_adjacent.assert_not_called()
@patch('mes_dashboard.routes.query_tool_routes.get_equipment_status_hours')
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 9))
def test_equipment_period_rate_limited_returns_429(
self,
_mock_limit,
mock_equipment,
client,
):
response = client.post(
'/api/query-tool/equipment-period',
json={
'equipment_ids': ['EQ001'],
'start_date': '2024-01-01',
'end_date': '2024-01-31',
'query_type': 'status_hours',
},
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '9'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
mock_equipment.assert_not_called()
@patch('mes_dashboard.routes.query_tool_routes.get_lot_history')
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 10))
def test_export_rate_limited_returns_429(self, _mock_limit, mock_history, client):
response = client.post(
'/api/query-tool/export-csv',
json={
'export_type': 'lot_history',
'params': {'container_id': '488103800029578b'},
},
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '10'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
mock_history.assert_not_called()
class TestEquipmentPeriodEndpoint:

View File

@@ -8,15 +8,17 @@ Tests the core service functions without database dependencies:
"""
import pytest
from mes_dashboard.services.query_tool_service import (
validate_date_range,
validate_lot_input,
validate_equipment_input,
_build_in_clause,
_build_in_filter,
BATCH_SIZE,
MAX_LOT_IDS,
MAX_SERIAL_NUMBERS,
from mes_dashboard.services.query_tool_service import (
validate_date_range,
validate_lot_input,
validate_equipment_input,
_resolve_by_lot_id,
_resolve_by_serial_number,
_resolve_by_work_order,
get_lot_split_merge_history,
BATCH_SIZE,
MAX_LOT_IDS,
MAX_SERIAL_NUMBERS,
MAX_WORK_ORDERS,
MAX_EQUIPMENTS,
MAX_DATE_RANGE_DAYS,
@@ -184,86 +186,124 @@ class TestValidateEquipmentInput:
assert result is None
class TestBuildInClause:
"""Tests for _build_in_clause function."""
def test_empty_list(self):
"""Should return empty list for empty input."""
result = _build_in_clause([])
assert result == []
def test_single_value(self):
"""Should return single chunk for single value."""
result = _build_in_clause(['VAL001'])
assert len(result) == 1
assert result[0] == "'VAL001'"
def test_multiple_values(self):
"""Should join multiple values with comma."""
result = _build_in_clause(['VAL001', 'VAL002', 'VAL003'])
assert len(result) == 1
assert "'VAL001'" in result[0]
assert "'VAL002'" in result[0]
assert "'VAL003'" in result[0]
assert result[0] == "'VAL001', 'VAL002', 'VAL003'"
def test_chunking(self):
"""Should chunk when exceeding batch size."""
# Create more than BATCH_SIZE values
values = [f'VAL{i:06d}' for i in range(BATCH_SIZE + 10)]
result = _build_in_clause(values)
assert len(result) == 2
# First chunk should have BATCH_SIZE items
assert result[0].count("'") == BATCH_SIZE * 2 # 2 quotes per value
def test_escape_single_quotes(self):
"""Should escape single quotes in values."""
result = _build_in_clause(["VAL'001"])
assert len(result) == 1
assert "VAL''001" in result[0] # Escaped
def test_custom_chunk_size(self):
"""Should respect custom chunk size."""
values = ['V1', 'V2', 'V3', 'V4', 'V5']
result = _build_in_clause(values, max_chunk_size=2)
assert len(result) == 3 # 2+2+1
class TestBuildInFilter:
"""Tests for _build_in_filter function."""
def test_empty_list(self):
"""Should return 1=0 for empty input (no results)."""
result = _build_in_filter([], 'COL')
assert result == "1=0"
def test_single_value(self):
"""Should build simple IN clause for single value."""
result = _build_in_filter(['VAL001'], 'COL')
assert "COL IN" in result
assert "'VAL001'" in result
def test_multiple_values(self):
"""Should build IN clause with multiple values."""
result = _build_in_filter(['VAL001', 'VAL002'], 'COL')
assert "COL IN" in result
assert "'VAL001'" in result
assert "'VAL002'" in result
def test_custom_column(self):
"""Should use custom column name."""
result = _build_in_filter(['VAL001'], 't.MYCOL')
assert "t.MYCOL IN" in result
def test_large_list_uses_or(self):
"""Should use OR for chunked results."""
# Create more than BATCH_SIZE values
values = [f'VAL{i:06d}' for i in range(BATCH_SIZE + 10)]
result = _build_in_filter(values, 'COL')
assert " OR " in result
# Should have parentheses wrapping the OR conditions
assert result.startswith("(")
assert result.endswith(")")
class TestResolveQueriesUseBindParams:
"""Queries with user input should always use bind params."""
def test_resolve_by_lot_id_uses_query_builder_params(self):
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load:
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read:
mock_load.return_value = "SELECT * FROM DUAL"
mock_read.return_value = pd.DataFrame([
{
'CONTAINERID': 'CID-1',
'CONTAINERNAME': 'LOT-1',
'SPECNAME': 'SPEC-1',
'QTY': 100,
}
])
result = _resolve_by_lot_id(['LOT-1'])
assert result['total'] == 1
mock_load.assert_called_once()
sql_params = mock_load.call_args.kwargs
assert 'CONTAINER_FILTER' in sql_params
assert ':p0' in sql_params['CONTAINER_FILTER']
_, query_params = mock_read.call_args.args
assert query_params == {'p0': 'LOT-1'}
def test_resolve_by_serial_number_uses_query_builder_params(self):
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load:
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read:
mock_load.return_value = "SELECT * FROM DUAL"
mock_read.return_value = pd.DataFrame([
{
'CONTAINERID': 'CID-1',
'FINISHEDNAME': 'SN-1',
'CONTAINERNAME': 'LOT-1',
'SPECNAME': 'SPEC-1',
}
])
result = _resolve_by_serial_number(['SN-1'])
assert result['total'] == 1
sql_params = mock_load.call_args.kwargs
assert ':p0' in sql_params['SERIAL_FILTER']
_, query_params = mock_read.call_args.args
assert query_params == {'p0': 'SN-1'}
def test_resolve_by_work_order_uses_query_builder_params(self):
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load:
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read:
mock_load.return_value = "SELECT * FROM DUAL"
mock_read.return_value = pd.DataFrame([
{
'CONTAINERID': 'CID-1',
'PJ_WORKORDER': 'WO-1',
'CONTAINERNAME': 'LOT-1',
'SPECNAME': 'SPEC-1',
}
])
result = _resolve_by_work_order(['WO-1'])
assert result['total'] == 1
sql_params = mock_load.call_args.kwargs
assert ':p0' in sql_params['WORK_ORDER_FILTER']
_, query_params = mock_read.call_args.args
assert query_params == {'p0': 'WO-1'}
class TestSplitMergeHistoryMode:
"""Fast mode should use read_sql_df, full mode should use read_sql_df_slow."""
def test_fast_mode_uses_time_window_and_row_limit(self):
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load:
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_fast:
with patch('mes_dashboard.services.query_tool_service.read_sql_df_slow') as mock_slow:
mock_load.return_value = "SELECT * FROM DUAL"
mock_fast.return_value = pd.DataFrame([])
result = get_lot_split_merge_history('WO-1', full_history=False)
assert result['mode'] == 'fast'
kwargs = mock_load.call_args.kwargs
assert "ADD_MONTHS(SYSDATE, -6)" in kwargs['TIME_WINDOW']
assert "FETCH FIRST 500 ROWS ONLY" == kwargs['ROW_LIMIT']
mock_fast.assert_called_once()
mock_slow.assert_not_called()
def test_full_mode_uses_slow_query_without_limits(self):
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.SQLLoader.load_with_params') as mock_load:
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_fast:
with patch('mes_dashboard.services.query_tool_service.read_sql_df_slow') as mock_slow:
mock_load.return_value = "SELECT * FROM DUAL"
mock_slow.return_value = pd.DataFrame([])
result = get_lot_split_merge_history('WO-1', full_history=True)
assert result['mode'] == 'full'
kwargs = mock_load.call_args.kwargs
assert kwargs['TIME_WINDOW'] == ''
assert kwargs['ROW_LIMIT'] == ''
mock_fast.assert_not_called()
mock_slow.assert_called_once()
class TestServiceConstants:
@@ -323,98 +363,78 @@ class TestGetWorkcenterForGroups:
assert result == []
class TestGetLotHistoryWithWorkcenterFilter:
"""Tests for get_lot_history with workcenter_groups filter."""
def test_no_filter_returns_all(self):
"""When no workcenter_groups, should not add filter to SQL."""
from unittest.mock import patch, MagicMock
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read:
with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader:
from mes_dashboard.services.query_tool_service import get_lot_history
mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}'
mock_read.return_value = pd.DataFrame({
'CONTAINERID': ['abc123'],
'WORKCENTERNAME': ['DB_1'],
})
result = get_lot_history('abc123', workcenter_groups=None)
assert 'error' not in result
assert result['filtered_by_groups'] == []
# Verify SQL does not contain WORKCENTERNAME IN
sql_called = mock_read.call_args[0][0]
assert 'WORKCENTERNAME IN' not in sql_called
assert '{{ WORKCENTER_FILTER }}' not in sql_called
def test_with_filter_adds_condition(self):
"""When workcenter_groups provided, should filter by workcenters."""
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read:
with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader:
with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc:
from mes_dashboard.services.query_tool_service import get_lot_history
mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}'
mock_get_wc.return_value = ['DB_1', 'DB_2']
mock_read.return_value = pd.DataFrame({
'CONTAINERID': ['abc123'],
'WORKCENTERNAME': ['DB_1'],
})
result = get_lot_history('abc123', workcenter_groups=['DB'])
mock_get_wc.assert_called_once_with(['DB'])
assert result['filtered_by_groups'] == ['DB']
# Verify SQL contains filter
sql_called = mock_read.call_args[0][0]
assert 'WORKCENTERNAME' in sql_called
def test_empty_groups_list_no_filter(self):
"""Empty groups list should return all (no filter)."""
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read:
with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader:
from mes_dashboard.services.query_tool_service import get_lot_history
mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}'
mock_read.return_value = pd.DataFrame({
'CONTAINERID': ['abc123'],
'WORKCENTERNAME': ['DB_1'],
})
result = get_lot_history('abc123', workcenter_groups=[])
assert result['filtered_by_groups'] == []
# Verify SQL does not contain WORKCENTERNAME IN
sql_called = mock_read.call_args[0][0]
assert 'WORKCENTERNAME IN' not in sql_called
def test_filter_with_empty_workcenters_result(self):
"""When group has no workcenters, should not add filter."""
from unittest.mock import patch
import pandas as pd
with patch('mes_dashboard.services.query_tool_service.read_sql_df') as mock_read:
with patch('mes_dashboard.services.query_tool_service.SQLLoader') as mock_loader:
with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc:
from mes_dashboard.services.query_tool_service import get_lot_history
mock_loader.load.return_value = 'SELECT * FROM t WHERE c = :container_id {{ WORKCENTER_FILTER }}'
mock_get_wc.return_value = [] # No workcenters for this group
mock_read.return_value = pd.DataFrame({
'CONTAINERID': ['abc123'],
'WORKCENTERNAME': ['DB_1'],
})
result = get_lot_history('abc123', workcenter_groups=['UNKNOWN'])
# Should still succeed, just no filter applied
assert 'error' not in result
class TestGetLotHistoryWithWorkcenterFilter:
"""Tests for get_lot_history with workcenter_groups filter."""
def test_no_filter_returns_all(self):
"""When no workcenter_groups, should not add filter to SQL."""
from unittest.mock import patch
from mes_dashboard.services.query_tool_service import get_lot_history
with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch:
mock_fetch.return_value = {
'abc123': [
{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'},
{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'WB_1'},
]
}
result = get_lot_history('abc123', workcenter_groups=None)
assert 'error' not in result
assert result['filtered_by_groups'] == []
assert result['total'] == 2
def test_with_filter_adds_condition(self):
"""When workcenter_groups provided, should filter by workcenters."""
from unittest.mock import patch
from mes_dashboard.services.query_tool_service import get_lot_history
with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch:
with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc:
mock_fetch.return_value = {
'abc123': [
{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'},
{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'WB_1'},
]
}
mock_get_wc.return_value = ['DB_1']
result = get_lot_history('abc123', workcenter_groups=['DB'])
mock_get_wc.assert_called_once_with(['DB'])
assert result['filtered_by_groups'] == ['DB']
assert result['total'] == 1
assert result['data'][0]['WORKCENTERNAME'] == 'DB_1'
def test_empty_groups_list_no_filter(self):
"""Empty groups list should return all (no filter)."""
from unittest.mock import patch
from mes_dashboard.services.query_tool_service import get_lot_history
with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch:
mock_fetch.return_value = {
'abc123': [{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'}]
}
result = get_lot_history('abc123', workcenter_groups=[])
assert result['filtered_by_groups'] == []
assert result['total'] == 1
def test_filter_with_empty_workcenters_result(self):
"""When group has no workcenters, should not add filter."""
from unittest.mock import patch
from mes_dashboard.services.query_tool_service import get_lot_history
with patch('mes_dashboard.services.query_tool_service.EventFetcher.fetch_events') as mock_fetch:
with patch('mes_dashboard.services.filter_cache.get_workcenters_for_groups') as mock_get_wc:
mock_fetch.return_value = {
'abc123': [{'CONTAINERID': 'abc123', 'WORKCENTERNAME': 'DB_1'}]
}
mock_get_wc.return_value = []
result = get_lot_history('abc123', workcenter_groups=['UNKNOWN'])
assert 'error' not in result
assert result['total'] == 1

245
tests/test_trace_routes.py Normal file
View File

@@ -0,0 +1,245 @@
# -*- coding: utf-8 -*-
"""Route tests for staged trace API endpoints."""
from __future__ import annotations
from unittest.mock import patch
import mes_dashboard.core.database as db
from mes_dashboard.app import create_app
from mes_dashboard.core.cache import NoOpCache
from mes_dashboard.core.rate_limit import reset_rate_limits_for_tests
def _client():
db._ENGINE = None
app = create_app('testing')
app.config['TESTING'] = True
app.extensions["cache"] = NoOpCache()
return app.test_client()
def setup_function():
reset_rate_limits_for_tests()
def teardown_function():
reset_rate_limits_for_tests()
@patch('mes_dashboard.routes.trace_routes.resolve_lots')
def test_seed_resolve_query_tool_success(mock_resolve_lots):
mock_resolve_lots.return_value = {
'data': [
{
'container_id': 'CID-001',
'lot_id': 'LOT-001',
}
]
}
client = _client()
response = client.post(
'/api/trace/seed-resolve',
json={
'profile': 'query_tool',
'params': {
'resolve_type': 'lot_id',
'values': ['LOT-001'],
},
},
)
assert response.status_code == 200
payload = response.get_json()
assert payload['stage'] == 'seed-resolve'
assert payload['seed_count'] == 1
assert payload['seeds'][0]['container_id'] == 'CID-001'
assert payload['seeds'][0]['container_name'] == 'LOT-001'
assert payload['cache_key'].startswith('trace:seed:query_tool:')
def test_seed_resolve_invalid_profile_returns_400():
client = _client()
response = client.post(
'/api/trace/seed-resolve',
json={
'profile': 'invalid',
'params': {},
},
)
assert response.status_code == 400
payload = response.get_json()
assert payload['error']['code'] == 'INVALID_PROFILE'
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 8))
def test_seed_resolve_rate_limited_returns_429(_mock_rate_limit):
client = _client()
response = client.post(
'/api/trace/seed-resolve',
json={
'profile': 'query_tool',
'params': {'resolve_type': 'lot_id', 'values': ['LOT-001']},
},
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '8'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
@patch('mes_dashboard.routes.trace_routes.LineageEngine.resolve_full_genealogy')
def test_lineage_success_returns_snake_case(mock_resolve_genealogy):
mock_resolve_genealogy.return_value = {
'CID-001': {'CID-A', 'CID-B'}
}
client = _client()
response = client.post(
'/api/trace/lineage',
json={
'profile': 'query_tool',
'container_ids': ['CID-001'],
},
)
assert response.status_code == 200
payload = response.get_json()
assert payload['stage'] == 'lineage'
assert sorted(payload['ancestors']['CID-001']) == ['CID-A', 'CID-B']
assert payload['total_nodes'] == 3
assert 'totalNodes' not in payload
@patch(
'mes_dashboard.routes.trace_routes.LineageEngine.resolve_full_genealogy',
side_effect=TimeoutError('lineage timed out'),
)
def test_lineage_timeout_returns_504(_mock_resolve_genealogy):
client = _client()
response = client.post(
'/api/trace/lineage',
json={
'profile': 'query_tool',
'container_ids': ['CID-001'],
},
)
assert response.status_code == 504
payload = response.get_json()
assert payload['error']['code'] == 'LINEAGE_TIMEOUT'
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 6))
def test_lineage_rate_limited_returns_429(_mock_rate_limit):
client = _client()
response = client.post(
'/api/trace/lineage',
json={
'profile': 'query_tool',
'container_ids': ['CID-001'],
},
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '6'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'
@patch('mes_dashboard.routes.trace_routes.EventFetcher.fetch_events')
def test_events_partial_failure_returns_200_with_code(mock_fetch_events):
def _side_effect(_container_ids, domain):
if domain == 'history':
return {
'CID-001': [{'CONTAINERID': 'CID-001', 'EVENTTYPE': 'TRACK_IN'}]
}
raise RuntimeError('domain failed')
mock_fetch_events.side_effect = _side_effect
client = _client()
response = client.post(
'/api/trace/events',
json={
'profile': 'query_tool',
'container_ids': ['CID-001'],
'domains': ['history', 'materials'],
},
)
assert response.status_code == 200
payload = response.get_json()
assert payload['stage'] == 'events'
assert payload['code'] == 'EVENTS_PARTIAL_FAILURE'
assert 'materials' in payload['failed_domains']
assert payload['results']['history']['count'] == 1
@patch('mes_dashboard.routes.trace_routes.build_trace_aggregation_from_events')
@patch('mes_dashboard.routes.trace_routes.EventFetcher.fetch_events')
def test_events_mid_section_defect_with_aggregation(
mock_fetch_events,
mock_build_aggregation,
):
mock_fetch_events.return_value = {
'CID-001': [
{
'CONTAINERID': 'CID-001',
'WORKCENTER_GROUP': '測試',
'EQUIPMENTID': 'EQ-01',
'EQUIPMENTNAME': 'EQ-01',
}
]
}
mock_build_aggregation.return_value = {
'kpi': {'total_input': 100},
'charts': {'by_station': []},
'daily_trend': [],
'available_loss_reasons': [],
'genealogy_status': 'ready',
'detail_total_count': 0,
}
client = _client()
response = client.post(
'/api/trace/events',
json={
'profile': 'mid_section_defect',
'container_ids': ['CID-001'],
'domains': ['upstream_history'],
'params': {
'start_date': '2025-01-01',
'end_date': '2025-01-31',
},
'lineage': {'ancestors': {'CID-001': ['CID-A']}},
'seed_container_ids': ['CID-001'],
},
)
assert response.status_code == 200
payload = response.get_json()
assert payload['aggregation']['kpi']['total_input'] == 100
assert payload['aggregation']['genealogy_status'] == 'ready'
mock_build_aggregation.assert_called_once()
@patch('mes_dashboard.core.rate_limit.check_and_record', return_value=(True, 5))
def test_events_rate_limited_returns_429(_mock_rate_limit):
client = _client()
response = client.post(
'/api/trace/events',
json={
'profile': 'query_tool',
'container_ids': ['CID-001'],
'domains': ['history'],
},
)
assert response.status_code == 429
assert response.headers.get('Retry-After') == '5'
payload = response.get_json()
assert payload['error']['code'] == 'TOO_MANY_REQUESTS'