20160116

backend/app/routers/etl.py

@@ -81,10 +81,28 @@ def clean_value(val, default=''):
     if val is None or (isinstance(val, float) and pd.isna(val)):
         return default
     str_val = str(val).strip()
+    # Remove leading apostrophe often added by Excel (e.g. '001)
+    str_val = str_val.lstrip("'")
     if str_val.lower() in ('nan', 'none', 'null', ''):
         return default
     return str_val
 
+def normalize_date(val):
+    """將日期標準化為 YYYY-MM-DD 格式"""
+    val = clean_value(val, None)
+    if not val:
+        return None
+    
+    # 嘗試解析常見格式
+    from datetime import datetime
+    for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%d %H:%M:%S", "%Y/%m/%d %H:%M:%S", "%d-%b-%y"):
+        try:
+            # Handle Excel default string format often like 2025/9/30
+            dt = datetime.strptime(val.split(' ')[0], fmt.split(' ')[0])
+            return dt.strftime("%Y-%m-%d")
+        except ValueError:
+            continue
+    return val # Return original if parse failed
 
 @router.post("/import", response_model=ImportResponse)
 def import_data(request: ImportRequest, db: Session = Depends(get_db)):
@@ -150,13 +168,14 @@ def import_data(request: ImportRequest, db: Session = Depends(get_db)):
                     seen_ids.add(unique_key)
                     record = DitRecord(
                         op_id=op_id,
+                        op_name=clean_value(row.get('op_name')),
                         erp_account=erp_account,
                         customer=customer,
                         customer_normalized=normalize_customer_name(customer),
                         pn=sanitize_pn(pn),
                         eau=int(row.get('eau', 0)) if row.get('eau') and not pd.isna(row.get('eau')) else 0,
                         stage=clean_value(row.get('stage')),
-                        date=clean_value(row.get('date'))
+                        date=normalize_date(row.get('date'))
                     )
                 elif file_type == 'sample':
                     sample_id = clean_value(row.get('sample_id'), f'S{idx}')
@@ -177,7 +196,7 @@ def import_data(request: ImportRequest, db: Session = Depends(get_db)):
                         customer_normalized=normalize_customer_name(customer),
                         pn=sanitize_pn(pn),
                         qty=int(row.get('qty', 0)) if row.get('qty') and not pd.isna(row.get('qty')) else 0,
-                        date=clean_value(row.get('date'))
+                        date=normalize_date(row.get('date'))
                     )
                 elif file_type == 'order':
                     order_id = clean_value(row.get('order_id'), f'O{idx}')
@@ -195,9 +214,10 @@ def import_data(request: ImportRequest, db: Session = Depends(get_db)):
                         customer=customer,
                         customer_normalized=normalize_customer_name(customer),
                         pn=sanitize_pn(pn),
-                        qty=int(row.get('qty', 0)) if row.get('qty') and not pd.isna(row.get('qty')) else 0,
+                        qty=int(float(row.get('qty', 0)) * 1000) if row.get('qty') and not pd.isna(row.get('qty')) else 0,
                         status=clean_value(row.get('status'), 'Backlog'),
-                        amount=float(row.get('amount', 0)) if row.get('amount') and not pd.isna(row.get('amount')) else 0
+                        amount=float(row.get('amount', 0)) if row.get('amount') and not pd.isna(row.get('amount')) else 0,
+                        date=normalize_date(row.get('date'))
                     )
                 else:
                     continue
@@ -244,3 +264,21 @@ def get_data(data_type: str, db: Session = Depends(get_db)):
         }
         for record in records
     ]
+
+@router.delete("/data")
+def clear_all_data(db: Session = Depends(get_db)):
+    """清除所有匯入的資料與分析結果"""
+    try:
+        print("[ETL] Clearing all data...")
+        db.query(ReviewLog).delete()
+        db.query(MatchResult).delete()
+        db.query(DitRecord).delete()
+        db.query(SampleRecord).delete()
+        db.query(OrderRecord).delete()
+        db.commit()
+        print("[ETL] All data cleared successfully.")
+        return {"message": "All data cleared successfully"}
+    except Exception as e:
+        db.rollback()
+        print(f"[ETL] Error clearing data: {e}")
+        raise HTTPException(status_code=500, detail=str(e))