feat: Add Dify audio transcription with VAD chunking and SSE progress
- Add audio file upload transcription via Dify STT API - Implement VAD-based audio segmentation in sidecar (3-min chunks) - Add SSE endpoint for real-time transcription progress updates - Fix chunk size enforcement for reliable uploads - Add retry logic with exponential backoff for API calls - Support Python 3.13+ with audioop-lts package - Update frontend with Chinese progress messages and chunk display - Improve start.sh health check with retry loop 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
egg
@@ -20,9 +20,10 @@ import tempfile
|
||||
import base64
|
||||
import uuid
|
||||
import re
|
||||
import wave
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
from typing import Optional, List, Tuple
|
||||
|
||||
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
|
||||
|
||||
@@ -105,8 +106,7 @@ class SileroVAD:
|
||||
def __init__(self, model_path: Optional[str] = None, threshold: float = 0.5):
|
||||
self.threshold = threshold
|
||||
self.session = None
|
||||
self._h = np.zeros((2, 1, 64), dtype=np.float32)
|
||||
self._c = np.zeros((2, 1, 64), dtype=np.float32)
|
||||
self._state = np.zeros((2, 1, 128), dtype=np.float32)
|
||||
self.sample_rate = 16000
|
||||
|
||||
if not ONNX_AVAILABLE:
|
||||
@@ -141,8 +141,7 @@ class SileroVAD:
|
||||
|
||||
def reset_states(self):
|
||||
"""Reset hidden states."""
|
||||
self._h = np.zeros((2, 1, 64), dtype=np.float32)
|
||||
self._c = np.zeros((2, 1, 64), dtype=np.float32)
|
||||
self._state = np.zeros((2, 1, 128), dtype=np.float32)
|
||||
|
||||
def __call__(self, audio: np.ndarray) -> float:
|
||||
"""Run VAD on audio chunk, return speech probability."""
|
||||
@@ -153,15 +152,14 @@ class SileroVAD:
|
||||
if audio.ndim == 1:
|
||||
audio = audio[np.newaxis, :]
|
||||
|
||||
# Run inference
|
||||
# Run inference with updated model format
|
||||
ort_inputs = {
|
||||
'input': audio.astype(np.float32),
|
||||
'sr': np.array([self.sample_rate], dtype=np.int64),
|
||||
'h': self._h,
|
||||
'c': self._c
|
||||
'state': self._state,
|
||||
'sr': np.array(self.sample_rate, dtype=np.int64)
|
||||
}
|
||||
|
||||
output, self._h, self._c = self.session.run(None, ort_inputs)
|
||||
output, self._state = self.session.run(None, ort_inputs)
|
||||
return float(output[0][0])
|
||||
|
||||
|
||||
@@ -406,6 +404,193 @@ class Transcriber:
|
||||
print(json.dumps({"error": f"Transcription error: {e}"}), file=sys.stderr)
|
||||
return ""
|
||||
|
||||
def segment_audio_file(
|
||||
self,
|
||||
audio_path: str,
|
||||
max_chunk_seconds: int = 300,
|
||||
min_silence_ms: int = 500,
|
||||
output_dir: Optional[str] = None
|
||||
) -> dict:
|
||||
"""
|
||||
Segment an audio file using VAD for natural speech boundaries.
|
||||
|
||||
Args:
|
||||
audio_path: Path to the audio file
|
||||
max_chunk_seconds: Maximum duration per chunk (default 5 minutes)
|
||||
min_silence_ms: Minimum silence duration to consider as boundary (default 500ms)
|
||||
output_dir: Directory to save chunks (default: temp directory)
|
||||
|
||||
Returns:
|
||||
dict with segments list and metadata
|
||||
"""
|
||||
try:
|
||||
# Import audio processing libraries
|
||||
try:
|
||||
from pydub import AudioSegment
|
||||
except ImportError:
|
||||
return {"error": "pydub not installed. Run: pip install pydub"}
|
||||
|
||||
if not os.path.exists(audio_path):
|
||||
return {"error": f"File not found: {audio_path}"}
|
||||
|
||||
# Create output directory
|
||||
if output_dir is None:
|
||||
output_dir = tempfile.mkdtemp(prefix="audio_segments_")
|
||||
else:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# Load audio file and convert to mono 16kHz
|
||||
print(json.dumps({"status": "loading_audio", "file": audio_path}), file=sys.stderr)
|
||||
audio = AudioSegment.from_file(audio_path)
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
total_duration_ms = len(audio)
|
||||
total_duration_sec = total_duration_ms / 1000
|
||||
|
||||
print(json.dumps({
|
||||
"status": "audio_loaded",
|
||||
"duration_seconds": total_duration_sec
|
||||
}), file=sys.stderr)
|
||||
|
||||
# Convert to numpy for VAD processing
|
||||
samples = np.array(audio.get_array_of_samples(), dtype=np.float32) / 32768.0
|
||||
|
||||
# Run VAD to detect speech regions
|
||||
segments = []
|
||||
current_start = 0
|
||||
max_chunk_samples = max_chunk_seconds * 16000
|
||||
min_silence_samples = int(min_silence_ms * 16 ) # 16 samples per ms at 16kHz
|
||||
|
||||
if self.vad_model is None or self.vad_model.session is None:
|
||||
# No VAD available, use fixed-time splitting
|
||||
print(json.dumps({"warning": "VAD not available, using fixed-time splitting"}), file=sys.stderr)
|
||||
chunk_idx = 0
|
||||
for start_sample in range(0, len(samples), max_chunk_samples):
|
||||
end_sample = min(start_sample + max_chunk_samples, len(samples))
|
||||
chunk_samples = samples[start_sample:end_sample]
|
||||
|
||||
# Export chunk
|
||||
chunk_path = os.path.join(output_dir, f"chunk_{chunk_idx:03d}.wav")
|
||||
self._export_wav(chunk_samples, chunk_path)
|
||||
|
||||
segments.append({
|
||||
"index": chunk_idx,
|
||||
"path": chunk_path,
|
||||
"start": start_sample / 16000,
|
||||
"end": end_sample / 16000,
|
||||
"duration": (end_sample - start_sample) / 16000
|
||||
})
|
||||
chunk_idx += 1
|
||||
else:
|
||||
# Use VAD for intelligent splitting
|
||||
print(json.dumps({"status": "running_vad"}), file=sys.stderr)
|
||||
self.vad_model.reset_states()
|
||||
|
||||
# Find silence regions for splitting
|
||||
window_size = 512
|
||||
silence_starts = []
|
||||
in_silence = False
|
||||
silence_start = 0
|
||||
|
||||
for i in range(0, len(samples) - window_size, window_size):
|
||||
window = samples[i:i + window_size]
|
||||
speech_prob = self.vad_model(window)
|
||||
|
||||
if speech_prob < 0.3: # Silence threshold
|
||||
if not in_silence:
|
||||
in_silence = True
|
||||
silence_start = i
|
||||
else:
|
||||
if in_silence:
|
||||
silence_duration = i - silence_start
|
||||
if silence_duration >= min_silence_samples:
|
||||
# Mark middle of silence as potential split point
|
||||
silence_starts.append(silence_start + silence_duration // 2)
|
||||
in_silence = False
|
||||
|
||||
# Add end of file as final split point
|
||||
silence_starts.append(len(samples))
|
||||
|
||||
# Create segments based on silence boundaries
|
||||
chunk_idx = 0
|
||||
current_start = 0
|
||||
|
||||
for split_point in silence_starts:
|
||||
# Check if we need to split here
|
||||
chunk_duration = split_point - current_start
|
||||
|
||||
if chunk_duration >= max_chunk_samples or split_point == len(samples):
|
||||
# Find the best split point before max duration
|
||||
if chunk_duration > max_chunk_samples:
|
||||
# Find nearest silence point before max
|
||||
best_split = current_start + max_chunk_samples
|
||||
for sp in silence_starts:
|
||||
if current_start < sp <= current_start + max_chunk_samples:
|
||||
best_split = sp
|
||||
split_point = best_split
|
||||
|
||||
# Export chunk
|
||||
chunk_samples = samples[current_start:split_point]
|
||||
if len(chunk_samples) > 8000: # At least 0.5 seconds
|
||||
chunk_path = os.path.join(output_dir, f"chunk_{chunk_idx:03d}.wav")
|
||||
self._export_wav(chunk_samples, chunk_path)
|
||||
|
||||
segments.append({
|
||||
"index": chunk_idx,
|
||||
"path": chunk_path,
|
||||
"start": current_start / 16000,
|
||||
"end": split_point / 16000,
|
||||
"duration": (split_point - current_start) / 16000
|
||||
})
|
||||
chunk_idx += 1
|
||||
|
||||
current_start = split_point
|
||||
|
||||
# Handle any remaining audio - split into max_chunk_samples pieces
|
||||
while current_start < len(samples):
|
||||
remaining_len = len(samples) - current_start
|
||||
if remaining_len < 8000: # Less than 0.5 seconds
|
||||
break
|
||||
|
||||
# Determine chunk end (respect max_chunk_samples)
|
||||
chunk_end = min(current_start + max_chunk_samples, len(samples))
|
||||
chunk_samples = samples[current_start:chunk_end]
|
||||
|
||||
chunk_path = os.path.join(output_dir, f"chunk_{chunk_idx:03d}.wav")
|
||||
self._export_wav(chunk_samples, chunk_path)
|
||||
segments.append({
|
||||
"index": chunk_idx,
|
||||
"path": chunk_path,
|
||||
"start": current_start / 16000,
|
||||
"end": chunk_end / 16000,
|
||||
"duration": len(chunk_samples) / 16000
|
||||
})
|
||||
chunk_idx += 1
|
||||
current_start = chunk_end
|
||||
|
||||
print(json.dumps({
|
||||
"status": "segmentation_complete",
|
||||
"total_segments": len(segments)
|
||||
}), file=sys.stderr)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"segments": segments,
|
||||
"total_segments": len(segments),
|
||||
"total_duration": total_duration_sec,
|
||||
"output_dir": output_dir
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {"error": f"Segmentation error: {str(e)}"}
|
||||
|
||||
def _export_wav(self, samples: np.ndarray, output_path: str):
|
||||
"""Export numpy samples to WAV file."""
|
||||
with wave.open(output_path, 'wb') as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes((samples * 32768).astype(np.int16).tobytes())
|
||||
|
||||
def handle_command(self, cmd: dict) -> Optional[dict]:
|
||||
"""Handle a JSON command."""
|
||||
action = cmd.get("action")
|
||||
@@ -447,6 +632,21 @@ class Transcriber:
|
||||
self.streaming_session = None
|
||||
return result
|
||||
|
||||
elif action == "segment_audio":
|
||||
# Segment audio file using VAD
|
||||
file_path = cmd.get("file_path")
|
||||
if not file_path:
|
||||
return {"error": "No file_path specified"}
|
||||
max_chunk_seconds = cmd.get("max_chunk_seconds", 300)
|
||||
min_silence_ms = cmd.get("min_silence_ms", 500)
|
||||
output_dir = cmd.get("output_dir")
|
||||
return self.segment_audio_file(
|
||||
file_path,
|
||||
max_chunk_seconds=max_chunk_seconds,
|
||||
min_silence_ms=min_silence_ms,
|
||||
output_dir=output_dir
|
||||
)
|
||||
|
||||
elif action == "ping":
|
||||
return {"status": "pong"}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user