| | import os
|
| | import sys
|
| | import soundfile as sf
|
| | from tqdm import tqdm
|
| |
|
| | def is_significant_audio(file_path, silence_threshold=-40, silence_percent=90):
|
| | """
|
| | Check if an audio file contains significant non-silent parts.
|
| | """
|
| | try:
|
| | data, samplerate = sf.read(file_path)
|
| | if len(data) == 0:
|
| | return False
|
| |
|
| |
|
| | energy = (data ** 2).mean()
|
| | silence_ratio = (energy < silence_threshold).sum() / len(data) * 100
|
| | return silence_ratio < silence_percent
|
| | except Exception as e:
|
| | print(f"Error processing {file_path}: {e}")
|
| | return False
|
| |
|
| | def filter_manifest(manifest_path, output_path, dataset_dir):
|
| | """
|
| | Read the manifest file, check for silence, and write filtered files.
|
| | """
|
| | with open(manifest_path, 'r') as f:
|
| | lines = f.readlines()
|
| |
|
| | filtered_lines = [lines[0]]
|
| | for line in tqdm(lines[1:], desc=f"Processing {manifest_path}"):
|
| | file_path = os.path.join(dataset_dir, line.split("\t")[0])
|
| | if is_significant_audio(file_path):
|
| | filtered_lines.append(line)
|
| | else:
|
| | print(f"Skipping file due to silence: {file_path}")
|
| |
|
| | with open(output_path, 'w') as f_out:
|
| | f_out.writelines(filtered_lines)
|
| |
|
| | if __name__ == "__main__":
|
| | train_manifest = sys.argv[1]
|
| | valid_manifest = sys.argv[2]
|
| | output_dir = sys.argv[3]
|
| |
|
| | os.makedirs(output_dir, exist_ok=True)
|
| |
|
| | dataset_dir = "dataset"
|
| | filter_manifest(train_manifest, os.path.join(output_dir, "train.tsv"), dataset_dir)
|
| | filter_manifest(valid_manifest, os.path.join(output_dir, "valid.tsv"), dataset_dir)
|
| |
|
