Meta 重磅开源 SAM Audio：第一次，剪音频像剪视频一样简单了。

git clone https://github.com/facebookresearch/sam-audio.git
cd sam-audio

pip install .

from sam_audio import SAMAudio, SAMAudioProcessor
import torchaudio

model = SAMAudio.from_pretrained("facebook/sam-audio-large")
processor = SAMAudioProcessor.from_pretrained("facebook/sam-audio-large")
model = model.eval().cuda()

file = "<audio file>" # audio file path or torch tensor
description = "<description>"

batch = processor(
    audios=[file],
    descriptions=[description],
).to("cuda")

result = model.separate(batch)

# Save separated audio
sample_rate = processor.audio_sampling_rate
torchaudio.save("target.wav", result.target.cpu(), sample_rate)      # The isolated sound
torchaudio.save("residual.wav", result.residual.cpu(), sample_rate)  # Everything else

import torch
import numpy as np
from sam_audio import SAMAudio, SAMAudioProcessor
from torchcodec.decoders import VideoDecoder

# NOTE: Requires SAM3 for creating masks
# pip install git+https://github.com/facebookresearch/sam3.git
from sam3.model_builder import build_sam3_video_predictor

# Load SAM-Audio model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = SAMAudio.from_pretrained("facebook/sam-audio-large").to(device).eval()
processor = SAMAudioProcessor.from_pretrained("facebook/sam-audio-large")

# Load video
video_file = "path/to/video.mp4"
decoder = VideoDecoder(video_file)
frames = decoder[:]

# Create mask using SAM3 (example with text prompt)
video_predictor = build_sam3_video_predictor()
response = video_predictor.handle_request({
    "type": "start_session",
    "resource_path": video_file,
})
session_id = response["session_id"]

masks = []
for frame_index in range(len(decoder)):
    response = video_predictor.handle_request({
        "type": "add_prompt",
        "session_id": session_id,
        "frame_index": frame_index,
        "text": "The person on the left",  # Visual object to isolate
    })
    mask = response["outputs"]["out_binary_masks"]
    if mask.shape[0] == 0:
        mask = np.zeros_like(frames[0, [0]], dtype=bool)
    masks.append(mask[:1])

mask = torch.from_numpy(np.concatenate(masks)).unsqueeze(1)

# Process with visual prompting
inputs = processor(
    audios=[video_file],
    descriptions=[""],
    masked_videos=processor.mask_videos([frames], [mask]),
).to(device)

with torch.inference_mode():
    result = model.separate(inputs)

import torch
import torchaudio
from sam_audio import SAMAudio, SAMAudioProcessor

# Load model and processor
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = SAMAudio.from_pretrained("facebook/sam-audio-large").to(device).eval()
processor = SAMAudioProcessor.from_pretrained("facebook/sam-audio-large")

# Define anchors: [type, start_time, end_time]
# "+" means the sound IS present in this time range
# "-" means the sound is NOT present in this time range
anchors = [
    ["+", 6.3, 7.0],  # Sound occurs between 6.3 and 7.0 seconds
]

# Process with span prompting
inputs = processor(
    audios=[audio_file],
    descriptions=["A horn honking"],
    anchors=[anchors],
).to(device)

with torch.inference_mode():
    result = model.separate(inputs)

anchors = [
    ["+", 2.0, 3.5],   # Sound present from 2.0 to 3.5 seconds
    ["+", 8.0, 9.0],   # Sound present from 8.0 to 9.0 seconds
    ["-", 0.0, 1.0],   # Sound NOT present from 0.0 to 1.0 seconds
]