#028

silu_and_mul

bf16 vllm · both · vllm._C.silu_and_mul / csrc/activation_kernels.cu · importance 1.8%

Reference Implementation

reference.py 
import torch
import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):

    def __init__(self) -> None:
        super().__init__()

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        if x.shape[-1] >= 2:
            split = x.shape[-1] // 2
            gate = x[..., :split]
            up = x[..., split:split + split]
            activated = F.silu(gate.float()).to(x.dtype)
            return (activated * up).to(x.dtype)
        return F.silu(x.float()).to(x.dtype)

Shapes

TSOL hardware:
# shape TSOL(XPU-A)TProdS
0 [1, 1536] 0.00 us 7.60 us 0.0%
1 [1630, 1536] 1.42 us 12.20 us 11.6%
2 [4688, 1536] 4.08 us 21.20 us 19.2%
3 [10704, 1536] 9.31 us 38.10 us 24.4%
4 [16024, 1536] 13.93 us 52.60 us 26.5%
5 [24656, 1536] 21.44 us 76.20 us 28.1%
6 [49032, 1536] 42.63 us 143.80 us 29.6%
7 [128632, 1536] 111.84 us 362.70 us 30.8%
8 [175680, 1536] 152.74 us 490.70 us 31.1%
9 [1, 2048] 0.00 us 7.50 us 0.0%
10 [183, 2048] 0.21 us 7.70 us 2.7%
11 [1113, 2048] 1.29 us 9.30 us 13.9%
12 [1, 2560] 0.00 us 7.50 us 0.0%
13 [1736, 3072] 3.02 us 15.20 us 19.9%
14 [5000, 3072] 8.69 us 31.40 us 27.7%
15 [11570, 3072] 20.12 us 63.20 us 31.8%
16 [14320, 3072] 24.90 us 102.20 us 24.4%
17 [128, 4096] 0.30 us 7.70 us 3.9%
18 [256, 4096] 0.59 us 8.00 us 7.4%
19 [512, 4096] 1.19 us 9.10 us 13.1%
20 [1023, 4096] 2.37 us 12.50 us 19.0%
21 [2044, 4096] 4.74 us 19.50 us 24.3%
22 [4093, 4096] 9.49 us 33.10 us 28.7%
23 [8192, 4096] 18.99 us 58.30 us 32.6%
24 [16418, 4096] 38.06 us 109.60 us 34.7%
25 [20538, 4096] 47.62 us 135.70 us 35.1%
26 [528, 5120] 1.53 us 12.30 us 12.4%
27 [1056, 5120] 3.06 us 19.40 us 15.8%
28 [1910, 5120] 5.54 us 29.50 us 18.8%
29 [4177, 5120] 12.11 us 57.00 us 21.2%
30 [8192, 5120] 23.74 us 105.10 us 22.6%
31 [540, 8192] 2.50 us 12.90 us 19.4%
32 [1003, 8192] 4.65 us 19.40 us 24.0%
33 [2046, 8192] 9.49 us 33.20 us 28.6%
34 [4098, 8192] 19.00 us 58.90 us 32.3%
35 [8179, 8192] 37.93 us 110.80 us 34.2%
36 [15381, 8192] 71.32 us 202.60 us 35.2%

Input Generation

input.py 
import torch

def _make_inputs(shape: list[int]) -> dict[str, torch.Tensor]:
    tensor_shape = tuple((int(dim) for dim in shape))
    if len(tensor_shape) == 0:
        tensor_shape = (1,)
    x = torch.randn(tensor_shape, dtype=torch.bfloat16, device='cuda')
    return {'x': x}