diff --git "a/reduced_ppl_average copy.json" "b/reduced_ppl_average copy.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average copy.json" @@ -0,0 +1,17965 @@ +{ + "original_ppl": 3.697141408920288, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3464.481201171875 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3677.32177734375 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3220.70751953125 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 2131.33447265625 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3050.191650390625 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3169.390380859375 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 2524.9228515625 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3505.794189453125 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3512.329833984375 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3000.78466796875 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3446.185791015625 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3538.577880859375 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3145.047119140625 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3348.456787109375 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3593.623779296875 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3501.1298828125 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3036.5185546875 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3866.78857421875 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3348.3193359375 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3167.444580078125 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3323.9296875 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3516.482421875 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3708.910400390625 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4347.1240234375 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3240.3759765625 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3352.1630859375 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3381.581298828125 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3230.160400390625 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3474.3310546875 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3624.54541015625 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3904.429443359375 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3931.96875 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3287.423583984375 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3556.094970703125 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3329.392578125 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3602.624267578125 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3520.4921875 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3685.15771484375 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3495.671630859375 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3414.18310546875 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3327.97998046875 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3443.551025390625 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3444.01416015625 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3339.546142578125 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 2957.610595703125 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3380.78125 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3418.731689453125 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3345.331787109375 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3653.060302734375 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3615.57275390625 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3484.28857421875 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3334.352783203125 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3082.19921875 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3499.62109375 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3387.6494140625 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3406.943115234375 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3531.309814453125 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3506.603515625 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3868.710205078125 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3764.70654296875 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3482.587890625 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3693.3271484375 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3579.640869140625 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3575.9287109375 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3613.32177734375 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3786.64208984375 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3601.834228515625 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3556.9462890625 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3534.129638671875 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3533.364501953125 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3586.906005859375 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3581.355224609375 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3393.70947265625 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3100.755615234375 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3498.159423828125 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3489.486328125 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3524.822509765625 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3541.474609375 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3562.73486328125 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3857.061279296875 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3233.695556640625 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3500.315185546875 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3616.383056640625 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3580.91796875 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3646.1826171875 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3648.7255859375 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3388.560546875 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3332.454833984375 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3620.230712890625 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3646.46435546875 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3618.018310546875 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3555.70849609375 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3578.958251953125 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3403.302734375 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3548.899169921875 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3505.10205078125 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3525.427734375 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3479.354736328125 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3556.078125 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3519.56591796875 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3695.744140625 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3425.1474609375 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3614.25927734375 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3449.5693359375 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3472.6650390625 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3519.465087890625 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3514.176025390625 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3397.32958984375 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3332.216796875 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3525.00732421875 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3529.767578125 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3414.82470703125 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3572.935546875 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3578.476806640625 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3620.157958984375 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3347.22412109375 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3481.14013671875 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3397.452880859375 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3501.714111328125 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3550.26318359375 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3554.250732421875 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3657.511962890625 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3503.755126953125 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3498.2060546875 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3591.5576171875 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3502.08154296875 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3557.892822265625 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3562.761962890625 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3387.365234375 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3597.06640625 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3500.82275390625 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3503.26416015625 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3435.848876953125 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3592.839111328125 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3547.57958984375 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3612.81884765625 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3530.255615234375 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3581.252685546875 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3535.090087890625 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3484.102783203125 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3616.645263671875 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3582.776123046875 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3615.041748046875 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3424.03369140625 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3534.27783203125 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3601.5146484375 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3554.345703125 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3570.53076171875 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3552.1259765625 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3440.91162109375 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3528.848876953125 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3523.068359375 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3497.592041015625 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3476.7373046875 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3579.920654296875 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3554.05078125 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3449.1318359375 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3632.650146484375 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3574.29541015625 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3606.746337890625 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3529.7841796875 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3531.518310546875 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3541.02197265625 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3489.4365234375 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3556.959716796875 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3601.68994140625 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3603.177734375 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3553.97265625 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3526.6650390625 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3569.91455078125 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3655.628662109375 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3565.99462890625 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3552.451171875 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3576.16064453125 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3590.4892578125 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3606.357666015625 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3585.305419921875 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3383.326171875 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3453.83544921875 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3570.1494140625 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3518.126220703125 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3598.716796875 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3585.339599609375 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3540.32666015625 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3647.01416015625 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3611.468505859375 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3496.431640625 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3465.0263671875 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3481.375732421875 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3568.396240234375 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3558.585205078125 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3567.113525390625 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3462.037109375 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3537.369873046875 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3454.96240234375 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3482.454833984375 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3580.498046875 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3562.57177734375 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3555.57958984375 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3659.325927734375 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3480.834716796875 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3484.60107421875 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3482.33544921875 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3526.134033203125 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3547.802978515625 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3615.197021484375 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3383.19384765625 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3525.115234375 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3534.8408203125 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3586.694091796875 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3581.105712890625 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3566.310791015625 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3430.217529296875 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3412.60107421875 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3544.95166015625 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3576.293701171875 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3588.49365234375 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3560.6220703125 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3558.55126953125 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3364.076416015625 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3563.0439453125 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3550.591796875 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3594.59716796875 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3594.569580078125 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3594.50146484375 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3576.072021484375 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3421.145263671875 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3577.722900390625 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3514.330322265625 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3542.4677734375 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3514.33349609375 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3542.069091796875 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3515.218505859375 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3684.802490234375 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3762.018310546875 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3573.9306640625 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3572.294921875 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3504.58056640625 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3569.55712890625 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3580.8291015625 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3705.78515625 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3521.28466796875 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3599.454833984375 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3466.814453125 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3495.891357421875 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3594.898681640625 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3569.083984375 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3679.317626953125 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3586.215087890625 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3479.630126953125 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3414.84423828125 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3582.878662109375 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3625.776123046875 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3583.28857421875 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3446.153076171875 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3570.9599609375 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3478.093994140625 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3520.408447265625 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3611.344482421875 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3527.7685546875 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3514.8330078125 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3553.281494140625 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3497.55224609375 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3538.7939453125 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3457.945556640625 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3492.0498046875 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3582.2705078125 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3596.208740234375 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3400.165771484375 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3518.549072265625 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3572.260986328125 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3459.627685546875 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3417.1669921875 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3565.590087890625 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3547.9111328125 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3573.610595703125 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3516.264404296875 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3507.37255859375 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3456.105712890625 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3545.1376953125 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3559.121337890625 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3523.32373046875 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3329.42138671875 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3736.813232421875 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3528.31689453125 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3544.701416015625 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3549.63037109375 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3636.865234375 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3560.995361328125 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3538.18994140625 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3502.953369140625 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3496.99853515625 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3557.852294921875 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3510.022705078125 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3564.0771484375 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3545.306640625 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3403.52978515625 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3596.146728515625 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3488.4248046875 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3589.684814453125 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3531.50146484375 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3578.11181640625 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3580.8837890625 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3546.92333984375 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3489.386474609375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3459.037109375 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3493.471923828125 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3558.7548828125 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3516.304931640625 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3520.086181640625 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3563.96826171875 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3482.913330078125 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3543.123046875 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3548.310302734375 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3490.035400390625 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3539.091064453125 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3532.9736328125 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3556.837890625 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3466.3583984375 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3571.3857421875 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3528.11181640625 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3523.404296875 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3550.706787109375 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3552.03125 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3618.64306640625 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3700.209228515625 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3508.275634765625 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3543.61328125 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3558.97900390625 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3557.580810546875 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3568.529052734375 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3550.137939453125 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3618.56689453125 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3565.09375 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3551.69580078125 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3554.857421875 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3553.267822265625 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3556.234130859375 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3574.789794921875 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3567.1953125 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3521.724609375 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3572.42822265625 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3505.3828125 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3557.526611328125 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3562.70751953125 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3523.824462890625 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3506.0615234375 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3544.02880859375 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3531.734130859375 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3566.7734375 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3565.3994140625 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3555.77294921875 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3636.140380859375 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3626.523193359375 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3547.261474609375 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3565.2294921875 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3543.94775390625 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3546.78466796875 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3545.637939453125 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3511.783935546875 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3548.662109375 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3539.570068359375 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3569.689697265625 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3555.94921875 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3569.839599609375 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3568.117431640625 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3489.090087890625 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3524.620849609375 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3515.684326171875 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3541.211181640625 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3559.54248046875 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3559.8037109375 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3552.529296875 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3529.31298828125 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3501.303466796875 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3544.8671875 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3548.6689453125 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3551.418212890625 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3552.16015625 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3551.309814453125 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3567.399658203125 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3499.06689453125 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3537.140380859375 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3532.791748046875 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3530.0234375 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3560.214599609375 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3543.17724609375 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3515.684326171875 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3533.792724609375 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3533.68798828125 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3562.948974609375 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3537.25537109375 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3569.530029296875 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3553.112060546875 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3495.90478515625 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3468.282958984375 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3518.354248046875 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3540.329833984375 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3532.724365234375 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3551.790771484375 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3559.3115234375 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3564.862548828125 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3495.2314453125 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3538.206787109375 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3534.928466796875 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3577.497802734375 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3562.66015625 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3554.98291015625 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3525.93212890625 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3551.787353515625 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3512.072021484375 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3554.606689453125 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3531.20849609375 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3566.161376953125 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3554.548828125 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3594.090087890625 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3638.391357421875 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3517.961669921875 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3524.392333984375 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3545.21875 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3570.564697265625 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3556.678466796875 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3582.2021484375 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3548.87890625 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3518.823974609375 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3520.982421875 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3587.408935546875 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3551.90576171875 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3552.94580078125 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3533.260009765625 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3465.793212890625 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3572.6630859375 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3570.96337890625 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3549.400146484375 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3550.20263671875 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3552.08544921875 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3561.250244140625 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3524.570556640625 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3564.461181640625 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3547.227783203125 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3566.998046875 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3563.434814453125 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3560.856201171875 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3517.49560546875 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3594.71728515625 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3570.108642578125 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3522.974365234375 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3528.97314453125 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3553.93212890625 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3550.4091796875 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3491.064208984375 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3587.966552734375 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3556.359619140625 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3566.749755859375 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3528.774658203125 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3555.28466796875 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3557.7333984375 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3516.355224609375 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3582.998291015625 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3549.5625 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3512.741943359375 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3550.351318359375 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3539.79638671875 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3551.49951171875 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3550.679931640625 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3606.00341796875 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3535.595947265625 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3560.23486328125 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3591.150390625 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3569.308837890625 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3568.093505859375 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3533.839599609375 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3567.950439453125 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3572.162109375 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3518.13623046875 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3523.737060546875 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3541.10302734375 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3543.457763671875 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3527.93017578125 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3612.70166015625 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3546.033447265625 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3601.41845703125 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3519.5390625 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3547.81640625 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3554.748779296875 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3654.381103515625 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3527.40185546875 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3526.718994140625 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3566.67822265625 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3479.480712890625 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3577.4873046875 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3581.877685546875 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3606.828857421875 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3558.83642578125 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3533.543212890625 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3586.919921875 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3466.576416015625 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3552.190673828125 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3544.238525390625 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3628.2978515625 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3480.3369140625 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3502.919921875 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3559.270751953125 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3519.834228515625 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3542.278564453125 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3545.928955078125 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3588.63037109375 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3549.474609375 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3498.77001953125 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3538.935791015625 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3528.431396484375 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3553.376220703125 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3546.984130859375 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3502.365478515625 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3529.730712890625 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3485.910888671875 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3542.099365234375 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3547.71484375 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3530.568603515625 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3592.58203125 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3440.7705078125 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3562.411865234375 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3584.35498046875 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3511.000244140625 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3602.765380859375 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3510.27392578125 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3519.12255859375 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3594.0078125 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3542.25146484375 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3545.79345703125 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3633.08642578125 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3593.10302734375 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3584.33447265625 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3594.144775390625 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3612.37109375 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3576.371826171875 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3455.83544921875 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3628.678466796875 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3574.20361328125 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3504.771240234375 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3512.302978515625 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3581.53955078125 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3587.593505859375 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3645.18505859375 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3749.11376953125 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3590.612548828125 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3561.281005859375 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3498.61962890625 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3530.710205078125 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3567.9404296875 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3521.49609375 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3712.47412109375 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3666.239013671875 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3564.026123046875 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3569.373291015625 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3541.224609375 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3621.1181640625 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3426.92822265625 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3298.002685546875 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3188.660400390625 + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.587299823760986 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.621645927429199 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.606184482574463 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.591529846191406 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.595300197601318 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.606298923492432 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.578024864196777 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.558347225189209 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.565422534942627 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.539042949676514 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.525199890136719 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.55272912979126 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.567893028259277 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.547338962554932 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.578011989593506 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.579082012176514 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.542652130126953 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.5580949783325195 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.531676769256592 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.560765743255615 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.541767120361328 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.579802989959717 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.601284503936768 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.570128917694092 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.569732189178467 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.539124488830566 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.561048984527588 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.572926998138428 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.5919389724731445 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.621241569519043 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.600897789001465 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.587358474731445 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.54921817779541 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.582107067108154 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.573081970214844 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.612077713012695 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.620278835296631 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.593531608581543 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.605550289154053 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.571804523468018 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.580696105957031 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.580711841583252 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.605426788330078 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.603277206420898 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.587345123291016 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.597412109375 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.5576558113098145 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.583559989929199 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.582685947418213 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.619879245758057 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.603657245635986 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.585909843444824 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.600263595581055 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.580042362213135 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.583291530609131 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.5865478515625 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.607184886932373 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.629411220550537 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.588752746582031 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.598271369934082 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.586933612823486 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.582657814025879 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.591380596160889 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.608002185821533 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.605777740478516 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.598511695861816 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.586959362030029 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.582998275756836 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.595479488372803 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.591337203979492 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.6023688316345215 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.600842475891113 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.5933027267456055 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.587803840637207 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.579985618591309 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.587390422821045 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.576834201812744 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.60318660736084 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.603156089782715 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.598178863525391 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.595784664154053 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.581279754638672 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.592929363250732 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.599449157714844 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.607228755950928 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.60359001159668 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.591875076293945 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.5998334884643555 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.584235191345215 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.582729339599609 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.591054916381836 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.602621555328369 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.607583045959473 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.578174591064453 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.580382347106934 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.600681781768799 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.601696968078613 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.586922645568848 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.600650787353516 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.599786281585693 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.593698024749756 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.588168144226074 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.5824480056762695 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.58252477645874 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.589319705963135 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.601154327392578 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.602333068847656 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.596490859985352 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.599719524383545 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.591432571411133 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.603636264801025 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.603952407836914 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.6040825843811035 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.601742267608643 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.602610111236572 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.602569103240967 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.585309982299805 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.603806495666504 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.587503910064697 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.593698024749756 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.5925798416137695 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.5892462730407715 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.591400623321533 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.60642671585083 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.597780704498291 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.601413726806641 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.59487247467041 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.600255012512207 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.594669818878174 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.588927268981934 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.597878932952881 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.606900215148926 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.590006351470947 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.594246864318848 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.596985340118408 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.601166725158691 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.593268871307373 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.598250865936279 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.605244159698486 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.604512691497803 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.587610244750977 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.601518630981445 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.6021952629089355 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.605716705322266 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.603775501251221 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.607298374176025 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.6122212409973145 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.593820571899414 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.599548816680908 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.596066474914551 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.593171119689941 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.599617004394531 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.605527400970459 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.602389812469482 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.598952293395996 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.602376937866211 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.606771945953369 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.602109432220459 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.592378616333008 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.590651512145996 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.6001057624816895 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.598791122436523 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.604039669036865 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.604513168334961 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.603310585021973 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.588095188140869 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.601797580718994 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.600030422210693 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.606400489807129 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.600368499755859 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.597190856933594 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.589382648468018 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.587274074554443 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.596391677856445 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.59694242477417 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.592281341552734 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.59852933883667 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.57909631729126 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.581469535827637 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.600376129150391 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.596221923828125 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.605307579040527 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.600579738616943 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.60133171081543 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.605119705200195 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.60227108001709 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.598724365234375 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.606266975402832 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.598618507385254 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.598852157592773 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.601508140563965 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.598553657531738 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.601412296295166 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.59518575668335 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.594697952270508 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.597851276397705 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.6041178703308105 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.600735187530518 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.601080894470215 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.60016393661499 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.5956621170043945 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.600144863128662 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.597470760345459 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.6035685539245605 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.603593826293945 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.605932712554932 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.599174976348877 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.595711708068848 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.600107669830322 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.5986552238464355 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.601630210876465 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.602270603179932 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.6023478507995605 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.600612163543701 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.587116241455078 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.594443321228027 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.597299575805664 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.605952262878418 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.60903263092041 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.610475540161133 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.610025405883789 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.594229221343994 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.595969200134277 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.597070693969727 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 6.599231719970703 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 6.601463794708252 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 6.602692127227783 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 6.603835105895996 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 6.584806442260742 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 6.598879814147949 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 6.592294216156006 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 6.599766254425049 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 6.602532386779785 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 6.606451988220215 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 6.607882022857666 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 6.596158981323242 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 6.59637451171875 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 6.607106685638428 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 6.607349872589111 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 6.609874248504639 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 6.612264156341553 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 6.604864597320557 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 6.598066806793213 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 6.592206954956055 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 6.596440315246582 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 6.604358196258545 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 6.605189800262451 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 6.613621234893799 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 6.606575965881348 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 6.6003241539001465 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 6.607431888580322 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 6.603500843048096 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 6.61030387878418 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 6.605193614959717 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 6.612126350402832 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 6.603789806365967 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 6.59227991104126 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 6.593422889709473 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 6.596105575561523 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 6.604356288909912 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 6.607855796813965 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 6.598106384277344 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 6.600514888763428 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 6.605868339538574 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 6.606652736663818 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 6.601559162139893 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 6.599547386169434 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 6.603506088256836 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 6.5987725257873535 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 6.601705074310303 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 6.595846176147461 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 6.605261325836182 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 6.6010050773620605 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 6.605607986450195 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 6.607065200805664 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 6.599732875823975 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 6.601710319519043 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 6.601534843444824 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 6.6002373695373535 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 6.601970672607422 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 6.599880218505859 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 6.599278926849365 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 6.596271514892578 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 6.596656799316406 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 6.596795558929443 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 6.602492809295654 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 6.600967884063721 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 6.599152565002441 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 6.597898483276367 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 6.594899654388428 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 6.597871780395508 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 6.596582412719727 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 6.601064205169678 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 6.595006465911865 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 6.603090286254883 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 6.601056098937988 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 6.60007381439209 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 6.606298923492432 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 6.59986686706543 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 6.601884365081787 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 6.59735631942749 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 6.601327896118164 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 6.606303691864014 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 6.604212760925293 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 6.601311206817627 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 6.597829341888428 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 6.6004838943481445 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 6.598099231719971 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 6.608369827270508 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 6.606858730316162 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 6.600128650665283 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 6.6029767990112305 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 6.6003546714782715 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 6.59958028793335 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 6.593913555145264 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 6.604043006896973 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 6.604269027709961 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 6.606225967407227 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 6.606950283050537 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 6.595943450927734 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 6.599369525909424 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 6.598005294799805 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 6.603532791137695 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 6.607446193695068 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 6.605018138885498 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 6.60512638092041 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 6.598582744598389 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 6.5997185707092285 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 6.5999531745910645 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 6.6023406982421875 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 6.602704048156738 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 6.601614952087402 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 6.599346160888672 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 6.5980119705200195 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 6.596429347991943 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 6.596161365509033 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 6.605160236358643 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 6.611623764038086 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 6.610204219818115 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 6.608973026275635 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 6.599440574645996 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 6.597598075866699 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 6.597370624542236 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 6.605222225189209 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 6.604358196258545 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 6.602878570556641 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 6.605030536651611 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 6.5970940589904785 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 6.600498676300049 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 6.596414566040039 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 6.6046833992004395 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 6.6062541007995605 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 6.605898857116699 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 6.607046604156494 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 6.597235679626465 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 6.599125862121582 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 6.597239017486572 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 6.603255748748779 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 6.604758262634277 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 6.60517692565918 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 6.607698440551758 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 6.5967912673950195 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 6.599171161651611 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 6.596045017242432 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 6.606232643127441 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 6.608950138092041 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 6.60835599899292 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 6.607090950012207 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 6.6011457443237305 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 6.600656509399414 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 6.596714019775391 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 6.604354381561279 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 6.605943202972412 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 6.6068572998046875 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 6.608158111572266 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 6.598452568054199 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 6.5994553565979 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 6.598047733306885 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 6.603742599487305 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 6.6061906814575195 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 6.606211185455322 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 6.604738712310791 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 6.5995354652404785 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 6.599943161010742 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 6.5973687171936035 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 6.604788780212402 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 6.605924606323242 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 6.60308837890625 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 6.603743553161621 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 6.598557472229004 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 6.598172664642334 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 6.6009602546691895 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 6.604080677032471 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 6.605562210083008 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 6.605836391448975 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 6.603350639343262 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 6.598554611206055 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 6.6019134521484375 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 6.597433090209961 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 6.606156349182129 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 6.606710433959961 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 6.607789516448975 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 6.6072096824646 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 6.602711200714111 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 6.600829601287842 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 6.599113941192627 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 6.6030168533325195 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 6.602009296417236 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 6.60439920425415 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 6.603659629821777 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 6.598233699798584 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 6.5990753173828125 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 6.597658634185791 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 6.6041483879089355 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 6.604606628417969 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 6.604905605316162 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 6.603374481201172 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 6.599079132080078 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 6.5978217124938965 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 6.6002726554870605 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 6.604966640472412 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 6.604852199554443 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 6.6043243408203125 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 6.604616641998291 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 6.599572658538818 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 6.601479530334473 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 6.600064754486084 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 6.604489803314209 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 6.604806423187256 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 6.606635093688965 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 6.60684871673584 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 6.602787017822266 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 6.6002655029296875 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 6.60007905960083 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 6.6053385734558105 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 6.604893207550049 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 6.605632305145264 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 6.604212760925293 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 6.600472450256348 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 6.59916353225708 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 6.600151538848877 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 6.6047563552856445 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 6.605164051055908 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 6.60540246963501 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 6.605785846710205 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 6.597067356109619 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 6.599972248077393 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 6.600137233734131 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 6.604395389556885 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 6.6043524742126465 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 6.604781150817871 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 6.605875492095947 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 6.599791049957275 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 6.601497173309326 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 6.599152565002441 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 6.603846073150635 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 6.604429721832275 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 6.6054158210754395 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 6.604636192321777 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 6.599225997924805 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 6.6016764640808105 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 6.600290298461914 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 6.604108810424805 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 6.604352951049805 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 6.604194641113281 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 6.605434417724609 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 6.599965572357178 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 6.600320339202881 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 6.600098133087158 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 6.603488445281982 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 6.604409694671631 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 6.605992317199707 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 6.605588912963867 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 6.5988593101501465 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 6.599915504455566 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 6.602593898773193 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 6.603749752044678 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 6.603925704956055 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 6.603025913238525 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 6.603877544403076 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 6.600656509399414 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 6.601109981536865 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 6.597596168518066 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 6.605644226074219 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 6.605431079864502 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 6.603664875030518 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 6.604753017425537 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 6.60113525390625 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 6.6016106605529785 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 6.600804328918457 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 6.604508876800537 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 6.606046676635742 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 6.603638648986816 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 6.607027053833008 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 6.5996527671813965 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 6.599191665649414 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 6.600780487060547 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 6.603200435638428 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 6.603689670562744 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 6.599296569824219 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 6.601716041564941 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 6.599742889404297 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 6.598677158355713 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 6.601217746734619 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 6.6124982833862305 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 6.612306118011475 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 6.606039524078369 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 6.602809906005859 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 6.601545333862305 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 6.604678630828857 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 6.602466583251953 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 6.603470802307129 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 6.603786945343018 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 6.604178428649902 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 6.602931022644043 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 6.599786281585693 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 6.602194309234619 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 6.6041646003723145 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 6.619690418243408 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 6.620758533477783 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 6.615204811096191 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 6.61479377746582 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 6.595831871032715 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 6.594951629638672 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 6.601993083953857 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 6.605774402618408 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 6.605727672576904 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 6.606330871582031 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 6.608585357666016 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 6.6034440994262695 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 6.605523109436035 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 6.60240364074707 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 6.633901596069336 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 6.626533508300781 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 6.61317253112793 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 6.613747596740723 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 6.583765506744385 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 6.603389263153076 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 6.60793924331665 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 6.605195999145508 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 6.6029372215271 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 6.603430271148682 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 6.605612277984619 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 6.606897830963135 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 6.604814052581787 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 6.606776714324951 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 6.604368686676025 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 6.604959011077881 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 6.617969036102295 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 6.60932731628418 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 6.589051246643066 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 6.588523864746094 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 6.589207649230957 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 6.603640079498291 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 6.6061506271362305 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 6.60843563079834 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 6.607334136962891 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 6.621618747711182 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 6.6069841384887695 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 6.545148849487305 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.370658874511719 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.3709917068481445 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.372006416320801 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.370563507080078 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.37232780456543 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.373668670654297 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.3709635734558105 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.367496013641357 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.368441581726074 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.367706775665283 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.365130424499512 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.367547988891602 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.369000434875488 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.367656230926514 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.368114948272705 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.369100570678711 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.367117881774902 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.366388320922852 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.36487340927124 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.368755340576172 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.365574836730957 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.368518352508545 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.369019508361816 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.369363307952881 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.368993282318115 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.3654866218566895 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.367645263671875 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.368037223815918 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.369693279266357 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.369786262512207 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.369330883026123 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.36944580078125 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.366352081298828 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.368671894073486 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.368444442749023 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.371212482452393 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.372028827667236 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.370617389678955 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.370952606201172 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.366947174072266 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.368783473968506 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.369964122772217 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.3701066970825195 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.370917320251465 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.37069034576416 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.369626522064209 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.367837429046631 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.3685102462768555 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.367172718048096 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.371077060699463 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.370008945465088 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.370212078094482 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.370441436767578 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.369170188903809 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.366943836212158 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.369058132171631 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.370612621307373 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.371523857116699 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.369246959686279 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.367514133453369 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.367551326751709 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.366318702697754 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.368446350097656 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.370789051055908 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.3708600997924805 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.370893955230713 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.370158672332764 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.367159843444824 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.370361804962158 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.367939472198486 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.37029504776001 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.370350360870361 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.369771480560303 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.369573593139648 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.366819381713867 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.3677144050598145 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.368299961090088 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.37019157409668 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.371087074279785 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.370404243469238 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.3695831298828125 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.366743564605713 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.369704723358154 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.36887264251709 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.370004653930664 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.370041847229004 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.370377063751221 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.371026992797852 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.36906623840332 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.370051383972168 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.369397163391113 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.370617389678955 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.3704142570495605 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.368880748748779 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.36940336227417 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.369184970855713 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.369528770446777 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.3685126304626465 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.369605541229248 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.369452476501465 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.370082855224609 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.369543075561523 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.36824893951416 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.368817329406738 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.370004177093506 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.371281147003174 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.3708906173706055 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.370126724243164 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.369634628295898 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.368972301483154 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.37039041519165 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.369751453399658 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.370290279388428 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.370081424713135 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.370199680328369 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.369494438171387 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.36939001083374 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.369959354400635 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.36924409866333 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.369386196136475 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.3701677322387695 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.368776798248291 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.370244979858398 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.370024681091309 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.371572971343994 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.369118690490723 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.369749546051025 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.36985969543457 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.36874532699585 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.36946439743042 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.369516372680664 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.369718551635742 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.370192527770996 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.37045431137085 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.370177268981934 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.3698554039001465 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.36964750289917 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.368263244628906 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.370279312133789 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.370323657989502 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.3698954582214355 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.370196342468262 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.36991024017334 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.369865417480469 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.369717121124268 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.368814468383789 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.368879318237305 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.369174480438232 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.369974613189697 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.369632720947266 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.369807720184326 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.369532585144043 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.369607448577881 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.370362281799316 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.370495319366455 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.370677947998047 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.370682716369629 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.370794296264648 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.369177341461182 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.3691606521606445 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.369308948516846 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.369301795959473 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.370370388031006 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.370275020599365 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.369775772094727 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.369058132171631 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.369899272918701 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.369477272033691 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.3701043128967285 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.370334148406982 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.370373725891113 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.370621681213379 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.368663311004639 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.369245529174805 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.370733261108398 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.369558811187744 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.37009859085083 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.3681817054748535 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.367650032043457 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.369901657104492 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.369140148162842 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.370650768280029 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.37069034576416 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.370615482330322 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.370699405670166 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.370719909667969 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.369001865386963 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.369406700134277 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.370663166046143 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.369678497314453 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.369998455047607 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.37005615234375 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.369414806365967 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.369797229766846 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.369734764099121 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.369627952575684 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.370367527008057 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.370331764221191 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.370481491088867 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.369726657867432 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.369414806365967 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.369966506958008 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.368894577026367 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.370353698730469 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.3705620765686035 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.370404243469238 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.369851589202881 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.369944095611572 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.36950159072876 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.37017297744751 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.370259761810303 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.370347499847412 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.370332717895508 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.370377540588379 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.369397163391113 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.369746685028076 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.368403911590576 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.369879722595215 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.370453834533691 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.3704681396484375 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.3705668449401855 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.369260787963867 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.3699421882629395 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.368878364562988 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.370258331298828 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.370457172393799 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.3696722984313965 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.3701300621032715 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.367971420288086 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.369325637817383 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.36854887008667 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.370546817779541 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.370273113250732 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.369988918304443 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.369546413421631 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.367751598358154 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.369838237762451 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.369894027709961 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.370205879211426 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.370344638824463 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.369682312011719 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.369753360748291 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.369647026062012 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.369004726409912 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.370223522186279 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.370478630065918 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.370723724365234 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.36993408203125 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.370380878448486 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.369374752044678 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.370112419128418 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.369381427764893 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.370210647583008 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.370704174041748 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.370889186859131 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.370656967163086 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.36891508102417 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.370216369628906 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.36879825592041 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.370626926422119 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.3706440925598145 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.370516300201416 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.370355129241943 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.369748592376709 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.370649814605713 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.37078332901001 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.370316982269287 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.370417594909668 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.369919776916504 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.3699798583984375 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.3686203956604 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.369569301605225 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.3697309494018555 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.369585990905762 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.370404243469238 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.370459079742432 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.369085311889648 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.369277000427246 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.370120525360107 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.3698225021362305 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.369800567626953 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.369987964630127 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.369215965270996 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.369477272033691 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.369507789611816 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.3702311515808105 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.369599342346191 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.369622230529785 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.369960784912109 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.369366645812988 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.370115280151367 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.368876934051514 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.369987964630127 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.3692946434021 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.371050834655762 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.370308876037598 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.370563507080078 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.370288372039795 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.368695259094238 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.370031833648682 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.37069845199585 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.369816780090332 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.370055198669434 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.369913101196289 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.369883060455322 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.369180679321289 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.3699164390563965 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.369873523712158 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.370911598205566 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.370357036590576 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.3701300621032715 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.369647979736328 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.36823844909668 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.368480682373047 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.368778228759766 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.370745658874512 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.370642185211182 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.370589256286621 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.370792388916016 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.369303226470947 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.369446754455566 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.368327617645264 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.370411396026611 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.370616436004639 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.370595932006836 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.370485305786133 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.369943141937256 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.369462490081787 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.370121002197266 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.3701372146606445 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.370247840881348 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.370376110076904 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.369966506958008 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.3690266609191895 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.3697829246521 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.369048595428467 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.370481014251709 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.370688438415527 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.370548725128174 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.370810508728027 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.368679523468018 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.369833469390869 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.3692827224731445 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.370330333709717 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.370555877685547 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.37021541595459 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.37031364440918 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.368698596954346 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.369692325592041 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.3687238693237305 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.370598793029785 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.3705058097839355 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.370670795440674 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.370545387268066 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.369062900543213 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.369428634643555 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.369572639465332 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.370123386383057 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.370323657989502 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.37005615234375 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.370494842529297 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.368701934814453 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.370166301727295 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.368327617645264 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.3705291748046875 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.370993614196777 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.370922088623047 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.370280742645264 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.368600845336914 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.369245529174805 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.368486404418945 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.3705644607543945 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.370523452758789 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.3705854415893555 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.370707035064697 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.368817329406738 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.369629383087158 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.36944055557251 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.370484352111816 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.370792388916016 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.370545864105225 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.370151519775391 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.369105815887451 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.369824409484863 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.36972188949585 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.370765686035156 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.370667457580566 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.370337009429932 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.37004280090332 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.369317054748535 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.369020462036133 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.3690080642700195 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.370368957519531 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.370558261871338 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.370368957519531 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.370271682739258 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.367918014526367 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.369907855987549 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.368680953979492 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.370500087738037 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.37069845199585 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.370538234710693 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.370504379272461 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.368698596954346 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.370347023010254 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.369478702545166 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.370362281799316 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.369499683380127 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.370457172393799 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.370245933532715 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.369790554046631 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.369180679321289 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.3686676025390625 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.370546817779541 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.3705244064331055 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.370411396026611 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.370611190795898 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.368659973144531 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.370439529418945 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.368627548217773 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.370355129241943 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.3705339431762695 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.370579719543457 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.370670318603516 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.368842124938965 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.368569374084473 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.368898391723633 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.370463848114014 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.370635986328125 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.370703220367432 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.370877265930176 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.369894027709961 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.369246482849121 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.368661880493164 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.370605945587158 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.370710372924805 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.370368957519531 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.370603084564209 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.368288993835449 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.368137359619141 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.369358539581299 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.370471000671387 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.370543479919434 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.3706560134887695 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.3705315589904785 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.367757320404053 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.369256019592285 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.368664741516113 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.370599746704102 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.370527267456055 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.370788097381592 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.370481967926025 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.3685994148254395 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.369345188140869 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.3690643310546875 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.370444297790527 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.370588302612305 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.370532989501953 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.3704423904418945 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.368338108062744 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.369311332702637 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.3697028160095215 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.370581150054932 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.370548248291016 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.370380878448486 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.370533466339111 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.368878364562988 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.369256019592285 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.369171142578125 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.370511531829834 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.370306015014648 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.372114658355713 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.37081241607666 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.368305683135986 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.369886875152588 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.369573593139648 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.370284080505371 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.370397567749023 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.370340824127197 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.3701066970825195 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.3685431480407715 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.369591236114502 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.369800090789795 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.370703220367432 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.370458126068115 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.3702239990234375 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.370509147644043 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.369507789611816 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.36972713470459 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.3690361976623535 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.370020389556885 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.369931221008301 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.370157241821289 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.370376110076904 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.3678412437438965 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.369663238525391 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.370247840881348 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.370338439941406 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.370195388793945 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.370934009552002 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.370158672332764 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.368406772613525 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.369710922241211 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.3704047203063965 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.37039852142334 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.370025157928467 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.370766639709473 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.370181083679199 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.368619918823242 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.369656085968018 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.369997024536133 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.370120525360107 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.370104789733887 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.370178699493408 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.370549201965332 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.369039058685303 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.369682788848877 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.370556354522705 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.3708176612854 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.371338844299316 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.3713698387146 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.370495319366455 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.367344379425049 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.370899677276611 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.369388103485107 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.3706560134887695 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.370896816253662 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.370218753814697 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.370737552642822 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.366901397705078 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.369959831237793 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.370751857757568 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.372854232788086 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.37371826171875 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.372132301330566 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.371866226196289 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.368535995483398 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.370369911193848 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.370703220367432 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.370266437530518 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.370609283447266 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.3709564208984375 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.370542526245117 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.367404937744141 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.371592998504639 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.369471549987793 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.370596408843994 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.370932102203369 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.371057510375977 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.370301723480225 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.365617752075195 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.367737293243408 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.370293617248535 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.370628833770752 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.3704962730407715 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.370964050292969 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.370872974395752 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.37193489074707 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.3698039054870605 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.364621639251709 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.989259958267212 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.989586591720581 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.989457607269287 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.989313840866089 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.9897987842559814 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.9898786544799805 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.990248203277588 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.9888205528259277 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.9888482093811035 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.988792657852173 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.988403797149658 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.9887535572052 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.9891655445098877 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.9890999794006348 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.988828659057617 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.9891767501831055 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.9891536235809326 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.9891157150268555 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.9883594512939453 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.9891726970672607 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.9888460636138916 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.9889020919799805 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.9890296459198 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.9892048835754395 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.9892704486846924 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.988804340362549 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.9889495372772217 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.988839864730835 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.989001750946045 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.9892921447753906 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.9890191555023193 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.989163875579834 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.9881536960601807 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.9889028072357178 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.988525152206421 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.989436149597168 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.98948335647583 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.989192247390747 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.9888899326324463 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.9885690212249756 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.989027738571167 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.9890570640563965 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.9894745349884033 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.9904227256774902 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.9890286922454834 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.9892079830169678 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.988530397415161 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.9888336658477783 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.9892683029174805 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.98945951461792 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.9891886711120605 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.9894843101501465 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.9893927574157715 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.9892241954803467 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.9896702766418457 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.988649368286133 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.9897477626800537 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.9896512031555176 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.9891316890716553 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.9886972904205322 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.9881975650787354 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.988278865814209 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.988563060760498 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.9895379543304443 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.9894497394561768 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.9892513751983643 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.989354372024536 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.988262891769409 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.9891200065612793 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.989325523376465 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.9892842769622803 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.9892308712005615 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.9894282817840576 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.9892523288726807 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.987931251525879 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.9885494709014893 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.9885199069976807 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.9891483783721924 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.989225149154663 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.989548921585083 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.988978385925293 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.989313840866089 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.9890308380126953 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.9890964031219482 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.9892144203186035 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.9892191886901855 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.9894723892211914 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.9888992309570312 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.9884533882141113 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.988980293273926 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.988579750061035 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.9894180297851562 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.9894254207611084 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.9893949031829834 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.988919258117676 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.988879680633545 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.989042043685913 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.9887733459472656 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.9894254207611084 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.989112377166748 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.9888577461242676 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.9885380268096924 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.9889726638793945 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.9891836643218994 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.9896981716156006 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.9893062114715576 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.989243745803833 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.9898037910461426 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.988999128341675 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.989009380340576 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.9892008304595947 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.9893577098846436 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.989319086074829 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.9888699054718018 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.9891715049743652 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.9891586303710938 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.9890296459198 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.9885408878326416 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.9893174171447754 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.9891855716705322 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.9888226985931396 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.988914728164673 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.9891855716705322 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.989671468734741 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.9894156455993652 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.9892466068267822 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.9891998767852783 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.988955020904541 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.9889097213745117 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.9881837368011475 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.988870143890381 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.988237142562866 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.989407777786255 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.9892232418060303 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.988959312438965 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.9897563457489014 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.988868474960327 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.9888534545898438 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.988537073135376 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.989206552505493 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.989356756210327 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.9894490242004395 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.9891295433044434 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.9892303943634033 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.989528179168701 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.9892919063568115 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.989149332046509 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.9892704486846924 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.9888741970062256 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.98921799659729 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.988981246948242 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.9894840717315674 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.98880672454834 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.989325523376465 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.9894766807556152 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.989553928375244 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.989331007003784 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.9888322353363037 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.9893970489501953 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.989107608795166 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.989248275756836 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.9893062114715576 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.989201784133911 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.9893617630004883 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.9887330532073975 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.988969326019287 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.9886586666107178 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.9891247749328613 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.9890761375427246 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.989189624786377 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.9892144203186035 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.9881975650787354 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.9889137744903564 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.9891319274902344 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.9891629219055176 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.9892027378082275 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.9889299869537354 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.988969326019287 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.9887330532073975 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.9891817569732666 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.988999128341675 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.9893929958343506 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.989374876022339 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.9892189502716064 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.9891698360443115 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.988983631134033 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.9890239238739014 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.9891390800476074 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.9892122745513916 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.988948345184326 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.9888322353363037 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.9887423515319824 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.9891672134399414 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.9893393516540527 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.9892256259918213 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.989337921142578 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.9892892837524414 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.989330291748047 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.989302635192871 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.989403009414673 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.9891579151153564 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.9892680644989014 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.989086866378784 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.9888813495635986 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.9889028072357178 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.989297866821289 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.988936424255371 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.9893620014190674 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.989352226257324 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.9891886711120605 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.9892687797546387 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.9893035888671875 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.9888899326324463 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.9887938499450684 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.9891417026519775 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.9893686771392822 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.9892961978912354 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.988906145095825 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.9887096881866455 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.9892475605010986 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.989457607269287 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.989461660385132 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.9895946979522705 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.989490509033203 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.9885165691375732 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.9887421131134033 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.989222526550293 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.989403486251831 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.9890494346618652 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.9894766807556152 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.988926887512207 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.988981246948242 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.989095687866211 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.9890990257263184 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.9892289638519287 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.988813877105713 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.9888675212860107 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.9890201091766357 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.989264488220215 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.9891629219055176 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.9892687797546387 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.9892332553863525 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.9892725944519043 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.9892780780792236 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.988875150680542 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.9892826080322266 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.9892654418945312 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.9891510009765625 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.9892423152923584 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.9888503551483154 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.989189624786377 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.9888744354248047 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.988682746887207 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.988732099533081 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.989354372024536 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.989346981048584 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.989440441131592 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.9893927574157715 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.988666534423828 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.9890897274017334 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.989128589630127 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.9891953468322754 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.9893593788146973 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.9888129234313965 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.9893574714660645 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.9888243675231934 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.9891157150268555 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.9889512062072754 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.988999843597412 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.989187717437744 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.9890007972717285 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.989269495010376 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.9888978004455566 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.9890239238739014 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.989375591278076 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.9889957904815674 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.989133834838867 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.9887492656707764 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.9885671138763428 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.9892354011535645 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.9889469146728516 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.989124298095703 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.988983631134033 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.988938570022583 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.989008665084839 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.9890785217285156 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.9894680976867676 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.98892879486084 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.989581346511841 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.9892544746398926 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.9894206523895264 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.988901138305664 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.9890878200531006 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.9896774291992188 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.9888594150543213 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.989075183868408 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.98915433883667 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.9888830184936523 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.9889955520629883 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.98901629447937 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.9893953800201416 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.9888975620269775 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.989729642868042 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.9893274307250977 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.9893341064453125 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.9893274307250977 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.989074945449829 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.989069700241089 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.988879442214966 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.989363670349121 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.989429235458374 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.9894800186157227 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.9893863201141357 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.988861083984375 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.988797664642334 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.9886624813079834 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.989293098449707 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.989344835281372 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.9892749786376953 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.9892332553863525 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.9891910552978516 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.9891397953033447 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.988936424255371 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.9892332553863525 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.9892208576202393 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.9889087677001953 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.989064931869507 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.9886155128479004 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.989206075668335 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.988797664642334 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.9893131256103516 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.9894802570343018 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.989400625228882 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.989354133605957 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.9890501499176025 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.988982677459717 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.9890353679656982 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.98925518989563 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.989328622817993 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.989320755004883 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.989302635192871 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.989107131958008 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.9888741970062256 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.989039421081543 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.9893078804016113 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.989363670349121 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.989313840866089 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.989330291748047 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.9889299869537354 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.9892492294311523 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.9886844158172607 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.98925518989563 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.9892446994781494 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.989201784133911 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.9889707565307617 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.9891231060028076 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.988461971282959 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.9891316890716553 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.9892756938934326 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.9894351959228516 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.989367961883545 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.9893722534179688 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.988734722137451 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.989142894744873 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.988715887069702 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.9892406463623047 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.9892940521240234 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.9893600940704346 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.989360809326172 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.9887373447418213 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.9891562461853027 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.9893147945404053 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.989366292953491 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.9893455505371094 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.9892489910125732 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.988682985305786 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.9891204833984375 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.98909330368042 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.9894819259643555 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.9893782138824463 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.9893758296966553 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.9894521236419678 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.9892423152923584 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.989129066467285 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.9885218143463135 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.9893016815185547 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.9893746376037598 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.989356756210327 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.98934268951416 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.9886624813079834 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.9887986183166504 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.9892826080322266 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.989372968673706 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.98942232131958 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.989434242248535 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.989335060119629 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.989321708679199 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.988795757293701 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.989323139190674 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.9892122745513916 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.989243745803833 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.989248275756836 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.989243745803833 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.9882888793945312 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.9884207248687744 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.988530397415161 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.9893617630004883 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.989356756210327 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.9893007278442383 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.989471197128296 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.988717555999756 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.9889183044433594 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.9884607791900635 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.9893126487731934 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.9893693923950195 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.989370346069336 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.989490032196045 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.9890732765197754 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.9888288974761963 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.988673210144043 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.989349126815796 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.989354133605957 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.989321231842041 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.9893245697021484 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.98811936378479 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.9893627166748047 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.9885494709014893 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.9893686771392822 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.9893672466278076 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.989360809326172 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.989272356033325 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.989089012145996 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.989013671875 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.9893293380737305 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.9893362522125244 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.9893863201141357 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.989257574081421 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.9890921115875244 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.988661527633667 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.989065408706665 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.9893274307250977 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.989316940307617 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.989312171936035 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.9893853664398193 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.9880311489105225 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.9887709617614746 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.9890472888946533 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.9893574714660645 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.989388942718506 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.9894111156463623 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.989398717880249 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.988504648208618 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.988600730895996 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.988816499710083 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.9893383979797363 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.9893414974212646 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.9893970489501953 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.989328384399414 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.9888129234313965 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.988917350769043 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.9889075756073 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.989337205886841 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.9893174171447754 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.9898808002471924 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.9893643856048584 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.9893455505371094 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.988438129425049 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.989210844039917 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.989331007003784 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.9894258975982666 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.9890072345733643 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.988896608352661 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.9886176586151123 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.989475727081299 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.9894845485687256 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.989161968231201 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.989293098449707 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.988750457763672 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.988602638244629 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.9887380599975586 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.98919415473938 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.98934268951416 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.9889636039733887 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.989222526550293 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.9883649349212646 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.989312171936035 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.988905191421509 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.989315986633301 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.989269733428955 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.9892523288726807 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.9893949031829834 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.988466739654541 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.9892759323120117 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.989077091217041 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.989283561706543 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.9891998767852783 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.989274024963379 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.9886484146118164 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.9887261390686035 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.989163875579834 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.989224672317505 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.989614248275757 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.9892940521240234 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.987661361694336 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.9894137382507324 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.989142894744873 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.989269495010376 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.9892876148223877 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.989215612411499 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.9893929958343506 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.9886398315429688 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.9891321659088135 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.9897193908691406 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.9893102645874023 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.989388942718506 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.989494800567627 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.9892327785491943 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.98606014251709 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.9885873794555664 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.9890239238739014 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.9897992610931396 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.989866018295288 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.9900646209716797 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.98954701423645 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.9875223636627197 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.9895410537719727 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.9891552925109863 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.989344596862793 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.9893686771392822 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.989469051361084 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.98952317237854 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.9880974292755127 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.989020586013794 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.9886622428894043 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.9893083572387695 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.989287853240967 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.9888699054718018 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.989025831222534 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.9881908893585205 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.9891300201416016 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.9883642196655273 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.989490509033203 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.989255666732788 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.989182710647583 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.9893696308135986 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.989755868911743 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.9892501831054688 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.988036870956421 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.8670127391815186 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.8670167922973633 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.867168664932251 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.8669652938842773 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.8670926094055176 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.867605686187744 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.867151975631714 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.8668947219848633 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.8668062686920166 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.8667726516723633 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.8669021129608154 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.8667140007019043 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.8669979572296143 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.86676287651062 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.866762399673462 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.8667709827423096 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.866797924041748 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.8667521476745605 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.8666634559631348 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.866442918777466 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.8667874336242676 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.867043972015381 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.8668901920318604 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.8672027587890625 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.8670194149017334 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.8665802478790283 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.8666906356811523 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.866523265838623 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.866868734359741 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.8672447204589844 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.867022752761841 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.866962194442749 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.867201805114746 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.8669803142547607 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.8668248653411865 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.866929292678833 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.8671927452087402 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.8667633533477783 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.8669729232788086 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.8667445182800293 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.8669207096099854 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.866957902908325 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.867109537124634 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.867114305496216 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.867183208465576 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.8667140007019043 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.8670449256896973 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.866605281829834 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.8670549392700195 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.8671059608459473 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.8669729232788086 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.8672289848327637 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.8668394088745117 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.8669040203094482 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.867095708847046 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.8664863109588623 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.8671257495880127 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.8671748638153076 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.867020606994629 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.8672966957092285 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.8670146465301514 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.866345167160034 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.8670074939727783 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.8671298027038574 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.8669424057006836 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.8672258853912354 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.8671085834503174 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.8671653270721436 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.8668854236602783 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.8667538166046143 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.8670730590820312 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.866971015930176 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.8669281005859375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.8668081760406494 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.866734266281128 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.8670272827148438 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.8666024208068848 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.866917371749878 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.8670432567596436 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.867030143737793 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.8669376373291016 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.8668699264526367 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.86669921875 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.8670156002044678 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.866981267929077 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.8672358989715576 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.866919994354248 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.8665575981140137 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.866607427597046 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.867175579071045 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.866893768310547 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.8669395446777344 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.866971969604492 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.867070436477661 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.8668861389160156 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.867039442062378 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.867255210876465 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.866832971572876 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.8672008514404297 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.8669052124023438 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.8666751384735107 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.866931438446045 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.866826057434082 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.8671183586120605 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.8671491146087646 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.8671178817749023 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.867034435272217 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.866884231567383 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.866901159286499 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.8668808937072754 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.8669114112854004 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.866931438446045 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.8670709133148193 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.866968870162964 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.8669142723083496 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.8668315410614014 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.8671939373016357 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.8668625354766846 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.867145538330078 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.8670687675476074 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.8665831089019775 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.8670477867126465 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.866914987564087 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.8668746948242188 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.8665854930877686 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.8667032718658447 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.8666491508483887 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.8666250705718994 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.8666863441467285 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.866941452026367 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.867124080657959 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.867203712463379 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.867032051086426 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.8667919635772705 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.866827964782715 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.867048740386963 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.8671011924743652 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.8669114112854004 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.8668222427368164 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.8669943809509277 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.8669159412384033 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.8672728538513184 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.8668999671936035 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.866879940032959 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.866971015930176 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.8670156002044678 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.8667304515838623 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.8665876388549805 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.8667845726013184 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.867292642593384 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.8671278953552246 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.866976022720337 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.866809129714966 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.8668365478515625 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.8670573234558105 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.866831064224243 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.8668503761291504 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.8669519424438477 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.8669090270996094 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.8671658039093018 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.867016553878784 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.8666462898254395 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.8669309616088867 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.8672611713409424 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.8668148517608643 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.8669114112854004 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.8668017387390137 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.866878032684326 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.867220401763916 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.867079734802246 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.866837501525879 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.8669886589050293 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.8669509887695312 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.8668007850646973 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.8667352199554443 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.867130994796753 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.8670597076416016 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.8667569160461426 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.8669676780700684 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.866913318634033 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.8670477867126465 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.8671603202819824 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.866837501525879 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.866730213165283 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.8670408725738525 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.866727113723755 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.866922616958618 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.8668394088745117 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.8669002056121826 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.866852283477783 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.8668315410614014 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.866859197616577 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.8670871257781982 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.866957426071167 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.866899251937866 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.866835594177246 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.866734266281128 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.8673980236053467 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.8668112754821777 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.8670239448547363 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.866957426071167 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.8668909072875977 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.8669309616088867 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.8669300079345703 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.866875171661377 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.866528272628784 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.8668975830078125 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.8669240474700928 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.8669769763946533 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.8667802810668945 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.8665931224823 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.8670718669891357 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.8667564392089844 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.8668699264526367 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.866770029067993 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.866856098175049 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.8670687675476074 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.8668198585510254 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.8670365810394287 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.8670175075531006 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.8670241832733154 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.867119789123535 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.867015838623047 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.867100238800049 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.866541624069214 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.866865634918213 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.8668019771575928 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.8669464588165283 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.8670976161956787 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.8671021461486816 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.867053270339966 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.8668465614318848 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.86690092086792 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.8669018745422363 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.866846799850464 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.8668315410614014 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.8667237758636475 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.8666577339172363 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.867318868637085 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.8671000003814697 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.8665852546691895 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.8668477535247803 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.867053270339966 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.8669049739837646 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.8667731285095215 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.8669071197509766 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.86676287651062 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.866442918777466 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.8671436309814453 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.8670554161071777 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.866931438446045 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.867128849029541 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.8666789531707764 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.86680006980896 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.866840362548828 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.866919755935669 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.8669896125793457 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.8670027256011963 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.8669090270996094 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.8669040203094482 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.867034912109375 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.867018461227417 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.8670291900634766 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.8669095039367676 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.867007255554199 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.8669488430023193 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.8669371604919434 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.8666892051696777 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.866647720336914 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.8667874336242676 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.8669538497924805 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.8667564392089844 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.8667709827423096 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.866901159286499 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.8670008182525635 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.867310047149658 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.8668439388275146 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.866814136505127 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.8666956424713135 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.866597890853882 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.8667941093444824 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.866978883743286 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.8668413162231445 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.86692476272583 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.8668882846832275 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.866881847381592 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.866659641265869 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.866884469985962 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.8670127391815186 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.8669960498809814 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.8668668270111084 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.866884469985962 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.866966724395752 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.8665950298309326 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.86700177192688 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.8667337894439697 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.866954803466797 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.866886854171753 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.8670308589935303 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.866755962371826 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.867039442062378 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.866835594177246 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.866539478302002 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.867145538330078 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.86712384223938 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.8672690391540527 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.8670644760131836 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.8667666912078857 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.867043972015381 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.866879940032959 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.8670523166656494 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.8670551776885986 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.867204427719116 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.8668909072875977 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.8669538497924805 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.867194414138794 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.8666107654571533 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.8670156002044678 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.8669216632843018 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.8668572902679443 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.866706609725952 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.8666112422943115 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.8667783737182617 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.866912364959717 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.866852283477783 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.8669021129608154 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.8669915199279785 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.866647720336914 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.867112636566162 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.8671019077301025 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.8669281005859375 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.866964817047119 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.866971015930176 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.866931200027466 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.866687297821045 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.867095470428467 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.8669960498809814 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.8669075965881348 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.866990566253662 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.866955041885376 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.866854190826416 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.8667290210723877 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.8670589923858643 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.8667638301849365 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.8669049739837646 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.866889238357544 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.8670589923858643 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.866976022720337 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.866912364959717 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.8667821884155273 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.866757869720459 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.8670151233673096 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.866995334625244 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.8669216632843018 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.8668010234832764 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.8669803142547607 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.8670432567596436 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.8664190769195557 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.866987705230713 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.8669779300689697 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.8670010566711426 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.866955518722534 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.866706609725952 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.8669986724853516 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.8669793605804443 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.866959571838379 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.866962194442749 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.867098569869995 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.866931200027466 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.8665478229522705 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.866999864578247 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.8666982650756836 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.8669779300689697 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.8670384883880615 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.866954803466797 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.8669638633728027 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.866706609725952 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.866779327392578 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.8665988445281982 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.8669958114624023 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.866950035095215 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.866917848587036 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.866973876953125 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.867079019546509 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.866995334625244 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.867033004760742 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.866969585418701 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.867032051086426 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.867035388946533 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.8670854568481445 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.8667314052581787 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.8671748638153076 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.8669769763946533 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.867033004760742 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.866990327835083 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.8669662475585938 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.8671324253082275 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.866642951965332 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.866832971572876 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.8667237758636475 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.8670384883880615 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.8670082092285156 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.8668737411499023 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.8669822216033936 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.8667140007019043 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.866879940032959 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.8669726848602295 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.866973876953125 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.8669960498809814 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.8670871257781982 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.8669381141662598 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.8667569160461426 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.866779327392578 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.8666467666625977 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.8669753074645996 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.8670196533203125 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.8669657707214355 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.8670027256011963 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.8666625022888184 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.866672992706299 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.866673707962036 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.866952896118164 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.866912364959717 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.8669450283050537 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.8669612407684326 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.8666653633117676 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.8666603565216064 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.8667988777160645 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.866905927658081 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.8669886589050293 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.8669209480285645 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.8669302463531494 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.8666305541992188 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.866831064224243 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.8667237758636475 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.867032051086426 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.866938591003418 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.8669049739837646 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.8667407035827637 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.8669593334198 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.866588592529297 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.8669769763946533 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.866992473602295 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.866990566253662 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.867050886154175 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.867063522338867 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.8667333126068115 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.867020606994629 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.8670220375061035 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.8669559955596924 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.866995334625244 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.86696720123291 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.8666436672210693 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.8666956424713135 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.8668606281280518 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.8669967651367188 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.867050886154175 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.8671352863311768 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.8670666217803955 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.8665802478790283 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.8670554161071777 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.86698842048645 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.8670082092285156 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.866926908493042 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.8672401905059814 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.8670403957366943 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.866525173187256 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.8667783737182617 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.8672873973846436 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.8669989109039307 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.8670506477355957 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.8670527935028076 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.867067337036133 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.866548776626587 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.8670737743377686 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.866884469985962 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.8670146465301514 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.867119789123535 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.8669698238372803 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.8663949966430664 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.8669896125793457 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.866818904876709 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.866894483566284 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.866882562637329 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.8667519092559814 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.8668718338012695 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.8668456077575684 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.866954803466797 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.867034912109375 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.866983652114868 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.8669350147247314 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.866926670074463 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.8672127723693848 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.8667821884155273 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.8672850131988525 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.866870880126953 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.8669188022613525 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.8668906688690186 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.866818428039551 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.8670101165771484 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.8665950298309326 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.8671324253082275 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.8667352199554443 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.866910457611084 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.866947889328003 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.8668580055236816 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.867016553878784 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.8664045333862305 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.866908550262451 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.8667988777160645 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.867063045501709 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.8669440746307373 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.8668596744537354 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.866786003112793 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.8663010597229004 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.86710524559021 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.8669612407684326 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.8669216632843018 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.8669464588165283 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.866957426071167 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.8668980598449707 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.8656322956085205 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.866597890853882 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.8668527603149414 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.867367744445801 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.8673739433288574 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.866809368133545 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.867016553878784 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.8669490814208984 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.8671114444732666 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.867187023162842 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.867084264755249 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.8670272827148438 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.866997003555298 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.8670194149017334 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.8663978576660156 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.8668532371520996 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.86665415763855 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.866929292678833 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.8669989109039307 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.866837501525879 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.8669850826263428 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.865818500518799 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.8666374683380127 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.8667685985565186 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.86702823638916 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.8668882846832275 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.866962194442749 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.866821527481079 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.8665425777435303 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.8673489093780518 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.866105318069458 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.811953067779541 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.812067747116089 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.812023878097534 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.8120362758636475 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.812218427658081 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.812202215194702 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.8120412826538086 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.812018632888794 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.8119757175445557 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.811859369277954 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.8117969036102295 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.811918258666992 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.812103033065796 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.8122310638427734 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.811908721923828 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.812023878097534 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.811892509460449 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.8117971420288086 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.8119747638702393 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.8118393421173096 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.812004566192627 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.811976671218872 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.8120157718658447 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.811953067779541 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.8118245601654053 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.8121347427368164 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.8120522499084473 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.8120148181915283 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.8119101524353027 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.8120365142822266 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.8117425441741943 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.8117568492889404 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.8121540546417236 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.8119711875915527 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.812000036239624 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.811941146850586 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.8121237754821777 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.8120956420898438 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.8118433952331543 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.812163829803467 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.811863899230957 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.8120720386505127 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.812004566192627 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.811872959136963 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.8120338916778564 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.81187105178833 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.8122856616973877 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.812018632888794 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.812044143676758 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.812101125717163 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.8120276927948 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.8121256828308105 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.811832904815674 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.811948537826538 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.812068223953247 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.81197452545166 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.8121230602264404 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.8121182918548584 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.811786651611328 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.8117587566375732 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.8115813732147217 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.811863899230957 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.8121345043182373 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.8119964599609375 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.812034845352173 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.8120927810668945 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.8117568492889404 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.8120133876800537 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.8119142055511475 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.811976432800293 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.8120574951171875 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.8121933937072754 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.8120884895324707 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.8119540214538574 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.8120179176330566 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.812209367752075 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.8120310306549072 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.812004566192627 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.812135696411133 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.8119568824768066 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.81164288520813 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.8121092319488525 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.8117592334747314 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.812107563018799 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.811912775039673 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.8119378089904785 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.8122520446777344 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.811957359313965 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.8120205402374268 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.8119077682495117 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.8119888305664062 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.8120040893554688 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.811908006668091 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.8119266033172607 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.811918258666992 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.8117384910583496 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.8119170665740967 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.8119559288024902 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.8120691776275635 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.8119335174560547 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.8120856285095215 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.8118064403533936 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.812035322189331 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.811746597290039 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.812074899673462 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.8120667934417725 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.811816692352295 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.8121328353881836 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.8119630813598633 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.812140703201294 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.812039852142334 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.8119747638702393 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.8119990825653076 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.811976671218872 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.8118319511413574 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.8120064735412598 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.8120226860046387 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.8120529651641846 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.8120484352111816 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.8119702339172363 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.8118464946746826 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.8118247985839844 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.811852216720581 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.812107563018799 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.8119611740112305 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.812030076980591 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.811927556991577 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.8121631145477295 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.812000036239624 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.8119306564331055 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.811715841293335 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.8117470741271973 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.812065839767456 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.8120739459991455 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.8121089935302734 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.8120639324188232 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.8119020462036133 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.8118793964385986 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.8118574619293213 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.8118975162506104 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.811980962753296 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.8120970726013184 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.812206506729126 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.8120505809783936 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.8118298053741455 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.81208872795105 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.812002182006836 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.8120548725128174 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.81180739402771 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.8119940757751465 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.812175989151001 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.8119640350341797 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.811868906021118 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.8119118213653564 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.811990261077881 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.811938762664795 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.812065839767456 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.812082529067993 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.811889410018921 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.811990261077881 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.8120357990264893 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.81199312210083 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.8120317459106445 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.8119068145751953 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.8120317459106445 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.812030076980591 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.8119702339172363 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.812089681625366 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.8118093013763428 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.81182861328125 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.811796188354492 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.8119757175445557 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.811932325363159 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.8117921352386475 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.811983823776245 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.8119313716888428 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.8120555877685547 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.812037229537964 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.8120903968811035 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.8121001720428467 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.8120710849761963 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.81196665763855 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.8118913173675537 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.8118176460266113 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.8118481636047363 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.8119068145751953 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.8119754791259766 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.811875104904175 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.8119099140167236 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.8119049072265625 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.811943769454956 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.812131881713867 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.811983823776245 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.811950922012329 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.8120601177215576 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.8120784759521484 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.8121531009674072 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.8119397163391113 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.8120980262756348 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.8120033740997314 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.8119466304779053 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.811765193939209 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.8120179176330566 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.8122012615203857 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.812054395675659 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.81227707862854 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.8120975494384766 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.8120310306549072 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.812103509902954 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.8119657039642334 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.811952590942383 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.8120064735412598 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.811948537826538 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.8120758533477783 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.8120129108428955 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.811995267868042 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.8120758533477783 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.8117847442626953 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.8120229244232178 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.812167167663574 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.811932325363159 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.8119776248931885 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.811983823776245 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.8118412494659424 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.811840295791626 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.8121092319488525 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.812023639678955 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.81183123588562 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.812039375305176 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.8116557598114014 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.8119983673095703 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.811959981918335 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.8119776248931885 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.811962366104126 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.811899423599243 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.8119406700134277 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.8119540214538574 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.812009334564209 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.812039852142334 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.811943769454956 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.8120474815368652 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.8118276596069336 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.812000036239624 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.8119940757751465 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.8120062351226807 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.8118784427642822 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.811964988708496 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.81189227104187 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.811835765838623 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.8121190071105957 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.8117685317993164 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.812039852142334 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.8120548725128174 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.8120148181915283 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.811972141265869 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.8119289875030518 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.8120267391204834 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.8118317127227783 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.8120033740997314 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.812138557434082 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.8119492530822754 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.8119912147521973 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.811814069747925 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.8119149208068848 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.81201171875 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.811953067779541 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.8119397163391113 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.8121907711029053 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.811936616897583 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.8120219707489014 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.812148332595825 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.8118813037872314 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.811962127685547 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.811882495880127 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.8119313716888428 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.8119425773620605 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.811889410018921 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.8118956089019775 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.8119611740112305 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.8118584156036377 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.8121025562286377 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.8119466304779053 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.8119404315948486 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.8118584156036377 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.812068223953247 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.8119418621063232 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.812082052230835 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.811946153640747 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.8119993209838867 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.8120903968811035 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.8117775917053223 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.811931610107422 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.8118813037872314 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.8119139671325684 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.8119120597839355 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.8118667602539062 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.81195330619812 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.8119561672210693 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.8121285438537598 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.8120357990264893 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.8119571208953857 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.8119053840637207 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.8118855953216553 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.811826705932617 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.812061071395874 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.8120176792144775 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.8120710849761963 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.8120837211608887 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.8119447231292725 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.8118317127227783 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.811842203140259 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.8120055198669434 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.812004327774048 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.812018632888794 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.8119490146636963 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.8117969036102295 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.811997413635254 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.8119304180145264 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.8119819164276123 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.8120739459991455 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.811976909637451 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.8118603229522705 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.8119211196899414 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.8119776248931885 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.8116812705993652 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.8119912147521973 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.811985731124878 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.812183380126953 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.8119568824768066 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.8118700981140137 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.811918020248413 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.8119499683380127 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.812065839767456 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.812034845352173 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.8119711875915527 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.8119611740112305 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.8119964599609375 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.812067747116089 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.8120648860931396 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.8119990825653076 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.811889171600342 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.8121747970581055 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.811988592147827 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.811986207962036 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.8120148181915283 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.81192946434021 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.8120229244232178 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.811805009841919 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.8120052814483643 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.8119826316833496 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.8119993209838867 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.812070846557617 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.811978578567505 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.812049388885498 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.811901569366455 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.811918258666992 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.8117897510528564 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.8120205402374268 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.812091588973999 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.8120803833007812 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.8121068477630615 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.811980962753296 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.8119633197784424 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.8120319843292236 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.8119921684265137 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.81209135055542 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.811983585357666 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.8120195865631104 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.8119266033172607 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.8121531009674072 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.811922550201416 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.8120574951171875 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.8119728565216064 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.812039375305176 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.812119245529175 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.8119256496429443 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.81182861328125 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.811680793762207 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.812034845352173 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.8119161128997803 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.811932325363159 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.8118362426757812 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.8118932247161865 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.811666488647461 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.8120129108428955 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.812072992324829 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.812044620513916 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.8120760917663574 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.811943769454956 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.8121719360351562 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.811946153640747 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.8119940757751465 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.8119921684265137 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.8119583129882812 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.8118112087249756 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.8120005130767822 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.81203293800354 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.8119540214538574 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.8121912479400635 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.812061071395874 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.8117458820343018 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.8120298385620117 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.812049388885498 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.812062978744507 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.812103033065796 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.8119003772735596 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.8119139671325684 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.8119444847106934 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.8120412826538086 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.8119983673095703 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.8120594024658203 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.8120205402374268 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.8119711875915527 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.8119795322418213 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.811739444732666 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.8120994567871094 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.812002182006836 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.8120267391204834 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.8120384216308594 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.8120055198669434 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.8116984367370605 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.8117785453796387 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.812039375305176 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.8120741844177246 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.812047243118286 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.8117620944976807 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.8117940425872803 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.812026023864746 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.8120760917663574 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.8119256496429443 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.8119728565216064 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.81164288520813 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.8120381832122803 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.811826705932617 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.8120267391204834 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.8120229244232178 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.8120474815368652 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.812044143676758 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.8120317459106445 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.8118598461151123 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.8121604919433594 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.812023878097534 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.8120362758636475 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.811946153640747 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.8117077350616455 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.811798334121704 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.8119797706604004 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.8120431900024414 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.812101125717163 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.812138557434082 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.8120710849761963 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.8118412494659424 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.812044620513916 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.811950206756592 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.8120052814483643 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.812025785446167 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.8119404315948486 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.8119845390319824 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.8118460178375244 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.811918020248413 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.8119702339172363 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.812093734741211 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.8121750354766846 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.8119852542877197 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.8120765686035156 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.812150001525879 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.811652183532715 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.8120594024658203 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.811958074569702 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.811985731124878 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.811957359313965 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.8120975494384766 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.811612129211426 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.8118059635162354 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.812114953994751 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.8119559288024902 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.8119239807128906 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.811992645263672 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.8118975162506104 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.8119876384735107 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.811938762664795 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.811852216720581 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.8120288848876953 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.8121237754821777 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.8120594024658203 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.811544895172119 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.8118984699249268 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.8119912147521973 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.8119819164276123 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.811951160430908 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.8121016025543213 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.8120431900024414 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.811936616897583 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.811927556991577 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.8120102882385254 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.8118855953216553 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.8120152950286865 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.8118720054626465 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.8119418621063232 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.811483144760132 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.81193470954895 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.8120338916778564 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.811954975128174 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.8117940425872803 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.81189227104187 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.811387777328491 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.811798095703125 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.8117294311523438 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.8121025562286377 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.8122661113739014 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.812173843383789 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.8120055198669434 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.8114922046661377 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.8120813369750977 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.8118834495544434 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.811973810195923 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.812002420425415 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.812131404876709 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.812042236328125 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.811657667160034 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.811825752258301 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.8120081424713135 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.8119354248046875 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.811699390411377 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.8119394779205322 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.81173038482666 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.8116745948791504 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.812105178833008 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.8120031356811523 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.8120102882385254 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.8119466304779053 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.812030792236328 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.812103033065796 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.812028408050537 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.8117189407348633 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7788569927215576 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7788515090942383 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7789053916931152 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7788496017456055 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.778892755508423 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.778860330581665 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.778761148452759 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.778714418411255 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7788546085357666 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7787883281707764 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7785069942474365 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.778850555419922 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7787721157073975 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.778820037841797 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.778681993484497 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7788171768188477 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.7787508964538574 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.77873158454895 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7789013385772705 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.7787539958953857 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7787132263183594 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.778726577758789 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.778618574142456 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.7784905433654785 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.77874493598938 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7786648273468018 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7786591053009033 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.778801918029785 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.77899432182312 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.778766632080078 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.7788162231445312 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7786829471588135 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7788567543029785 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7786824703216553 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7785558700561523 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.7789838314056396 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.7789037227630615 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.7788302898406982 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7789392471313477 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7790157794952393 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.7785587310791016 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.778400421142578 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7786033153533936 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.778892755508423 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7785966396331787 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7789058685302734 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.778653621673584 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.7786612510681152 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.778815269470215 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.778920888900757 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7787883281707764 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7786970138549805 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.778644561767578 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.778822183609009 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7787559032440186 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7788407802581787 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.7790017127990723 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.778625011444092 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.778693914413452 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.7786378860473633 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.778810977935791 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.77893328666687 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.778764486312866 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7787044048309326 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7785332202911377 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.778604507446289 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.7788009643554688 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7788004875183105 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.7787091732025146 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.778820514678955 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7788045406341553 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7790114879608154 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7785675525665283 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.778764009475708 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7787692546844482 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.778848648071289 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7788569927215576 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7786316871643066 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.778681993484497 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.778670310974121 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.778895854949951 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.778808116912842 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.7788829803466797 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.779042959213257 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.778642177581787 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.778795003890991 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7788515090942383 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7789573669433594 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.7787153720855713 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.778670310974121 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7787399291992188 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7788212299346924 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7789251804351807 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7788610458374023 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7787561416625977 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.778777837753296 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.779005527496338 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7786693572998047 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.778872013092041 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.778799533843994 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7788174152374268 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.778820037841797 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7788522243499756 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7788684368133545 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.77880597114563 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.778775215148926 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.7788515090942383 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.778984785079956 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.778782844543457 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7789249420166016 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.778827428817749 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7785487174987793 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.778865098953247 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.778604507446289 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7786495685577393 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.7788891792297363 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7789080142974854 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.7789089679718018 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.778848886489868 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7787277698516846 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7787609100341797 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.778838634490967 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.778801918029785 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.7786495685577393 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.778679132461548 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7787837982177734 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.7787656784057617 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7788162231445312 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7788679599761963 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7787415981292725 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7787909507751465 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7785935401916504 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.7787487506866455 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.7787399291992188 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.778773069381714 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.7789435386657715 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.778796434402466 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7786169052124023 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7788140773773193 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7787134647369385 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7787649631500244 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7788124084472656 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.778897285461426 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.778655767440796 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.778895378112793 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7787890434265137 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7789053916931152 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.7788140773773193 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.778994560241699 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.7787346839904785 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.7788426876068115 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.778846502304077 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7788333892822266 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7788751125335693 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7787439823150635 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.7789392471313477 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.7788233757019043 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7786521911621094 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.77888560295105 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.778801202774048 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.77874493598938 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.7788028717041016 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7787318229675293 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.778663158416748 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.778749942779541 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7787365913391113 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7787187099456787 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7787766456604004 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.778703451156616 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.7789385318756104 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.778743028640747 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.778750419616699 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.778801202774048 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.7788383960723877 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7788119316101074 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.7785496711730957 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.7787539958953857 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.778663158416748 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.778803586959839 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.778757095336914 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.778687000274658 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.778872489929199 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7787559032440186 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.77878999710083 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.778822422027588 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.778944253921509 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7789130210876465 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.778846502304077 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.778843402862549 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7788619995117188 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7788774967193604 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.7789242267608643 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.778902769088745 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.7786293029785156 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7788262367248535 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.778881072998047 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7788071632385254 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7789111137390137 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.778921127319336 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.778857707977295 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.778954029083252 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.778743028640747 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7788424491882324 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.77868914604187 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.778803586959839 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7786757946014404 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.778670310974121 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.77895450592041 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7789466381073 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7787978649139404 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7788312435150146 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7788164615631104 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.778658628463745 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.778761625289917 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7787559032440186 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7788326740264893 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.778864860534668 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7787816524505615 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.778895854949951 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.778623580932617 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.778773784637451 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7786340713500977 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7788233757019043 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7788071632385254 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.7789371013641357 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.778762102127075 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.778897285461426 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.778902769088745 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.778799057006836 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.7788004875183105 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.778740882873535 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.778691053390503 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.7787866592407227 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.7785513401031494 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.7787692546844482 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7787253856658936 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.778838634490967 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.7788872718811035 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7788333892822266 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7785911560058594 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7787747383117676 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7787020206451416 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7787044048309326 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.7787375450134277 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.778801202774048 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.778872013092041 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.778866767883301 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7786717414855957 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7788021564483643 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.7788171768188477 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.778866767883301 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7785446643829346 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.7786717414855957 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.77880597114563 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.778782844543457 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.7787487506866455 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.77868914604187 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7787697315216064 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.77873158454895 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7789666652679443 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7788610458374023 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7789313793182373 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.7788476943969727 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.7787256240844727 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7785842418670654 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.778740882873535 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.7788476943969727 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.77884578704834 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.778852939605713 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.778815984725952 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.778855800628662 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7787978649139404 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.778799057006836 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7788619995117188 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7788357734680176 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7786879539489746 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.77874493598938 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7786905765533447 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7788150310516357 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7787349224090576 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7788333892822266 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7787253856658936 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.778698205947876 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7787981033325195 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.778737783432007 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7788124084472656 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7787513732910156 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7787153720855713 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.778865098953247 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.778756856918335 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7787652015686035 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7787771224975586 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7786941528320312 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.77864408493042 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.778764486312866 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7787671089172363 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.7788264751434326 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.778696298599243 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7786431312561035 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7787020206451416 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7788243293762207 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7788522243499756 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.77866268157959 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7788496017456055 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.778414487838745 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.778841495513916 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7788245677948 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7788100242614746 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7788870334625244 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.778730630874634 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.778993606567383 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.778714418411255 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7787716388702393 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7787539958953857 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7787134647369385 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.7787253856658936 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7788071632385254 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.7786593437194824 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7787721157073975 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7787609100341797 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.7787747383117676 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7787399291992188 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7786591053009033 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.778780221939087 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.7788643836975098 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.778796434402466 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7788569927215576 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.778759479522705 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.778958559036255 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.7787206172943115 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7785544395446777 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.778843641281128 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7788352966308594 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7788243293762207 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.7786552906036377 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.778730630874634 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.778820276260376 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7788000106811523 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7788150310516357 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.77882719039917 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.778879165649414 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7787227630615234 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7786004543304443 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.77878737449646 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7788093090057373 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.778630018234253 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.778768539428711 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.778787851333618 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7788476943969727 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7786002159118652 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7786953449249268 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.778782844543457 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.778838634490967 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.778749942779541 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7786738872528076 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.778701066970825 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.778590202331543 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.778759479522705 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7787907123565674 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7787928581237793 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.778747797012329 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7788314819335938 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.778787851333618 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.778700113296509 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.778947591781616 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.7787561416625977 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.778771162033081 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.778780937194824 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7788286209106445 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7786848545074463 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7788000106811523 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.778757095336914 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7788052558898926 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.778820514678955 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.778625726699829 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.77864408493042 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.778822422027588 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.778761625289917 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7787423133850098 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7787981033325195 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.7785487174987793 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7787225246429443 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.778674840927124 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.778809070587158 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7788100242614746 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.778762102127075 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7785167694091797 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.778796434402466 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.7784907817840576 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7787325382232666 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.7788281440734863 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7787909507751465 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7788658142089844 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7787954807281494 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.7787041664123535 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.778773069381714 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7787747383117676 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.778820514678955 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.778749942779541 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.778629779815674 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.778740644454956 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.778721809387207 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7788050174713135 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.778773784637451 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.778735637664795 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7785487174987793 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.7785468101501465 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7788586616516113 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.778866767883301 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7788617610931396 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7787909507751465 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.7785801887512207 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.778791904449463 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.778590202331543 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7787044048309326 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7787041664123535 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7784793376922607 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7786097526550293 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.778527021408081 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.778764009475708 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.7787554264068604 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.778848886489868 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.7787487506866455 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.778724431991577 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7786784172058105 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.778712511062622 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7787325382232666 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7786707878112793 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7784423828125 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.7786951065063477 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.7787458896636963 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.7787091732025146 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.7788233757019043 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.7786824703216553 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.778799533843994 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7786810398101807 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.778766632080078 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.7786829471588135 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.77878999710083 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.7787423133850098 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.778615951538086 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7787721157073975 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7787091732025146 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7787671089172363 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.778799533843994 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7788405418395996 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7788679599761963 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7784459590911865 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7787163257598877 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.778963565826416 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.778806209564209 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7787439823150635 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7787346839904785 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.778825283050537 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.778806209564209 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.778815984725952 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.778740882873535 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.7788326740264893 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.7787437438964844 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.7788171768188477 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.778773307800293 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.778698205947876 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7786779403686523 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7786593437194824 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7787206172943115 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7787134647369385 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.778733730316162 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.77873158454895 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.778473138809204 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.778780221939087 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7786571979522705 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7786991596221924 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7787489891052246 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.778804063796997 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.778677463531494 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7788352966308594 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.7787277698516846 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.7790026664733887 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7587106227874756 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7588627338409424 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.758840560913086 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.758925199508667 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.75884747505188 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.758901834487915 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.758840560913086 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7588655948638916 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.758852005004883 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.7588412761688232 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7588279247283936 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.7588601112365723 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.758765935897827 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7588984966278076 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.7588279247283936 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7587950229644775 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.7587051391601562 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7589309215545654 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.758718967437744 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.7587435245513916 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.75888729095459 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7587695121765137 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.7589104175567627 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.7588951587677 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.7586288452148438 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7587363719940186 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.7588679790496826 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.7587642669677734 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.758671283721924 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7587857246398926 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7587740421295166 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.758713960647583 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7588202953338623 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.758861780166626 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.758666515350342 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.7588205337524414 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.7586934566497803 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7588746547698975 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7588584423065186 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.758850336074829 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7589075565338135 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7588160037994385 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.758845806121826 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7588932514190674 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7588350772857666 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7588672637939453 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.758913516998291 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7588162422180176 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.75869083404541 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.758758544921875 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.758735179901123 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7587404251098633 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7588484287261963 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.758755683898926 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.758591651916504 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7586774826049805 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.758845806121826 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.758667469024658 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.758753538131714 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.758822441101074 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7587597370147705 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.758904457092285 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7586240768432617 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.758981227874756 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.758784770965576 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.758882761001587 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.7588119506835938 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.7587239742279053 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7586398124694824 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.758680582046509 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.758802890777588 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.758790969848633 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.7588515281677246 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7588233947753906 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7586686611175537 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.758671760559082 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.758824586868286 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7588624954223633 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.7587037086486816 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.7586286067962646 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.7585296630859375 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7588608264923096 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7587695121765137 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.758850336074829 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7587668895721436 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.7588751316070557 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7586727142333984 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7588181495666504 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.758835792541504 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7587032318115234 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.7588279247283936 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.758730173110962 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7587838172912598 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.7587618827819824 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7587859630584717 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7587363719940186 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.758742094039917 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7586960792541504 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.758917808532715 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.758779525756836 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.758742570877075 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.758726119995117 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7587575912475586 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7586781978607178 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7588555812835693 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7588601112365723 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7588484287261963 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.7586662769317627 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.758897304534912 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.758810043334961 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.75874662399292 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.758774518966675 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7588000297546387 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7587006092071533 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.7585248947143555 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.758824586868286 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.7587382793426514 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.758826971054077 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7587146759033203 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7589049339294434 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7587990760803223 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7587926387786865 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.758850574493408 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.7587695121765137 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.758786678314209 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.758819341659546 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.7587592601776123 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7588980197906494 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7589752674102783 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.758784532546997 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7586934566497803 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.758728265762329 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.75876522064209 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.758845806121826 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.7588438987731934 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7587835788726807 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7587058544158936 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.758793830871582 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.7587249279022217 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.7586405277252197 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.758779764175415 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.758821487426758 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.758723020553589 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.758694887161255 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7587404251098633 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.758582353591919 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7588112354278564 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.758795976638794 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7587344646453857 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.758789300918579 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.758723020553589 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7587006092071533 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7587077617645264 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.758680582046509 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.75876522064209 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7588555812835693 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7587344646453857 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.758831024169922 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.758746385574341 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.758681535720825 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.7587435245513916 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.7587735652923584 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.758852005004883 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7588083744049072 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.758821487426758 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.758685350418091 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7588438987731934 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.7588019371032715 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.75879168510437 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7588000297546387 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.758948802947998 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.758718967437744 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.758888006210327 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.7588062286376953 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.758786916732788 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7586684226989746 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.758707284927368 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7587273120880127 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.758739471435547 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.7586028575897217 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7587928771972656 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7586605548858643 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.758843183517456 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.758587598800659 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.758885145187378 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7589690685272217 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.758805274963379 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.758723258972168 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.758802890777588 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.758687973022461 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.75856351852417 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7588233947753906 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7587015628814697 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.758676767349243 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.758779525756836 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.758648157119751 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7588489055633545 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.7587292194366455 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.75879168510437 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7586541175842285 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7587051391601562 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7586898803710938 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.758800983428955 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.758786916732788 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7588109970092773 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7587978839874268 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7588729858398438 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.758697986602783 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7588284015655518 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.7587523460388184 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.758871078491211 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.7587223052978516 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.7588019371032715 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.758692502975464 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.758854627609253 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7588284015655518 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.75874924659729 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.758782148361206 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.758847951889038 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7588284015655518 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.758819103240967 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.758789300918579 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7585785388946533 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.758730173110962 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.758822441101074 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.758664608001709 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.758873701095581 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.758901357650757 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.758765935897827 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.758687734603882 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.758845329284668 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.758924722671509 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.7586991786956787 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7587058544158936 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7588913440704346 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7587344646453857 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7588143348693848 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.75877046585083 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.75883150100708 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7586581707000732 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.758695125579834 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.758635997772217 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7586898803710938 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.758820056915283 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7588295936584473 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.758695602416992 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7587497234344482 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.758924722671509 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7586863040924072 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7587616443634033 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7587990760803223 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.7587785720825195 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.758850336074829 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7586863040924072 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.7587525844573975 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7587969303131104 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7588438987731934 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7588000297546387 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.758680820465088 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.758715867996216 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.7586605548858643 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7589662075042725 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7588109970092773 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.758728265762329 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.7588679790496826 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.758742094039917 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.75882625579834 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.758667469024658 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7587473392486572 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7588324546813965 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.7587413787841797 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7587990760803223 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.758761167526245 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.758784770965576 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.75872802734375 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7587571144104004 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7588253021240234 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.758817434310913 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.758833408355713 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7586705684661865 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7587320804595947 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7588565349578857 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7588751316070557 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7588372230529785 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7587740421295166 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.758695125579834 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7587339878082275 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7587554454803467 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7587709426879883 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7587127685546875 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.75882625579834 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.758580207824707 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7588512897491455 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7586677074432373 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.758765697479248 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7588376998901367 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.7586405277252197 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7586827278137207 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7587554454803467 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7588319778442383 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7588162422180176 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7587106227874756 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.7588648796081543 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7587273120880127 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.7590277194976807 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7587673664093018 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7587811946868896 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7587485313415527 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7587733268737793 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.758805274963379 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.758598804473877 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.758692502975464 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7586469650268555 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.758866548538208 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7586798667907715 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.758822202682495 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7587897777557373 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7587740421295166 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7587811946868896 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.758793830871582 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.758777141571045 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.758711338043213 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7586827278137207 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.758742094039917 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.75881028175354 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.758746385574341 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.758786678314209 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.758516788482666 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.75864315032959 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7587804794311523 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.7587852478027344 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7587249279022217 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7587530612945557 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.758800983428955 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7587451934814453 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.758817195892334 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7588324546813965 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7588679790496826 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7587764263153076 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.75874924659729 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7588119506835938 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.7587530612945557 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.75876784324646 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.758708953857422 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7587435245513916 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.758652448654175 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.7587714195251465 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.758822441101074 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.758753538131714 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7587804794311523 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.75876522064209 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.758790969848633 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7587785720825195 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.75877046585083 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7588493824005127 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7587714195251465 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.7586169242858887 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.7587642669677734 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.758525848388672 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.758788824081421 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7586209774017334 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.758772373199463 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.758589744567871 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.758807897567749 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7587969303131104 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7587997913360596 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.758774995803833 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7588376998901367 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7586119174957275 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.7588143348693848 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.758819341659546 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.7585713863372803 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7587966918945312 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.7587413787841797 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7587714195251465 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.758781671524048 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.758732318878174 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7587547302246094 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.7588133811950684 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.758838176727295 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7587473392486572 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.758774995803833 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.758695125579834 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.7589356899261475 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7588162422180176 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.758782148361206 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.758840560913086 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.758479118347168 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.758807897567749 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.758709669113159 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7587673664093018 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.758777141571045 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.758819103240967 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.7588119506835938 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.7586894035339355 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7586827278137207 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.75877046585083 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.758817434310913 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.758784532546997 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7588448524475098 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7586565017700195 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.758920192718506 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.758819103240967 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.758852958679199 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.758842706680298 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7587811946868896 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7588627338409424 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.758753538131714 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.7585761547088623 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.758662223815918 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.758763313293457 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7587313652038574 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7588698863983154 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7586944103240967 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7588071823120117 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.7588467597961426 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7587764263153076 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.758737325668335 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7587571144104004 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.758532762527466 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7587454319000244 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.7588746547698975 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7587320804595947 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.7588601112365723 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7587544918060303 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7589023113250732 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.758735418319702 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.75874924659729 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.758800983428955 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.758641242980957 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.75876784324646 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.758819341659546 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.758726119995117 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.758843183517456 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7585527896881104 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7588584423065186 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7587966918945312 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7587785720825195 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.758725881576538 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.75856614112854 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7586429119110107 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.758655071258545 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.7588882446289062 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.758915662765503 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.758843183517456 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.7587687969207764 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.758679151535034 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7588231563568115 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.758749485015869 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.758755683898926 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.758852958679199 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.7588367462158203 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.758790969848633 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.7586307525634766 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7587432861328125 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.758852005004883 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7587497234344482 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.758779525756836 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.7587339878082275 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.758721113204956 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.7585947513580322 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.758655071258545 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7589023113250732 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7588584423065186 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7587525844573975 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7585508823394775 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.758737325668335 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.758657455444336 + } + ] + } + ] +} \ No newline at end of file