| { |
| "original_ppl": 8.587663650512695, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 45318.55078125 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 32770.234375 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 26241.677734375 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 25233.29296875 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 40532.66015625 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 24755.859375 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 39858.30859375 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 35979.7734375 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 30872.341796875 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 41396.47265625 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 30217.177734375 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 36737.41015625 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 46151.6015625 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 44020.02734375 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 41509.375 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 41448.6171875 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 32672.466796875 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 40829.41015625 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 44031.02734375 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 35173.2578125 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 37020.67578125 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 40696.88671875 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 38434.48046875 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 37516.05078125 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 33054.44921875 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 41836.375 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 40387.11328125 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 38883.71484375 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 40615.078125 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 41332.328125 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 31439.48046875 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 26171.099609375 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 37759.375 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 33791.51953125 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 45892.125 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 40606.08984375 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 41710.8046875 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 33213.35546875 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 40380.52734375 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 39349.79296875 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 48099.4921875 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 41170.484375 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 40137.609375 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 40081.87890625 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 44184.2265625 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 35470.671875 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 36684.7578125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 40682.21875 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 40389.421875 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 38645.33984375 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 38889.203125 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 39584.12890625 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 42743.91015625 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 38220.65234375 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 38085.83984375 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 42263.13671875 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 39309.84765625 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 42522.203125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 32455.236328125 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 39149.34765625 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 44187.59765625 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 51876.48046875 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 37253.6796875 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 39783.87890625 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 40836.37890625 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 39196.98046875 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 35807.55859375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 42060.08203125 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 37834.78125 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 37693.13671875 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 39477.73828125 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 41272.0234375 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 50337.6953125 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 49357.25 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 41352.4765625 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 36329.01171875 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 42663.640625 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 42033.296875 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 39899.84375 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 41978.6171875 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 50268.61328125 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 46983.203125 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 40861.85546875 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 39350.2421875 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 39328.6328125 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 39986.4296875 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 44732.3515625 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 49216.28515625 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 41102.92578125 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 36161.91796875 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 37339.58203125 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 40741.54296875 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 38205.16015625 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 40853.55859375 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 41136.26171875 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 43366.70703125 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 34056.421875 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 40142.47265625 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 41001.68359375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 40494.5625 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 38222.36328125 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 47030.09765625 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 39042.15625 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 40254.52734375 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 36862.28125 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 39561.484375 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 38841.8359375 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 41571.8515625 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 32977.33984375 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 39082.23828125 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 33946.75390625 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 38712.953125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 40659.953125 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 37947.2734375 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 42829.640625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 34512.296875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 40499.8125 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 36887.3203125 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 41259.43359375 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 40559.91796875 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 41097.59765625 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 36430.4609375 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 31096.677734375 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 39993.02734375 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 40124.44140625 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 39634.859375 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 38946.953125 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 39919.48046875 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 38031.03515625 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 34920.74609375 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 41390.1953125 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 38896.91796875 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 40343.19140625 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 40550.2109375 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 39578.91796875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 40626.81640625 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 34853.4765625 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 41521.453125 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 39491.78515625 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 39817.546875 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 40765.25 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 40613.9921875 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 42563.74609375 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 38923.078125 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 38684.12890625 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 38998.5078125 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 40641.54296875 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 39901.74609375 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 40221.375 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 38897.734375 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 35571.3828125 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 38228.19921875 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 37628.921875 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 39738.22265625 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 40439.72265625 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 40142.28125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 45867.31640625 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 38230.09375 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 39638.98046875 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 39465.3203125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 40344.8046875 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 40187.515625 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 40016.10546875 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 40152.5 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 46553.42578125 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 40151.2421875 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 38966.71875 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 39077.0234375 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 40165.02734375 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 40497.57421875 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 33724.10546875 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 36673.62890625 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 38663.328125 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 37719.390625 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 39455.609375 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 40176.7890625 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 40229.8125 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 41128.3359375 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 39696.33203125 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 37735.4375 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 39740.26953125 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 40049.81640625 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 40076.9453125 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 39994.703125 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 35972.66796875 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 36093.5625 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 36388.89453125 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 38517.078125 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 40491.43359375 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 40611.12890625 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 41047.7734375 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 40439.21875 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 37747.09765625 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 39697.921875 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 38388.39453125 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 41740.2109375 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 263.7067565917969 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 262.56658935546875 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 258.56268310546875 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 251.14454650878906 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 240.788330078125 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 236.70620727539062 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 240.2476806640625 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 254.30628967285156 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 256.2310791015625 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 259.8036193847656 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 243.26284790039062 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 260.27069091796875 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 243.23257446289062 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 251.16226196289062 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 255.42364501953125 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 245.4304656982422 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 254.32606506347656 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 263.7404479980469 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 253.3819122314453 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 253.93756103515625 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 252.70596313476562 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 260.4287414550781 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 256.6207275390625 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 248.2155303955078 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 243.39651489257812 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 256.2838439941406 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 257.07867431640625 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 260.4221496582031 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 252.46218872070312 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 258.88775634765625 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 249.13807678222656 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 257.9864501953125 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 249.3256072998047 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 242.12123107910156 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 251.05892944335938 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 250.94667053222656 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 265.34222412109375 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 253.54844665527344 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 251.27786254882812 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 255.88015747070312 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 251.31129455566406 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 262.16961669921875 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 252.6807861328125 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 258.6461486816406 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 253.02041625976562 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 250.1332244873047 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 257.13006591796875 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 262.6382141113281 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 257.8712158203125 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 255.8255157470703 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 260.92022705078125 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 252.57861328125 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 254.30738830566406 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 247.2131805419922 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 263.01580810546875 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 254.02682495117188 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 258.7832336425781 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 265.32501220703125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 261.6599426269531 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 245.06033325195312 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 257.026611328125 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 252.7449951171875 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 254.95550537109375 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 261.2465515136719 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 257.87933349609375 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 280.19158935546875 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 255.2540283203125 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 254.2615509033203 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 261.0050964355469 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 253.45269775390625 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 259.2049560546875 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 262.0826416015625 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 249.02749633789062 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 248.89170837402344 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 258.59857177734375 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 258.2031555175781 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 261.2415466308594 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 258.2111511230469 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 261.37921142578125 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 262.96942138671875 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 247.08685302734375 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 254.54347229003906 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 263.76019287109375 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 257.3077697753906 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 260.0513916015625 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 263.9483947753906 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 266.9032897949219 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 254.34498596191406 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 251.95635986328125 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 266.9073791503906 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 267.1764221191406 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 265.3748474121094 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 251.6942138671875 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 251.74822998046875 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 249.36532592773438 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 260.85540771484375 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 245.38165283203125 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 243.8300323486328 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 257.51129150390625 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 256.5939636230469 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 261.3288879394531 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 251.8917236328125 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 262.61376953125 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 257.76953125 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 263.3850402832031 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 263.33782958984375 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 261.03546142578125 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 265.83929443359375 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 264.8345642089844 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 257.7469177246094 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 261.10791015625 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 259.2467346191406 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 260.5517272949219 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 262.7996826171875 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 261.6365966796875 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 265.89544677734375 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 259.0224609375 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 263.78509521484375 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 260.838623046875 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 262.7560729980469 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 259.19085693359375 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 264.9053039550781 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 262.5563049316406 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 260.2261657714844 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 263.7578125 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 262.3126525878906 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 261.33135986328125 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 261.2516174316406 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 264.1031494140625 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 264.0303649902344 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 261.775634765625 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 260.5047302246094 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 259.3966064453125 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 263.4461975097656 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 262.43878173828125 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 262.5832214355469 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 263.82421875 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 261.7887268066406 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 263.3611755371094 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 263.7550354003906 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 261.74542236328125 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 261.86724853515625 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 260.93963623046875 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 260.1814880371094 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 260.3692626953125 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 263.2541809082031 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 260.69586181640625 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 261.24591064453125 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 262.4700622558594 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 262.78863525390625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 263.91998291015625 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 260.5673828125 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 264.1479797363281 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 263.1037292480469 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 260.8396301269531 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 259.05816650390625 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 261.3568115234375 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 261.6938781738281 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 261.41973876953125 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 262.37432861328125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 262.9941101074219 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 261.8000793457031 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 262.109619140625 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 261.3469543457031 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 261.8792419433594 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 261.5812072753906 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 263.0027770996094 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 263.6451416015625 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 260.93939208984375 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 259.5495300292969 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 261.5885925292969 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 261.9277038574219 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 260.8498229980469 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 263.7654724121094 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 261.35443115234375 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 262.9245300292969 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 263.6799621582031 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 263.1349792480469 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 262.3055419921875 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 262.5630798339844 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 260.3463134765625 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 262.1006164550781 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 261.9360656738281 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 261.4818420410156 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 265.6753234863281 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 265.9984130859375 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 262.0113830566406 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 264.6549072265625 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 262.9389343261719 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 265.06085205078125 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 263.7145690917969 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 263.9374694824219 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 260.6026611328125 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 253.15980529785156 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 255.02554321289062 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 271.776123046875 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 16.182289123535156 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 16.199119567871094 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 16.13261604309082 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 16.151628494262695 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 16.096834182739258 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 16.070838928222656 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 16.054658889770508 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 16.115259170532227 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 16.145849227905273 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 16.186464309692383 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 16.134389877319336 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 16.030492782592773 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 16.027122497558594 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 16.04235076904297 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 15.985359191894531 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 16.06905746459961 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 16.09482765197754 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 16.098539352416992 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 16.10824203491211 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 16.10410499572754 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 16.132251739501953 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 16.133712768554688 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 16.14194679260254 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 16.135866165161133 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 16.090940475463867 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 16.11913299560547 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 16.166671752929688 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 16.119836807250977 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 16.094602584838867 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 16.145063400268555 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 16.051210403442383 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 16.105886459350586 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 16.080448150634766 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 16.119766235351562 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 16.072551727294922 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 16.142824172973633 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 16.076557159423828 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 16.171627044677734 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 16.101207733154297 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 16.084884643554688 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 16.11320686340332 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 16.130489349365234 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 16.194292068481445 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 16.16667938232422 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 16.048845291137695 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 16.0606632232666 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 16.096158981323242 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 16.122337341308594 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 16.10666275024414 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 16.05306625366211 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 16.089521408081055 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 16.132165908813477 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 16.1407527923584 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 16.031047821044922 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 16.04306411743164 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 16.09600257873535 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 16.21443748474121 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 16.228626251220703 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 16.16290283203125 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 16.2089786529541 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 16.059120178222656 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 16.12525177001953 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 16.08757972717285 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 16.162841796875 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 16.169111251831055 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 16.157127380371094 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 16.13817024230957 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 16.0966739654541 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 16.167713165283203 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 16.142194747924805 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 16.163650512695312 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 16.160144805908203 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 16.15208625793457 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 16.161039352416992 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 16.138696670532227 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 16.161619186401367 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 16.114850997924805 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 16.146825790405273 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 16.160398483276367 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 16.12069320678711 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 16.189817428588867 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 16.11631202697754 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 16.144807815551758 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 16.14809799194336 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 16.17047691345215 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 16.17631721496582 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 16.179927825927734 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 16.11472511291504 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 16.137556076049805 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 16.192331314086914 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 16.123172760009766 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 16.144062042236328 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 16.1427001953125 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 16.161436080932617 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 16.055217742919922 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 16.137039184570312 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 16.154443740844727 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 16.080881118774414 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 16.150999069213867 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 16.171466827392578 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 16.165376663208008 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 16.11550521850586 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 16.13876724243164 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 16.129789352416992 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 16.116323471069336 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 16.19470977783203 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 16.17501449584961 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 16.140132904052734 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 16.17074203491211 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 16.13062286376953 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 16.155427932739258 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 16.10770606994629 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 16.151315689086914 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 16.19598388671875 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 16.14668083190918 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 16.1505069732666 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 16.138120651245117 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 16.145780563354492 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 16.14685821533203 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 16.17792510986328 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 16.168537139892578 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 16.22621726989746 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 16.16436004638672 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 16.14996337890625 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 16.164451599121094 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 16.14366340637207 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 16.181772232055664 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 16.18373680114746 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 16.17943572998047 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 16.178682327270508 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 16.13994789123535 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 16.166955947875977 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 16.15418815612793 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 16.197458267211914 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 16.180383682250977 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 16.171552658081055 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 16.186016082763672 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 16.15334129333496 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 16.164335250854492 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 16.144878387451172 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 16.17287254333496 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 16.168498992919922 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 16.172380447387695 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 16.172847747802734 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 16.126380920410156 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 16.161169052124023 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 16.148582458496094 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 16.18887710571289 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 16.185728073120117 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 16.180723190307617 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 16.191211700439453 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 16.13639259338379 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 16.158010482788086 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 16.151376724243164 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 16.167049407958984 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 16.18096923828125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 16.17455291748047 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 16.177444458007812 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 16.149581909179688 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 16.17035675048828 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 16.1458683013916 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 16.168468475341797 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 16.169401168823242 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 16.190879821777344 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 16.17108154296875 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 16.162063598632812 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 16.155414581298828 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 16.166248321533203 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 16.180477142333984 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 16.174243927001953 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 16.1884708404541 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 16.179096221923828 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 16.164382934570312 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 16.180097579956055 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 16.182165145874023 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 16.17144775390625 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 16.187297821044922 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 16.185747146606445 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 16.180145263671875 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 16.18990707397461 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 16.20075798034668 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 16.173742294311523 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 16.192447662353516 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 16.188579559326172 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 16.224258422851562 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 16.189315795898438 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 16.125398635864258 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 16.192232131958008 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 16.139598846435547 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 16.19194984436035 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 16.195634841918945 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 16.209991455078125 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 16.174327850341797 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 16.037893295288086 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 16.101295471191406 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 16.03105926513672 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.984415054321289 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.988714218139648 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.983264923095703 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.970619201660156 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.973557472229004 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.980630874633789 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.974761009216309 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.981932640075684 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.982094764709473 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 11.008584976196289 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.988651275634766 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.970258712768555 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.944421768188477 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.968194961547852 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.968851089477539 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.957413673400879 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.987578392028809 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.98246955871582 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.963944435119629 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.9833984375 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.972981452941895 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.979026794433594 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.975687026977539 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.97874927520752 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.97144889831543 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.968448638916016 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.972084999084473 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.969337463378906 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.97646427154541 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.97782039642334 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.98508071899414 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.984040260314941 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.963147163391113 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.965315818786621 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.972879409790039 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.990500450134277 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.975519180297852 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.98730182647705 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.979620933532715 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.974774360656738 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.968660354614258 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.969916343688965 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.984514236450195 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.980380058288574 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.976686477661133 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.98461627960205 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.968835830688477 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.973164558410645 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.977307319641113 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.978353500366211 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.973580360412598 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.985971450805664 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.98301887512207 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.969369888305664 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.983969688415527 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.96533203125 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.995275497436523 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.986479759216309 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 11.002350807189941 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.981280326843262 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.966469764709473 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.986130714416504 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.97494125366211 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.986114501953125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.973677635192871 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.99240779876709 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.99565601348877 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.97991943359375 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.98306655883789 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.984082221984863 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.987988471984863 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.984155654907227 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.98663330078125 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.97514533996582 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.97681999206543 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.977909088134766 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.982823371887207 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.982851028442383 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.982576370239258 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.980956077575684 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.976089477539062 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.97913932800293 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.982275009155273 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.985816955566406 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.984737396240234 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.985235214233398 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.984536170959473 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.97893238067627 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.97628116607666 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.980783462524414 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.984598159790039 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.984025001525879 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.971258163452148 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.98646068572998 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.982165336608887 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.98747730255127 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.972748756408691 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.978584289550781 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.98975658416748 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.986358642578125 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.983260154724121 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.974690437316895 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.974082946777344 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.978429794311523 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.97697925567627 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.989300727844238 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.98685073852539 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.984241485595703 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.986099243164062 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.979244232177734 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.986520767211914 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.980846405029297 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.983184814453125 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.984831809997559 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.985968589782715 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.982421875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.980029106140137 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.987120628356934 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.979412078857422 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.985198974609375 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.959378242492676 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.988347053527832 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.986235618591309 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.980927467346191 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.984569549560547 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.974442481994629 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.985679626464844 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.985572814941406 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.984551429748535 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.983359336853027 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.976314544677734 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.98386287689209 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.980036735534668 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.985884666442871 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.990401268005371 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.984082221984863 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.98348331451416 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.980050086975098 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.988699913024902 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.973612785339355 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.986513137817383 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.988492965698242 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.984596252441406 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.98643684387207 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.975875854492188 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.981901168823242 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.977019309997559 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.988081932067871 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.986775398254395 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.986154556274414 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.986015319824219 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.977940559387207 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.98366928100586 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.977380752563477 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.986462593078613 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.98283576965332 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.98786735534668 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.985541343688965 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.980447769165039 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.976503372192383 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.982765197753906 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.981094360351562 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.990788459777832 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.98523998260498 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.983285903930664 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.978338241577148 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.984087944030762 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.981380462646484 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.984977722167969 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.987424850463867 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.98242473602295 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.982975006103516 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.976686477661133 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.983516693115234 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.982501029968262 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.987969398498535 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.986465454101562 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.992785453796387 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.990128517150879 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.98015022277832 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.975202560424805 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.98741626739502 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.986727714538574 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.982298851013184 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.99670124053955 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.988021850585938 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.972393035888672 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.97407341003418 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.984721183776855 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.99268627166748 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.986979484558105 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.985062599182129 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.98464298248291 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.954378128051758 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.967020988464355 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.954277038574219 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 9.88216781616211 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 9.88330078125 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 9.883273124694824 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 9.874960899353027 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 9.88416862487793 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 9.877644538879395 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 9.880125045776367 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 9.88011360168457 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 9.881988525390625 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 9.883662223815918 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 9.878096580505371 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 9.877851486206055 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 9.869367599487305 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 9.878125190734863 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 9.88082504272461 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 9.874390602111816 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 9.88204288482666 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 9.879413604736328 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 9.877572059631348 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 9.877838134765625 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 9.879305839538574 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 9.880829811096191 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 9.881163597106934 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 9.877899169921875 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 9.881484985351562 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 9.875215530395508 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 9.878742218017578 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 9.880982398986816 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 9.881379127502441 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 9.880195617675781 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 9.877213478088379 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 9.883781433105469 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 9.876114845275879 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 9.877734184265137 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 9.87864875793457 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 9.88049030303955 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 9.879842758178711 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 9.881174087524414 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 9.880339622497559 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 9.874235153198242 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 9.88671588897705 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 9.88151741027832 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 9.883284568786621 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 9.882912635803223 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 9.876712799072266 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 9.881458282470703 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 9.875179290771484 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 9.883904457092285 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 9.877103805541992 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 9.878926277160645 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 9.878061294555664 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 9.877320289611816 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 9.878655433654785 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 9.869226455688477 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 9.874638557434082 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 9.879171371459961 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 9.879039764404297 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 9.882384300231934 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 9.887146949768066 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 9.878179550170898 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 9.878978729248047 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 9.882987976074219 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 9.878434181213379 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 9.88260555267334 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 9.878474235534668 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 9.881352424621582 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 9.886046409606934 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 9.880134582519531 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 9.880980491638184 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 9.88262939453125 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 9.8831148147583 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 9.882909774780273 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 9.881319999694824 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 9.881046295166016 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 9.879003524780273 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 9.883792877197266 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 9.881418228149414 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 9.882600784301758 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 9.883968353271484 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 9.882347106933594 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 9.881562232971191 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 9.878764152526855 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 9.881361961364746 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 9.88094711303711 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 9.884960174560547 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 9.8837251663208 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 9.880556106567383 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 9.877870559692383 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 9.881011009216309 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 9.884490966796875 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 9.884990692138672 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 9.880447387695312 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 9.878229141235352 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 9.882241249084473 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 9.879046440124512 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 9.882270812988281 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 9.883651733398438 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 9.882512092590332 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 9.881586074829102 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 9.880094528198242 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 9.884862899780273 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 9.8804292678833 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 9.879969596862793 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 9.879081726074219 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 9.880033493041992 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 9.88593578338623 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 9.880607604980469 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 9.880719184875488 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 9.883491516113281 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 9.88248062133789 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 9.881962776184082 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 9.87675952911377 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 9.883859634399414 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 9.885144233703613 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 9.881546020507812 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 9.882257461547852 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 9.881402015686035 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 9.880913734436035 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 9.881171226501465 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 9.88460922241211 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 9.882821083068848 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 9.88090991973877 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 9.88282299041748 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 9.87877368927002 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 9.879934310913086 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 9.88000774383545 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 9.882403373718262 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 9.882643699645996 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 9.883143424987793 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 9.883915901184082 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 9.879977226257324 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 9.881171226501465 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 9.882752418518066 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 9.884897232055664 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 9.885740280151367 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 9.881781578063965 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 9.8806791305542 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 9.88035774230957 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 9.880118370056152 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 9.878325462341309 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 9.882640838623047 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 9.884465217590332 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 9.88282299041748 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 9.882842063903809 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 9.88078784942627 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 9.881465911865234 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 9.880245208740234 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 9.884364128112793 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 9.88370418548584 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 9.884159088134766 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 9.883769989013672 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 9.879378318786621 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 9.87979793548584 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 9.881013870239258 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 9.883635520935059 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 9.883010864257812 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 9.88222885131836 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 9.883018493652344 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 9.879801750183105 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 9.882582664489746 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 9.880398750305176 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 9.882515907287598 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 9.882241249084473 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 9.883779525756836 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 9.88176441192627 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 9.880905151367188 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 9.881536483764648 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 9.882973670959473 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 9.88254451751709 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 9.88231372833252 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 9.882144927978516 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 9.882124900817871 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 9.878707885742188 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 9.881437301635742 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 9.881446838378906 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 9.883779525756836 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 9.8853120803833 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 9.882094383239746 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 9.883633613586426 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 9.881112098693848 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 9.884835243225098 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 9.881514549255371 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 9.883401870727539 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 9.883018493652344 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 9.884095191955566 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 9.88521671295166 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 9.878073692321777 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 9.883188247680664 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 9.879693984985352 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 9.883435249328613 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 9.883404731750488 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 9.883515357971191 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 9.881065368652344 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 9.870180130004883 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 9.869904518127441 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 9.872625350952148 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 9.405439376831055 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 9.405816078186035 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 9.401311874389648 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 9.404526710510254 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 9.399009704589844 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 9.40277099609375 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 9.399628639221191 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 9.40420150756836 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 9.407083511352539 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 9.406712532043457 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 9.405303955078125 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 9.40298843383789 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 9.401042938232422 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 9.401318550109863 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 9.403419494628906 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 9.402400970458984 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 9.404827117919922 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 9.404901504516602 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 9.402813911437988 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 9.403680801391602 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 9.402645111083984 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 9.40428638458252 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 9.40427303314209 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 9.40434455871582 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 9.402997016906738 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 9.403308868408203 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 9.404853820800781 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 9.407367706298828 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 9.403695106506348 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 9.403620719909668 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 9.404417037963867 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 9.401843070983887 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 9.401396751403809 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 9.403035163879395 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 9.4035062789917 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 9.404901504516602 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 9.405736923217773 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 9.405921936035156 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 9.405190467834473 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 9.403759956359863 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 9.402520179748535 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 9.405542373657227 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 9.405817985534668 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 9.405019760131836 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 9.404166221618652 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 9.402493476867676 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 9.403735160827637 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 9.40188980102539 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 9.405324935913086 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 9.40428638458252 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 9.401612281799316 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 9.403358459472656 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 9.403806686401367 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 9.400991439819336 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 9.404268264770508 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 9.403353691101074 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 9.406569480895996 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 9.4063720703125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 9.403474807739258 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 9.407190322875977 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 9.40300464630127 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 9.404012680053711 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 9.403522491455078 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 9.405118942260742 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 9.402484893798828 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 9.406081199645996 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 9.40512752532959 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 9.403044700622559 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 9.40645980834961 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 9.40463638305664 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 9.405288696289062 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 9.405610084533691 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 9.405909538269043 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 9.408462524414062 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 9.404212951660156 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 9.40505313873291 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 9.404683113098145 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 9.40605354309082 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 9.404773712158203 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 9.405412673950195 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 9.401592254638672 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 9.404070854187012 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 9.4044189453125 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 9.40511417388916 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 9.405210494995117 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 9.404105186462402 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 9.404779434204102 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 9.402244567871094 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 9.402815818786621 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 9.403318405151367 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 9.402447700500488 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 9.405376434326172 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 9.400717735290527 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 9.405461311340332 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 9.400538444519043 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 9.404191970825195 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 9.40478229522705 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 9.405829429626465 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 9.406780242919922 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 9.404358863830566 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 9.405253410339355 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 9.404463768005371 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 9.40306282043457 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 9.40516185760498 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 9.403308868408203 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 9.405937194824219 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 9.403142929077148 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 9.407000541687012 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 9.404455184936523 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 9.403013229370117 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 9.404523849487305 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 9.40401554107666 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 9.405174255371094 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 9.404733657836914 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 9.405928611755371 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 9.405542373657227 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 9.403051376342773 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 9.404394149780273 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 9.402941703796387 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 9.405990600585938 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 9.404026985168457 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 9.40605640411377 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 9.405116081237793 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 9.404855728149414 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 9.405356407165527 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 9.402928352355957 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 9.4052734375 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 9.405560493469238 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 9.405009269714355 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 9.405501365661621 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 9.403520584106445 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 9.405336380004883 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 9.403434753417969 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 9.40598201751709 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 9.406051635742188 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 9.405111312866211 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 9.405610084533691 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 9.40394401550293 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 9.405694961547852 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 9.403079986572266 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 9.405296325683594 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 9.405205726623535 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 9.405149459838867 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 9.405522346496582 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 9.403210639953613 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 9.404376983642578 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 9.40251350402832 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 9.405807495117188 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 9.405736923217773 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 9.405645370483398 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 9.405111312866211 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 9.402976989746094 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 9.406121253967285 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 9.403936386108398 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 9.405102729797363 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 9.404430389404297 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 9.406291007995605 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 9.40428638458252 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 9.404398918151855 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 9.403139114379883 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 9.404006958007812 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 9.404704093933105 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 9.404807090759277 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 9.405928611755371 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 9.404685974121094 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 9.404179573059082 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 9.404546737670898 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 9.403636932373047 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 9.404376983642578 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 9.404010772705078 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 9.4053316116333 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 9.405367851257324 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 9.403656959533691 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 9.404851913452148 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 9.403964042663574 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 9.404560089111328 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 9.403676986694336 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 9.406383514404297 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 9.405986785888672 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 9.402887344360352 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 9.406861305236816 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 9.403363227844238 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 9.405303001403809 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 9.405313491821289 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 9.40528678894043 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 9.404677391052246 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 9.403116226196289 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 9.402761459350586 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 9.402694702148438 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 9.405439376831055 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 9.405268669128418 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 9.407031059265137 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 9.405134201049805 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 9.399741172790527 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 9.403013229370117 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 9.399476051330566 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 9.180438041687012 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 9.180689811706543 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 9.181102752685547 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 9.179033279418945 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 9.182271957397461 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 9.178901672363281 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 9.1815824508667 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 9.180556297302246 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 9.179319381713867 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 9.182064056396484 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 9.180865287780762 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 9.178374290466309 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 9.17534351348877 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 9.178393363952637 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 9.178509712219238 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 9.177666664123535 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 9.18005657196045 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 9.179515838623047 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 9.178078651428223 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 9.180794715881348 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 9.177595138549805 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 9.180615425109863 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 9.179302215576172 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 9.177743911743164 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 9.178301811218262 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 9.179070472717285 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 9.179848670959473 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 9.178415298461914 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 9.181154251098633 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 9.179092407226562 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 9.180208206176758 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 9.180521011352539 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 9.17637825012207 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 9.17922592163086 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 9.177255630493164 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 9.180411338806152 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 9.179163932800293 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 9.18002700805664 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 9.180291175842285 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 9.178462982177734 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 9.180867195129395 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 9.17928409576416 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 9.180291175842285 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 9.17978286743164 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 9.177568435668945 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 9.178840637207031 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 9.178744316101074 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 9.18148136138916 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 9.178647994995117 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 9.180294036865234 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 9.178247451782227 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 9.178901672363281 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 9.17853832244873 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 9.177345275878906 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 9.177688598632812 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 9.177990913391113 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 9.179685592651367 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 9.17970085144043 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 9.18371295928955 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 9.178542137145996 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 9.178093910217285 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 9.18043327331543 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 9.18002700805664 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 9.180279731750488 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 9.178118705749512 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 9.17945671081543 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 9.179573059082031 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 9.179704666137695 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 9.179021835327148 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 9.179702758789062 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 9.180538177490234 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 9.17988395690918 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 9.17978572845459 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 9.179243087768555 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 9.178376197814941 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 9.180924415588379 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 9.180906295776367 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 9.180086135864258 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 9.180205345153809 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 9.181352615356445 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 9.180963516235352 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 9.179080963134766 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 9.179641723632812 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 9.179838180541992 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 9.180910110473633 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 9.180902481079102 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 9.179304122924805 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 9.180472373962402 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 9.179393768310547 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 9.180024147033691 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 9.180360794067383 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 9.180669784545898 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 9.17691707611084 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 9.180906295776367 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 9.18050765991211 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 9.179879188537598 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 9.17934799194336 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 9.177728652954102 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 9.180047988891602 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 9.179584503173828 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 9.180171012878418 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 9.17990779876709 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 9.179604530334473 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 9.179304122924805 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 9.179449081420898 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 9.180904388427734 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 9.178901672363281 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 9.179777145385742 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 9.180578231811523 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 9.179070472717285 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 9.180130958557129 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 9.178118705749512 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 9.179752349853516 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 9.180891036987305 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 9.1797456741333 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 9.179278373718262 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 9.179100036621094 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 9.180499076843262 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 9.180113792419434 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 9.180164337158203 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 9.180459976196289 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 9.17977523803711 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 9.17986011505127 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 9.179360389709473 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 9.180315971374512 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 9.17944622039795 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 9.179912567138672 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 9.180374145507812 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 9.180593490600586 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 9.179905891418457 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 9.179159164428711 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 9.180464744567871 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 9.17993450164795 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 9.180989265441895 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 9.180753707885742 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 9.179461479187012 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 9.180407524108887 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 9.179150581359863 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 9.179658889770508 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 9.179596900939941 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 9.180424690246582 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 9.18022346496582 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 9.180010795593262 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 9.180096626281738 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 9.179329872131348 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 9.178979873657227 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 9.179104804992676 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 9.180551528930664 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 9.180392265319824 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 9.180694580078125 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 9.180516242980957 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 9.179238319396973 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 9.179339408874512 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 9.180315971374512 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 9.179844856262207 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 9.180542945861816 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 9.180096626281738 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 9.180188179016113 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 9.17944622039795 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 9.179856300354004 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 9.180066108703613 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 9.180566787719727 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 9.180549621582031 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 9.180245399475098 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 9.17969799041748 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 9.178862571716309 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 9.178546905517578 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 9.17978286743164 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 9.180801391601562 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 9.179960250854492 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 9.180230140686035 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 9.179439544677734 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 9.17876148223877 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 9.17882251739502 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 9.180232048034668 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 9.180761337280273 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 9.180882453918457 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 9.180551528930664 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 9.179848670959473 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 9.178695678710938 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 9.179356575012207 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 9.180234909057617 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 9.180313110351562 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 9.179978370666504 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 9.181458473205566 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 9.180950164794922 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 9.177203178405762 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 9.177654266357422 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 9.179722785949707 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 9.180479049682617 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 9.179802894592285 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 9.18082332611084 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 9.179072380065918 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 9.175627708435059 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 9.175374031066895 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 9.176494598388672 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 9.022957801818848 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 9.022660255432129 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 9.020158767700195 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 9.023996353149414 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 9.020976066589355 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 9.022041320800781 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 9.021373748779297 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 9.022866249084473 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 9.023050308227539 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 9.020857810974121 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 9.023626327514648 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 9.022245407104492 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 9.02010726928711 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 9.02099323272705 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 9.023275375366211 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 9.023691177368164 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 9.021981239318848 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 9.022442817687988 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 9.022598266601562 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 9.020668983459473 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 9.022163391113281 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 9.022855758666992 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 9.022513389587402 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 9.023458480834961 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 9.023643493652344 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 9.020898818969727 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 9.021952629089355 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 9.022353172302246 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 9.022377014160156 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 9.022314071655273 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 9.022546768188477 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 9.020864486694336 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 9.022679328918457 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 9.021808624267578 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 9.022629737854004 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 9.022746086120605 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 9.022989273071289 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 9.022150993347168 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 9.022284507751465 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 9.02147102355957 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 9.022168159484863 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 9.022735595703125 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 9.023213386535645 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 9.023591995239258 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 9.021907806396484 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 9.022305488586426 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 9.02174186706543 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 9.021656036376953 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 9.022012710571289 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 9.022031784057617 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 9.020895004272461 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 9.021320343017578 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 9.02261734008789 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 9.021010398864746 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 9.021860122680664 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 9.021808624267578 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 9.023364067077637 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 9.022619247436523 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 9.022216796875 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 9.022516250610352 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 9.022191047668457 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 9.023258209228516 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 9.022819519042969 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 9.02321720123291 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 9.021346092224121 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 9.022692680358887 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 9.022092819213867 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 9.022133827209473 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 9.023918151855469 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 9.022550582885742 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 9.022658348083496 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 9.022636413574219 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 9.022490501403809 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 9.022500991821289 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 9.022899627685547 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 9.021827697753906 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 9.022608757019043 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 9.023299217224121 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 9.022374153137207 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 9.021692276000977 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 9.021707534790039 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 9.022522926330566 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 9.022006034851074 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 9.022170066833496 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 9.022929191589355 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 9.022449493408203 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 9.023295402526855 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 9.022722244262695 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 9.021988868713379 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 9.022355079650879 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 9.02192497253418 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 9.022644996643066 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 9.0195951461792 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 9.021894454956055 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 9.020451545715332 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 9.0219087600708 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 9.023103713989258 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 9.022588729858398 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 9.023092269897461 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 9.022552490234375 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 9.022698402404785 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 9.022483825683594 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 9.022154808044434 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 9.022455215454102 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 9.021191596984863 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 9.023441314697266 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 9.021164894104004 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 9.022576332092285 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 9.023239135742188 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 9.022546768188477 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 9.022623062133789 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 9.021905899047852 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 9.022869110107422 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 9.0230712890625 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 9.02259349822998 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 9.023045539855957 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 9.022370338439941 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 9.022709846496582 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 9.021956443786621 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 9.023028373718262 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 9.022916793823242 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 9.023001670837402 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 9.022841453552246 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 9.022212982177734 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 9.021923065185547 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 9.021219253540039 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 9.023161888122559 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 9.023275375366211 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 9.022821426391602 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 9.023138046264648 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 9.022797584533691 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 9.022255897521973 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 9.023261070251465 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 9.023367881774902 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 9.023299217224121 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 9.023042678833008 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 9.021984100341797 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 9.022038459777832 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 9.022890090942383 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 9.02182388305664 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 9.02288818359375 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 9.022918701171875 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 9.02256965637207 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 9.022954940795898 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 9.021642684936523 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 9.02271842956543 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 9.021750450134277 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 9.023054122924805 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 9.023282051086426 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 9.023144721984863 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 9.02295970916748 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 9.022378921508789 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 9.023092269897461 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 9.02225399017334 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 9.023232460021973 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 9.021803855895996 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 9.022924423217773 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 9.022226333618164 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 9.022296905517578 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 9.022705078125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 9.021913528442383 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 9.022464752197266 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 9.02304744720459 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 9.023505210876465 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 9.022989273071289 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 9.022520065307617 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 9.023133277893066 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 9.022079467773438 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 9.022735595703125 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 9.022951126098633 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 9.02310562133789 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 9.02276611328125 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 9.021926879882812 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 9.02207088470459 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 9.022595405578613 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 9.022649765014648 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 9.022989273071289 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 9.022374153137207 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 9.023283958435059 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 9.022279739379883 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 9.022651672363281 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 9.022486686706543 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 9.023092269897461 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 9.022976875305176 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 9.022520065307617 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 9.02291202545166 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 9.021174430847168 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 9.022836685180664 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 9.021724700927734 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 9.022668838500977 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 9.022623062133789 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 9.0241060256958 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 9.022963523864746 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 9.019908905029297 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 9.021815299987793 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 9.020004272460938 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.91749382019043 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.917819023132324 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.918954849243164 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.916154861450195 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.917566299438477 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.916701316833496 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.916592597961426 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.917922973632812 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.91726016998291 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.919426918029785 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.917502403259277 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.916309356689453 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.916343688964844 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.916905403137207 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.916839599609375 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.916711807250977 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.91857624053955 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.918230056762695 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.917154312133789 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.91786003112793 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.917768478393555 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.917716979980469 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.916940689086914 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.916234970092773 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.916643142700195 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.916847229003906 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.916646003723145 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.918033599853516 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.91769790649414 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.916388511657715 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.917200088500977 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.918643951416016 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.916205406188965 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.917034149169922 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.916397094726562 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.917983055114746 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.91726016998291 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.917951583862305 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.916481971740723 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.91707706451416 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.917689323425293 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.91655445098877 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.918242454528809 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.917043685913086 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.916108131408691 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.916830062866211 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.916558265686035 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.917189598083496 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.91742992401123 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.917734146118164 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.91647720336914 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.916160583496094 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.916179656982422 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.915620803833008 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.917118072509766 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.917383193969727 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.917230606079102 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.917387962341309 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.918280601501465 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.91889476776123 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.916936874389648 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.917047500610352 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.91601848602295 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.917777061462402 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.916732788085938 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.918314933776855 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.917421340942383 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.917536735534668 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.916924476623535 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.91702651977539 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.917725563049316 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.917032241821289 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.917455673217773 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.917787551879883 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.915937423706055 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.917919158935547 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.917383193969727 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.917398452758789 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.917655944824219 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.917604446411133 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.918268203735352 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.916839599609375 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.917685508728027 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.917540550231934 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.91813850402832 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.917533874511719 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.917115211486816 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.917170524597168 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.91726016998291 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.917867660522461 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.91711139678955 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.917030334472656 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.912911415100098 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.918259620666504 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.91749382019043 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.918004035949707 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.917158126831055 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.9170560836792 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.917694091796875 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.91734504699707 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.917863845825195 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.917426109313965 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.9169282913208 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.916996002197266 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.917742729187012 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.917732238769531 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.915848731994629 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.917081832885742 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.917469024658203 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.91707706451416 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.917115211486816 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.91688346862793 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.917442321777344 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.917374610900879 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.917434692382812 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.91732406616211 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.917306900024414 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.917277336120605 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.91680908203125 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.917490005493164 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.917502403259277 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.917230606079102 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.917264938354492 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.916911125183105 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.917266845703125 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.916962623596191 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.917440414428711 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.91740894317627 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.917396545410156 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.917096138000488 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.916746139526367 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.917285919189453 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.916143417358398 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.917972564697266 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.918100357055664 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.917651176452637 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.917898178100586 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.916851997375488 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.916994094848633 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.917362213134766 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.917536735534668 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.917552947998047 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.91752815246582 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.917391777038574 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.917459487915039 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.917096138000488 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.917068481445312 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.917706489562988 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.917774200439453 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.917742729187012 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.917804718017578 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.91672420501709 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.91728401184082 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.91691780090332 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.917448997497559 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.917974472045898 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.917625427246094 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.917957305908203 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.917251586914062 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.917037010192871 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.917312622070312 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.917625427246094 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.917457580566406 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.917867660522461 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.917353630065918 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.917536735534668 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.917222023010254 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.918359756469727 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.9176664352417 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.917410850524902 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.917255401611328 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.917668342590332 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.916810989379883 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.91711139678955 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.916952133178711 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.91806411743164 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.917961120605469 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.91822338104248 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.917583465576172 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.916426658630371 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.91675853729248 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.916120529174805 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.917630195617676 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.917630195617676 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.918590545654297 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.917799949645996 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.916810989379883 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.916520118713379 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.917346954345703 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.917723655700684 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.917400360107422 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.917804718017578 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.917051315307617 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.915822982788086 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.915026664733887 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.91596794128418 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.833033561706543 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.832845687866211 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.831260681152344 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.833221435546875 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.831351280212402 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.83283805847168 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.83339786529541 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.83254337310791 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.833003997802734 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.832913398742676 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.83359146118164 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.83246898651123 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.832505226135254 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.832298278808594 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.83315372467041 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.83311939239502 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.832406044006348 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.832467079162598 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.832277297973633 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.83348560333252 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.832369804382324 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.83288860321045 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.832871437072754 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.832778930664062 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.833216667175293 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.83245849609375 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.833183288574219 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.83214282989502 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.832749366760254 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.832462310791016 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.832239151000977 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.831254005432129 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.832772254943848 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.832976341247559 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.833002090454102 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.832732200622559 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.832616806030273 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.832054138183594 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.832549095153809 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.832306861877441 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.833556175231934 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.83265209197998 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.832334518432617 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.833061218261719 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.831772804260254 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.832911491394043 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.832709312438965 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.83313274383545 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.832925796508789 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.833436012268066 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.831789016723633 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.832357406616211 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.833237648010254 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.832622528076172 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.832723617553711 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.831993103027344 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.833610534667969 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.833174705505371 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.832963943481445 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.831809997558594 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.832282066345215 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.833514213562012 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.833317756652832 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.8328857421875 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.831865310668945 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.832439422607422 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.832690238952637 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.83226490020752 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.832555770874023 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.833301544189453 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.833028793334961 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.83285903930664 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.833006858825684 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.832695007324219 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.833250045776367 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.832829475402832 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.833025932312012 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.833166122436523 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.832789421081543 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.832512855529785 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.832269668579102 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.832892417907715 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.832167625427246 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.83306884765625 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.833052635192871 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.83275318145752 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.832732200622559 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.832793235778809 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.83270263671875 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.832395553588867 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.833385467529297 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.833250045776367 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.82950210571289 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.832279205322266 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.831672668457031 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.832564353942871 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.83250904083252 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.833292961120605 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.833446502685547 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.83263111114502 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.832770347595215 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.832711219787598 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.832639694213867 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.832761764526367 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.83161449432373 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.833313941955566 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.832928657531738 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.832826614379883 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.832818984985352 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.832568168640137 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.832303047180176 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.832854270935059 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.83277416229248 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.833195686340332 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.832857131958008 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.832627296447754 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.83221435546875 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.832904815673828 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.83228588104248 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.832864761352539 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.83310317993164 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.832571983337402 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.832862854003906 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.832602500915527 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.832730293273926 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.831368446350098 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.83284854888916 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.833035469055176 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.832984924316406 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.833157539367676 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.83275318145752 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.833392143249512 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.833537101745605 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.833023071289062 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.833134651184082 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.832646369934082 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.83250904083252 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.832785606384277 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.832686424255371 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.831586837768555 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.83314037322998 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.832612037658691 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.83254337310791 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.832951545715332 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.832088470458984 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.833040237426758 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.832602500915527 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.833145141601562 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.83316421508789 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.833357810974121 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.83313274383545 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.832498550415039 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.83296012878418 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.832804679870605 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.83298110961914 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.832566261291504 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.83298110961914 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.832262992858887 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.83261489868164 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.832883834838867 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.832233428955078 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.832533836364746 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.83278751373291 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.83327579498291 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.832897186279297 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.832503318786621 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.833111763000488 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.832086563110352 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.832989692687988 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.833252906799316 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.83341121673584 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.83263111114502 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.832488059997559 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.83325481414795 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.832728385925293 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.832850456237793 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.83298110961914 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.833211898803711 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.832676887512207 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.83240032196045 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.833312034606934 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.833412170410156 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.832987785339355 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.832976341247559 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.832435607910156 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.832925796508789 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.832043647766113 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.83290958404541 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.831332206726074 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.832836151123047 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.83313274383545 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.833465576171875 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.833317756652832 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.83080005645752 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.833427429199219 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.831515312194824 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.773863792419434 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.774065971374512 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.774942398071289 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.773872375488281 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.774513244628906 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.773024559020996 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.772635459899902 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.774144172668457 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.773988723754883 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.774921417236328 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.773579597473145 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.772859573364258 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.7725248336792 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.7737455368042 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.773627281188965 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.773902893066406 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.774214744567871 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.77449893951416 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.773603439331055 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.77354907989502 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.77388858795166 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.774524688720703 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.773541450500488 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.772521018981934 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.773306846618652 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.773629188537598 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.773712158203125 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.773974418640137 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.774210929870605 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.772892951965332 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.77393913269043 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.774503707885742 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.77329158782959 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.772533416748047 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.773508071899414 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.774091720581055 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.77430248260498 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.774227142333984 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.77428150177002 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.773763656616211 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.773612022399902 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.773388862609863 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.774833679199219 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.774311065673828 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.773494720458984 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.773348808288574 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.77346420288086 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.773150444030762 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.773303031921387 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.774086952209473 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.773386001586914 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.771855354309082 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.773490905761719 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.772941589355469 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.773072242736816 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.773934364318848 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.773704528808594 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.773880004882812 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.773972511291504 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.774643898010254 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.773727416992188 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.77387809753418 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.773591041564941 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.773909568786621 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.773210525512695 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.774165153503418 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.774283409118652 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.77385139465332 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.773336410522461 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.774210929870605 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.773587226867676 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.773737907409668 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.77398681640625 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.774415969848633 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.77354907989502 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.774260520935059 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.77393627166748 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.773950576782227 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.774016380310059 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.773787498474121 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.774332046508789 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.773417472839355 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.774076461791992 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.774026870727539 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.774181365966797 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.773783683776855 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.773918151855469 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.773591041564941 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.773820877075195 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.773636817932129 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.773553848266602 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.773622512817383 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.770675659179688 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.77454948425293 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.774116516113281 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.77437973022461 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.774332046508789 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.773056030273438 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.773818969726562 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.773240089416504 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.773881912231445 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.774419784545898 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.774091720581055 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.773981094360352 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.774137496948242 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.77428150177002 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.773581504821777 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.77353286743164 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.774081230163574 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.77357006072998 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.773636817932129 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.77335262298584 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.774104118347168 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.773695945739746 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.774051666259766 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.77397632598877 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.77424430847168 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.773344039916992 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.77353286743164 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.773974418640137 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.774072647094727 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.773907661437988 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.77408504486084 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.77340316772461 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.774060249328613 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.774532318115234 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.774097442626953 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.77354907989502 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.77383804321289 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.773907661437988 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.773411750793457 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.773629188537598 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.773369789123535 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.774072647094727 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.773926734924316 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.773750305175781 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.774123191833496 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.77363395690918 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.773636817932129 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.773993492126465 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.773884773254395 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.77426528930664 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.7741060256958 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.77372932434082 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.774002075195312 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.773399353027344 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.773080825805664 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.77380084991455 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.773834228515625 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.774043083190918 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.773846626281738 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.773681640625 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.773691177368164 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.773754119873047 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.77383041381836 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.77398681640625 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.773941040039062 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.773813247680664 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.773783683776855 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.773310661315918 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.774083137512207 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.773860931396484 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.773900985717773 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.773776054382324 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.773857116699219 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.774065971374512 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.773401260375977 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.774177551269531 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.773941040039062 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.773624420166016 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.773541450500488 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.774214744567871 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.773553848266602 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.773017883300781 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.77357292175293 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.773909568786621 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.774026870727539 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.77444076538086 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.773683547973633 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.773695945739746 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.773792266845703 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.773310661315918 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.773920059204102 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.773733139038086 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.774742126464844 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.774212837219238 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.772710800170898 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.773754119873047 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.774099349975586 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.774026870727539 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.773841857910156 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.773926734924316 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.77354907989502 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.773194313049316 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.770978927612305 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.772868156433105 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.72252082824707 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.72266674041748 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.721487045288086 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.722853660583496 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.721077919006348 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.722224235534668 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.722853660583496 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.72233772277832 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.722790718078613 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.722261428833008 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.722962379455566 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.722283363342285 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.722150802612305 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.721442222595215 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.722479820251465 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.723135948181152 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.72202205657959 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.722311019897461 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.721572875976562 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.72282886505127 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.722180366516113 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.722200393676758 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.722333908081055 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.722492218017578 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.722304344177246 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.722450256347656 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.722407341003418 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.722434043884277 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.722518920898438 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.7225341796875 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.722085952758789 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.72161865234375 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.722067832946777 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.723219871520996 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.721938133239746 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.722393989562988 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.722317695617676 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.721123695373535 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.722229957580566 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.72230052947998 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.72242546081543 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.722288131713867 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.72232723236084 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.722456932067871 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.721094131469727 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.722259521484375 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.722458839416504 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.723244667053223 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.722458839416504 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.72247314453125 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.722094535827637 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.722177505493164 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.722747802734375 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.72204875946045 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.722641944885254 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.72160816192627 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.722805976867676 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.722304344177246 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.722776412963867 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.721646308898926 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.722029685974121 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.722498893737793 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.722487449645996 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.72238826751709 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.721907615661621 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.72204303741455 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.722038269042969 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.722395896911621 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.72310733795166 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.722018241882324 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.722661972045898 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.722188949584961 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.722606658935547 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.72183895111084 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.722641944885254 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.722479820251465 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.722221374511719 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.722610473632812 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.722188949584961 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.721705436706543 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.722240447998047 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.722809791564941 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.722163200378418 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.722479820251465 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.722333908081055 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.722554206848145 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.72264575958252 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.722641944885254 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.72219181060791 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.722731590270996 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.722628593444824 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.72251033782959 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.720355033874512 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.721951484680176 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.720871925354004 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.721837043762207 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.722317695617676 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.722564697265625 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.722893714904785 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.722221374511719 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.72271728515625 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.721858024597168 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.722291946411133 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.722196578979492 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.721796989440918 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.722475051879883 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.722522735595703 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.722121238708496 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.722734451293945 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.7227144241333 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.722271919250488 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.722118377685547 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.722841262817383 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.722825050354004 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.722591400146484 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.722404479980469 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.72202205657959 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.722429275512695 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.722280502319336 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.722617149353027 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.722604751586914 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.72253704071045 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.722323417663574 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.722055435180664 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.722150802612305 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.721368789672852 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.722564697265625 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.722679138183594 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.722587585449219 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.722816467285156 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.722652435302734 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.722610473632812 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.722153663635254 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.722776412963867 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.722879409790039 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.722386360168457 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.722259521484375 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.722082138061523 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.72252082824707 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.721148490905762 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.722492218017578 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.722434043884277 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.722317695617676 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.722633361816406 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.721938133239746 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.72167682647705 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.722332000732422 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.722811698913574 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.72266674041748 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.722413063049316 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.722587585449219 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.722217559814453 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.72241497039795 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.722163200378418 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.722452163696289 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.722238540649414 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.722485542297363 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.722121238708496 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.722296714782715 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.722617149353027 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.721946716308594 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.72237491607666 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.722479820251465 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.722874641418457 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.722150802612305 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.722005844116211 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.722640037536621 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.721959114074707 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.72238826751709 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.722575187683105 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.722914695739746 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.722267150878906 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.722062110900879 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.722550392150879 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.72286605834961 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.722562789916992 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.722684860229492 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.722063064575195 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.722513198852539 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.722063064575195 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.72220516204834 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.722550392150879 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.722591400146484 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.722513198852539 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.722229957580566 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.72253704071045 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.721920013427734 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.721942901611328 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.721368789672852 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.722402572631836 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.722628593444824 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.722372055053711 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.722591400146484 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.720815658569336 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.723402976989746 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.721393585205078 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.677469253540039 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.677651405334473 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.678138732910156 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.676236152648926 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.678240776062012 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.677297592163086 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.677035331726074 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.677803993225098 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.677465438842773 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.678014755249023 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.677506446838379 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.676920890808105 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.676641464233398 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.677063941955566 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.67751693725586 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.67671012878418 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.678098678588867 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.677899360656738 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.678251266479492 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.677237510681152 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.67831802368164 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.677785873413086 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.677367210388184 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.67577075958252 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.67719841003418 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.67691707611084 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.677274703979492 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.67776107788086 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.677506446838379 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.676858901977539 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.676496505737305 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.67736530303955 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.67739486694336 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.676708221435547 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.677473068237305 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.677678108215332 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.6779203414917 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.678582191467285 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.677587509155273 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.677568435668945 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.677227020263672 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.67779541015625 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.677911758422852 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.677816390991211 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.676554679870605 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.676860809326172 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.677225112915039 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.676641464233398 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.677775382995605 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.677416801452637 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.676867485046387 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.676243782043457 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.676862716674805 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.677125930786133 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.676706314086914 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.677545547485352 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.677494049072266 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.677708625793457 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.677607536315918 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.677663803100586 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.677729606628418 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.677324295043945 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.677295684814453 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.677816390991211 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.676839828491211 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.678098678588867 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.677287101745605 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.677495956420898 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.677101135253906 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.677202224731445 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.677448272705078 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.677618026733398 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.677539825439453 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.677720069885254 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.676856994628906 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.677347183227539 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.677196502685547 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.677177429199219 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.677437782287598 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.677535057067871 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.677766799926758 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.676989555358887 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.677624702453613 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.677603721618652 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.677785873413086 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.677634239196777 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.677619934082031 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.676993370056152 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.677355766296387 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.677204132080078 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.677515029907227 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.677218437194824 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.67607307434082 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.677552223205566 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.67784595489502 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.677841186523438 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.67734146118164 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.677352905273438 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.677474975585938 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.676856994628906 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.677762985229492 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.677700996398926 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.677642822265625 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.677580833435059 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.677384376525879 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.67769718170166 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.67745304107666 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.677109718322754 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.67740535736084 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.677053451538086 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.677063941955566 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.677001953125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.677523612976074 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.67763614654541 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.677407264709473 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.677210807800293 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.677545547485352 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.677096366882324 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.67695426940918 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.67737865447998 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.677680015563965 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.677217483520508 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.677214622497559 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.677448272705078 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.67724323272705 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.677821159362793 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.677515029907227 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.677486419677734 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.677332878112793 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.677285194396973 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.677117347717285 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.677163124084473 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.677330017089844 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.677764892578125 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.677597045898438 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.677460670471191 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.67763614654541 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.677122116088867 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.677170753479004 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.678058624267578 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.677407264709473 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.677630424499512 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.677607536315918 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.677289962768555 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.677769660949707 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.677630424499512 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.676949501037598 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.677748680114746 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.677502632141113 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.677667617797852 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.677217483520508 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.67734432220459 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.677855491638184 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.677308082580566 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.677577018737793 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.677469253540039 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.67740249633789 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.677523612976074 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.677101135253906 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.677067756652832 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.677361488342285 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.67760181427002 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.677781105041504 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.677680015563965 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.677628517150879 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.677800178527832 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.677189826965332 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.677122116088867 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.677713394165039 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.677262306213379 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.677046775817871 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.6774320602417 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.677175521850586 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.677228927612305 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.677038192749023 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.677593231201172 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.677411079406738 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.6780424118042 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.677270889282227 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.677035331726074 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.677560806274414 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.677129745483398 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.677448272705078 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.677428245544434 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.677651405334473 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.67739486694336 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.676979064941406 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.677671432495117 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.677543640136719 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.677659034729004 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.677510261535645 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.677725791931152 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.67720890045166 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.677183151245117 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.675212860107422 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.676701545715332 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.641900062561035 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.641950607299805 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.641341209411621 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.642690658569336 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.640931129455566 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.641518592834473 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.641936302185059 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.64171028137207 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.642178535461426 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.64156436920166 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.641804695129395 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.641871452331543 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.641761779785156 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.641980171203613 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.64178466796875 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.642414093017578 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.64132308959961 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.641621589660645 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.641580581665039 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.641871452331543 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.641264915466309 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.641798973083496 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.64174747467041 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.642377853393555 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.642229080200195 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.642264366149902 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.641495704650879 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.641757011413574 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.642068862915039 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.641573905944824 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.641637802124023 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.64161491394043 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.640955924987793 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.642555236816406 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.640931129455566 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.641609191894531 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.641590118408203 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.641347885131836 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.642109870910645 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.641522407531738 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.642107963562012 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.641607284545898 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.641889572143555 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.641973495483398 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.640589714050293 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.641497611999512 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.641850471496582 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.642719268798828 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.641948699951172 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.641742706298828 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.641874313354492 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.641257286071777 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.642351150512695 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.641371726989746 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.641845703125 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.641268730163574 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.641932487487793 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.641834259033203 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.64218807220459 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.64224624633789 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.641465187072754 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.642559051513672 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.642303466796875 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.641838073730469 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.64145278930664 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.641143798828125 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.641590118408203 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.641669273376465 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.641716003417969 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.641797065734863 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.641836166381836 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.641852378845215 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.641731262207031 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.64197826385498 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.641860008239746 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.641996383666992 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.641666412353516 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.642056465148926 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.641568183898926 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.64169979095459 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.641328811645508 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.642175674438477 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.641701698303223 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.64189338684082 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.641860008239746 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.641887664794922 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.641845703125 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.64168930053711 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.641779899597168 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.641926765441895 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.642006874084473 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.641895294189453 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.6404447555542 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.641520500183105 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.640717506408691 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.641520500183105 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.641742706298828 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.641804695129395 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.641834259033203 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.641380310058594 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.641813278198242 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.64195728302002 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.641800880432129 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.641817092895508 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.641541481018066 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.64168930053711 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.6415433883667 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.641592025756836 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.641694068908691 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.642056465148926 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.642064094543457 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.641571998596191 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.642029762268066 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.64203929901123 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.641916275024414 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.641948699951172 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.641376495361328 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.642006874084473 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.641676902770996 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.641609191894531 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.641491889953613 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.6419677734375 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.641834259033203 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.641838073730469 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.641669273376465 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.641264915466309 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.642006874084473 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.642056465148926 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.641603469848633 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.642010688781738 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.641854286193848 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.641948699951172 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.641497611999512 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.6420316696167 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.6419677734375 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.641722679138184 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.641745567321777 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.64197826385498 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.64172649383545 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.641273498535156 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.64162540435791 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.641874313354492 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.641800880432129 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.641858100891113 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.640984535217285 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.64084529876709 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.641399383544922 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.641751289367676 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.641850471496582 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.641697883605957 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.642101287841797 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.641552925109863 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.64180850982666 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.641277313232422 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.641706466674805 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.641617774963379 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.641724586486816 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.641643524169922 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.641745567321777 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.642027854919434 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.641300201416016 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.641706466674805 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.641817092895508 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.641953468322754 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.64137077331543 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.64133358001709 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.641798973083496 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.64194107055664 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.641714096069336 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.641660690307617 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.64165210723877 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.641788482666016 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.641578674316406 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.641664505004883 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.64197826385498 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.642017364501953 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.641742706298828 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.641596794128418 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.641887664794922 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.641550064086914 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.641944885253906 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.64167308807373 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.64150619506836 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.641660690307617 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.641763687133789 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.641879081726074 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.641380310058594 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.641273498535156 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.640671730041504 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.641717910766602 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.641738891601562 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.641714096069336 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.641918182373047 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.640251159667969 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.642414093017578 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.641192436218262 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.615677833557129 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.615726470947266 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.615656852722168 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.614704132080078 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.616039276123047 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.615702629089355 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.616039276123047 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.615793228149414 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.615656852722168 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.61593246459961 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.616169929504395 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.616026878356934 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.614766120910645 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.615283012390137 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.615757942199707 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.615167617797852 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.616006851196289 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.615898132324219 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.61597728729248 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.6154146194458 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.616252899169922 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.615928649902344 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.615307807922363 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.614704132080078 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.615569114685059 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.61535930633545 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.61595344543457 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.615954399108887 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.615387916564941 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.615659713745117 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.614930152893066 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.615675926208496 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.615619659423828 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.614861488342285 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.615665435791016 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.615818977355957 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.615612030029297 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.61560344696045 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.615798950195312 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.615665435791016 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.61551570892334 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.615971565246582 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.615632057189941 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.615702629089355 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.614845275878906 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.615459442138672 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.615147590637207 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.615010261535645 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.615838050842285 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.61569595336914 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.615057945251465 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.614272117614746 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.61519718170166 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.615373611450195 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.614995956420898 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.615846633911133 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.615805625915527 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.61589527130127 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.6160306930542 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.61551284790039 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.615720748901367 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.615612030029297 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.615554809570312 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.615800857543945 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.615583419799805 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.616293907165527 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.615912437438965 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.615777969360352 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.615628242492676 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.615825653076172 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.615558624267578 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.615645408630371 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.615425109863281 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.615400314331055 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.615392684936523 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.615503311157227 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.616026878356934 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.615577697753906 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.615599632263184 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.615525245666504 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.616026878356934 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.615541458129883 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.615884780883789 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.615735054016113 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.615979194641113 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.615640640258789 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.615788459777832 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.615578651428223 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.615928649902344 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.615619659423828 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.615735054016113 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.615496635437012 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.614786148071289 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.615982055664062 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.615920066833496 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.616111755371094 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.615538597106934 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.615279197692871 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.615948677062988 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.615201950073242 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.615862846374512 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.615541458129883 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.615628242492676 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.615412712097168 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.615921974182129 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.615713119506836 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.615548133850098 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.61526870727539 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.615706443786621 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.61540699005127 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.615063667297363 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.615434646606445 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.615583419799805 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.615673065185547 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.61569881439209 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.615669250488281 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.616053581237793 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.6157865524292 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.61584186553955 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.615636825561523 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.615546226501465 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.61533260345459 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.615410804748535 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.615647315979004 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.615389823913574 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.61565113067627 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.615394592285156 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.615628242492676 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.615378379821777 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.615525245666504 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.615687370300293 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.615574836730957 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.615453720092773 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.615914344787598 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.615690231323242 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.615856170654297 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.615761756896973 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.61502456665039 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.615752220153809 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.615492820739746 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.615546226501465 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.615656852722168 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.615610122680664 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.61540699005127 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.615589141845703 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.616060256958008 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.615365028381348 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.615638732910156 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.615530014038086 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.615677833557129 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.615521430969238 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.615636825561523 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.615706443786621 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.615936279296875 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.615595817565918 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.61573314666748 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.61550521850586 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.615496635437012 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.615326881408691 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.61540699005127 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.615854263305664 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.615457534790039 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.61569881439209 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.615578651428223 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.61572265625 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.61562442779541 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.61519718170166 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.61558723449707 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.615640640258789 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.615724563598633 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.615427017211914 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.615412712097168 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.615229606628418 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.615408897399902 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.615389823913574 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.615455627441406 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.615524291992188 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.615357398986816 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.615246772766113 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.615418434143066 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.615147590637207 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.615626335144043 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.615471839904785 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.615389823913574 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.615617752075195 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.615811347961426 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.615193367004395 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.615647315979004 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.615862846374512 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.615449905395508 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.615677833557129 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.615900039672852 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.615363121032715 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.615694046020508 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.614174842834473 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.614274978637695 |
| } |
| ] |
| } |
| ] |
| } |
