55 #define X8(x) x,x,x,x,x,x,x,x
67 0x4000, 0x539f, 0xc000, 0xac61, 0x4000, 0xdd5d, 0x4000, 0xdd5d,
68 0x4000, 0x22a3, 0x4000, 0x22a3, 0xc000, 0x539f, 0x4000, 0xac61,
69 0x3249, 0x11a8, 0x4b42, 0xee58, 0x11a8, 0x4b42, 0x11a8, 0xcdb7,
70 0x58c5, 0x4b42, 0xa73b, 0xcdb7, 0x3249, 0xa73b, 0x4b42, 0xa73b
74 0x58c5, 0x73fc, 0xa73b, 0x8c04, 0x58c5, 0xcff5, 0x58c5, 0xcff5,
75 0x58c5, 0x300b, 0x58c5, 0x300b, 0xa73b, 0x73fc, 0x58c5, 0x8c04,
76 0x45bf, 0x187e, 0x6862, 0xe782, 0x187e, 0x6862, 0x187e, 0xba41,
77 0x7b21, 0x6862, 0x84df, 0xba41, 0x45bf, 0x84df, 0x6862, 0x84df
81 0x539f, 0x6d41, 0xac61, 0x92bf, 0x539f, 0xd2bf, 0x539f, 0xd2bf,
82 0x539f, 0x2d41, 0x539f, 0x2d41, 0xac61, 0x6d41, 0x539f, 0x92bf,
83 0x41b3, 0x1712, 0x6254, 0xe8ee, 0x1712, 0x6254, 0x1712, 0xbe4d,
84 0x73fc, 0x6254, 0x8c04, 0xbe4d, 0x41b3, 0x8c04, 0x6254, 0x8c04
88 0x4b42, 0x6254, 0xb4be, 0x9dac, 0x4b42, 0xd746, 0x4b42, 0xd746,
89 0x4b42, 0x28ba, 0x4b42, 0x28ba, 0xb4be, 0x6254, 0x4b42, 0x9dac,
90 0x3b21, 0x14c3, 0x587e, 0xeb3d, 0x14c3, 0x587e, 0x14c3, 0xc4df,
91 0x6862, 0x587e, 0x979e, 0xc4df, 0x3b21, 0x979e, 0x587e, 0x979e
95 65536, 65536, 65536, 65536,
96 3597, 3597, 3597, 3597,
97 2260, 2260, 2260, 2260,
98 1203, 1203, 1203, 1203,
104 #define ROW1 "%%xmm6"
105 #define ROW3 "%%xmm4"
106 #define ROW5 "%%xmm5"
107 #define ROW7 "%%xmm7"
109 #define CLEAR_ODD(r) "pxor "r","r" \n\t"
110 #define PUT_ODD(dst) "pshufhw $0x1B, %%xmm2, "dst" \n\t"
114 # define ROW0 "%%xmm8"
116 # define ROW2 "%%xmm9"
118 # define ROW4 "%%xmm10"
120 # define ROW6 "%%xmm11"
122 # define CLEAR_EVEN(r) CLEAR_ODD(r)
123 # define PUT_EVEN(dst) PUT_ODD(dst)
124 # define XMMS "%%xmm12"
125 # define MOV_32_ONLY "#"
127 # define TAN3 "%%xmm13"
128 # define TAN1 "%%xmm14"
133 # define REG0 "%%xmm4"
134 # define ROW2 "2*16(%0)"
135 # define REG2 "%%xmm4"
136 # define ROW4 "4*16(%0)"
137 # define REG4 "%%xmm6"
138 # define ROW6 "6*16(%0)"
139 # define REG6 "%%xmm6"
140 # define CLEAR_EVEN(r)
141 # define PUT_EVEN(dst) \
142 "pshufhw $0x1B, %%xmm2, %%xmm2 \n\t" \
143 "movdqa %%xmm2, "dst" \n\t"
144 # define XMMS "%%xmm2"
145 # define MOV_32_ONLY "movdqa "
146 # define SREG2 "%%xmm7"
147 # define TAN3 "%%xmm0"
148 # define TAN1 "%%xmm2"
152 #define ROUND(x) "paddd "MANGLE(x)
154 #define JZ(reg, to) \
155 "testl "reg","reg" \n\t" \
158 #define JNZ(reg, to) \
159 "testl "reg","reg" \n\t" \
162 #define TEST_ONE_ROW(src, reg, clear) \
164 "movq "src", %%mm1 \n\t" \
165 "por 8+"src", %%mm1 \n\t" \
166 "paddusb %%mm0, %%mm1 \n\t" \
167 "pmovmskb %%mm1, "reg" \n\t"
169 #define TEST_TWO_ROWS(row1, row2, reg1, reg2, clear1, clear2) \
172 "movq "row1", %%mm1 \n\t" \
173 "por 8+"row1", %%mm1 \n\t" \
174 "movq "row2", %%mm2 \n\t" \
175 "por 8+"row2", %%mm2 \n\t" \
176 "paddusb %%mm0, %%mm1 \n\t" \
177 "paddusb %%mm0, %%mm2 \n\t" \
178 "pmovmskb %%mm1, "reg1" \n\t" \
179 "pmovmskb %%mm2, "reg2" \n\t"
182 #define iMTX_MULT(src, table, rounder, put) \
183 "movdqa "src", %%xmm3 \n\t" \
184 "movdqa %%xmm3, %%xmm0 \n\t" \
185 "pshufd $0x11, %%xmm3, %%xmm1 \n\t" \
186 "punpcklqdq %%xmm0, %%xmm0 \n\t" \
187 "pmaddwd "table", %%xmm0 \n\t" \
188 "pmaddwd 16+"table", %%xmm1 \n\t" \
189 "pshufd $0xBB, %%xmm3, %%xmm2 \n\t" \
190 "punpckhqdq %%xmm3, %%xmm3 \n\t" \
191 "pmaddwd 32+"table", %%xmm2 \n\t" \
192 "pmaddwd 48+"table", %%xmm3 \n\t" \
193 "paddd %%xmm1, %%xmm0 \n\t" \
194 "paddd %%xmm3, %%xmm2 \n\t" \
195 rounder", %%xmm0 \n\t" \
196 "movdqa %%xmm2, %%xmm3 \n\t" \
197 "paddd %%xmm0, %%xmm2 \n\t" \
198 "psubd %%xmm3, %%xmm0 \n\t" \
199 "psrad $11, %%xmm2 \n\t" \
200 "psrad $11, %%xmm0 \n\t" \
201 "packssdw %%xmm0, %%xmm2 \n\t" \
206 "movdqa "MANGLE(tan3)", "TAN3" \n\t" \
207 "movdqa "MANGLE(tan1)", "TAN1" \n\t" \
209 #define iLLM_PASS(dct) \
211 "movdqa "TAN3", %%xmm1 \n\t" \
212 "movdqa "TAN1", %%xmm3 \n\t" \
213 "pmulhw %%xmm4, "TAN3" \n\t" \
214 "pmulhw %%xmm5, %%xmm1 \n\t" \
215 "paddsw %%xmm4, "TAN3" \n\t" \
216 "paddsw %%xmm5, %%xmm1 \n\t" \
217 "psubsw %%xmm5, "TAN3" \n\t" \
218 "paddsw %%xmm4, %%xmm1 \n\t" \
219 "pmulhw %%xmm7, %%xmm3 \n\t" \
220 "pmulhw %%xmm6, "TAN1" \n\t" \
221 "paddsw %%xmm6, %%xmm3 \n\t" \
222 "psubsw %%xmm7, "TAN1" \n\t" \
223 "movdqa %%xmm3, %%xmm7 \n\t" \
224 "movdqa "TAN1", %%xmm6 \n\t" \
225 "psubsw %%xmm1, %%xmm3 \n\t" \
226 "psubsw "TAN3", "TAN1" \n\t" \
227 "paddsw %%xmm7, %%xmm1 \n\t" \
228 "paddsw %%xmm6, "TAN3" \n\t" \
229 "movdqa %%xmm3, %%xmm6 \n\t" \
230 "psubsw "TAN3", %%xmm3 \n\t" \
231 "paddsw %%xmm6, "TAN3" \n\t" \
232 "movdqa "MANGLE(sqrt2)", %%xmm4 \n\t" \
233 "pmulhw %%xmm4, %%xmm3 \n\t" \
234 "pmulhw %%xmm4, "TAN3" \n\t" \
235 "paddsw "TAN3", "TAN3" \n\t" \
236 "paddsw %%xmm3, %%xmm3 \n\t" \
237 "movdqa "MANGLE(tan2)", %%xmm7 \n\t" \
238 MOV_32_ONLY ROW2", "REG2" \n\t" \
239 MOV_32_ONLY ROW6", "REG6" \n\t" \
240 "movdqa %%xmm7, %%xmm5 \n\t" \
241 "pmulhw "REG6", %%xmm7 \n\t" \
242 "pmulhw "REG2", %%xmm5 \n\t" \
243 "paddsw "REG2", %%xmm7 \n\t" \
244 "psubsw "REG6", %%xmm5 \n\t" \
245 MOV_32_ONLY ROW0", "REG0" \n\t" \
246 MOV_32_ONLY ROW4", "REG4" \n\t" \
247 MOV_32_ONLY" "TAN1", (%0) \n\t" \
248 "movdqa "REG0", "XMMS" \n\t" \
249 "psubsw "REG4", "REG0" \n\t" \
250 "paddsw "XMMS", "REG4" \n\t" \
251 "movdqa "REG4", "XMMS" \n\t" \
252 "psubsw %%xmm7, "REG4" \n\t" \
253 "paddsw "XMMS", %%xmm7 \n\t" \
254 "movdqa "REG0", "XMMS" \n\t" \
255 "psubsw %%xmm5, "REG0" \n\t" \
256 "paddsw "XMMS", %%xmm5 \n\t" \
257 "movdqa %%xmm5, "XMMS" \n\t" \
258 "psubsw "TAN3", %%xmm5 \n\t" \
259 "paddsw "XMMS", "TAN3" \n\t" \
260 "movdqa "REG0", "XMMS" \n\t" \
261 "psubsw %%xmm3, "REG0" \n\t" \
262 "paddsw "XMMS", %%xmm3 \n\t" \
263 MOV_32_ONLY" (%0), "TAN1" \n\t" \
264 "psraw $6, %%xmm5 \n\t" \
265 "psraw $6, "REG0" \n\t" \
266 "psraw $6, "TAN3" \n\t" \
267 "psraw $6, %%xmm3 \n\t" \
268 "movdqa "TAN3", 1*16("dct") \n\t" \
269 "movdqa %%xmm3, 2*16("dct") \n\t" \
270 "movdqa "REG0", 5*16("dct") \n\t" \
271 "movdqa %%xmm5, 6*16("dct") \n\t" \
272 "movdqa %%xmm7, %%xmm0 \n\t" \
273 "movdqa "REG4", %%xmm4 \n\t" \
274 "psubsw %%xmm1, %%xmm7 \n\t" \
275 "psubsw "TAN1", "REG4" \n\t" \
276 "paddsw %%xmm0, %%xmm1 \n\t" \
277 "paddsw %%xmm4, "TAN1" \n\t" \
278 "psraw $6, %%xmm1 \n\t" \
279 "psraw $6, %%xmm7 \n\t" \
280 "psraw $6, "TAN1" \n\t" \
281 "psraw $6, "REG4" \n\t" \
282 "movdqa %%xmm1, ("dct") \n\t" \
283 "movdqa "TAN1", 3*16("dct") \n\t" \
284 "movdqa "REG4", 4*16("dct") \n\t" \
285 "movdqa %%xmm7, 7*16("dct") \n\t"
288 #define iLLM_PASS_SPARSE(dct) \
289 "pmulhw %%xmm4, "TAN3" \n\t" \
290 "paddsw %%xmm4, "TAN3" \n\t" \
291 "movdqa %%xmm6, %%xmm3 \n\t" \
292 "pmulhw %%xmm6, "TAN1" \n\t" \
293 "movdqa %%xmm4, %%xmm1 \n\t" \
294 "psubsw %%xmm1, %%xmm3 \n\t" \
295 "paddsw %%xmm6, %%xmm1 \n\t" \
296 "movdqa "TAN1", %%xmm6 \n\t" \
297 "psubsw "TAN3", "TAN1" \n\t" \
298 "paddsw %%xmm6, "TAN3" \n\t" \
299 "movdqa %%xmm3, %%xmm6 \n\t" \
300 "psubsw "TAN3", %%xmm3 \n\t" \
301 "paddsw %%xmm6, "TAN3" \n\t" \
302 "movdqa "MANGLE(sqrt2)", %%xmm4 \n\t" \
303 "pmulhw %%xmm4, %%xmm3 \n\t" \
304 "pmulhw %%xmm4, "TAN3" \n\t" \
305 "paddsw "TAN3", "TAN3" \n\t" \
306 "paddsw %%xmm3, %%xmm3 \n\t" \
307 "movdqa "MANGLE(tan2)", %%xmm5 \n\t" \
308 MOV_32_ONLY ROW2", "SREG2" \n\t" \
309 "pmulhw "SREG2", %%xmm5 \n\t" \
310 MOV_32_ONLY ROW0", "REG0" \n\t" \
311 "movdqa "REG0", %%xmm6 \n\t" \
312 "psubsw "SREG2", %%xmm6 \n\t" \
313 "paddsw "REG0", "SREG2" \n\t" \
314 MOV_32_ONLY" "TAN1", (%0) \n\t" \
315 "movdqa "REG0", "XMMS" \n\t" \
316 "psubsw %%xmm5, "REG0" \n\t" \
317 "paddsw "XMMS", %%xmm5 \n\t" \
318 "movdqa %%xmm5, "XMMS" \n\t" \
319 "psubsw "TAN3", %%xmm5 \n\t" \
320 "paddsw "XMMS", "TAN3" \n\t" \
321 "movdqa "REG0", "XMMS" \n\t" \
322 "psubsw %%xmm3, "REG0" \n\t" \
323 "paddsw "XMMS", %%xmm3 \n\t" \
324 MOV_32_ONLY" (%0), "TAN1" \n\t" \
325 "psraw $6, %%xmm5 \n\t" \
326 "psraw $6, "REG0" \n\t" \
327 "psraw $6, "TAN3" \n\t" \
328 "psraw $6, %%xmm3 \n\t" \
329 "movdqa "TAN3", 1*16("dct") \n\t" \
330 "movdqa %%xmm3, 2*16("dct") \n\t" \
331 "movdqa "REG0", 5*16("dct") \n\t" \
332 "movdqa %%xmm5, 6*16("dct") \n\t" \
333 "movdqa "SREG2", %%xmm0 \n\t" \
334 "movdqa %%xmm6, %%xmm4 \n\t" \
335 "psubsw %%xmm1, "SREG2" \n\t" \
336 "psubsw "TAN1", %%xmm6 \n\t" \
337 "paddsw %%xmm0, %%xmm1 \n\t" \
338 "paddsw %%xmm4, "TAN1" \n\t" \
339 "psraw $6, %%xmm1 \n\t" \
340 "psraw $6, "SREG2" \n\t" \
341 "psraw $6, "TAN1" \n\t" \
342 "psraw $6, %%xmm6 \n\t" \
343 "movdqa %%xmm1, ("dct") \n\t" \
344 "movdqa "TAN1", 3*16("dct") \n\t" \
345 "movdqa %%xmm6, 4*16("dct") \n\t" \
346 "movdqa "SREG2", 7*16("dct") \n\t"
351 "movq "MANGLE(m127)
", %%mm0 \n\t"
352 iMTX_MULT(
"(%0)",
MANGLE(iTab1), ROUND(walkenIdctRounders), PUT_EVEN(ROW0))
353 iMTX_MULT("1*16(%0)",
MANGLE(iTab2), ROUND(walkenIdctRounders+1*16), PUT_ODD(ROW1))
354 iMTX_MULT("2*16(%0)",
MANGLE(iTab3), ROUND(walkenIdctRounders+2*16), PUT_EVEN(ROW2))
356 TEST_TWO_ROWS("3*16(%0)", "4*16(%0)", "%%eax", "%%ecx", CLEAR_ODD(ROW3), CLEAR_EVEN(ROW4))
358 iMTX_MULT("3*16(%0)",
MANGLE(iTab4), ROUND(walkenIdctRounders+3*16), PUT_ODD(ROW3))
360 TEST_TWO_ROWS("5*16(%0)", "6*16(%0)", "%%eax", "%%edx", CLEAR_ODD(ROW5), CLEAR_EVEN(ROW6))
361 TEST_ONE_ROW("7*16(%0)", "%%esi", CLEAR_ODD(ROW7))
368 iLLM_PASS_SPARSE("%0")
371 iMTX_MULT("4*16(%0)",
MANGLE(iTab1), "
#", PUT_EVEN(ROW4))
373 iMTX_MULT(
"5*16(%0)",
MANGLE(iTab4), ROUND(walkenIdctRounders+4*16), PUT_ODD(ROW5))
376 iMTX_MULT("6*16(%0)",
MANGLE(iTab3), ROUND(walkenIdctRounders+5*16), PUT_EVEN(ROW6))
379 iMTX_MULT("7*16(%0)",
MANGLE(iTab2), ROUND(walkenIdctRounders+5*16), PUT_ODD(ROW7))
388 "%xmm4" ,
"%xmm5" ,
"%xmm6" ,
"%xmm7" ,)
391 "%xmm12",
"%xmm13",
"%xmm14",)
393 "%eax",
"%ecx",
"%edx",
"%esi",
"memory"