GNU Radio 3.7.3 C++ API
volk_32fc_32f_dot_prod_32fc.h
Go to the documentation of this file.
1 #ifndef INCLUDED_volk_32fc_32f_dot_prod_32fc_a_H
2 #define INCLUDED_volk_32fc_32f_dot_prod_32fc_a_H
3 
4 #include <volk/volk_common.h>
5 #include<stdio.h>
6 
7 
8 #ifdef LV_HAVE_GENERIC
9 
10 
11 static inline void volk_32fc_32f_dot_prod_32fc_generic(lv_32fc_t* result, const lv_32fc_t* input, const float * taps, unsigned int num_points) {
12 
13  float res[2];
14  float *realpt = &res[0], *imagpt = &res[1];
15  const float* aPtr = (float*)input;
16  const float* bPtr= taps;
17  unsigned int number = 0;
18 
19  *realpt = 0;
20  *imagpt = 0;
21 
22  for(number = 0; number < num_points; number++){
23  *realpt += ((*aPtr++) * (*bPtr));
24  *imagpt += ((*aPtr++) * (*bPtr++));
25  }
26 
27  *result = *(lv_32fc_t*)(&res[0]);
28 }
29 
30 #endif /*LV_HAVE_GENERIC*/
31 
32 
33 #ifdef LV_HAVE_AVX
34 
35 #include <immintrin.h>
36 
37 static inline void volk_32fc_32f_dot_prod_32fc_a_avx( lv_32fc_t* result, const lv_32fc_t* input, const float* taps, unsigned int num_points) {
38 
39  unsigned int number = 0;
40  const unsigned int sixteenthPoints = num_points / 16;
41 
42  float res[2];
43  float *realpt = &res[0], *imagpt = &res[1];
44  const float* aPtr = (float*)input;
45  const float* bPtr = taps;
46 
47  __m256 a0Val, a1Val, a2Val, a3Val;
48  __m256 b0Val, b1Val, b2Val, b3Val;
49  __m256 x0Val, x1Val, x0loVal, x0hiVal, x1loVal, x1hiVal;
50  __m256 c0Val, c1Val, c2Val, c3Val;
51 
52  __m256 dotProdVal0 = _mm256_setzero_ps();
53  __m256 dotProdVal1 = _mm256_setzero_ps();
54  __m256 dotProdVal2 = _mm256_setzero_ps();
55  __m256 dotProdVal3 = _mm256_setzero_ps();
56 
57  for(;number < sixteenthPoints; number++){
58 
59  a0Val = _mm256_load_ps(aPtr);
60  a1Val = _mm256_load_ps(aPtr+8);
61  a2Val = _mm256_load_ps(aPtr+16);
62  a3Val = _mm256_load_ps(aPtr+24);
63 
64  x0Val = _mm256_load_ps(bPtr); // t0|t1|t2|t3|t4|t5|t6|t7
65  x1Val = _mm256_load_ps(bPtr+8);
66  x0loVal = _mm256_unpacklo_ps(x0Val, x0Val); // t0|t0|t1|t1|t4|t4|t5|t5
67  x0hiVal = _mm256_unpackhi_ps(x0Val, x0Val); // t2|t2|t3|t3|t6|t6|t7|t7
68  x1loVal = _mm256_unpacklo_ps(x1Val, x1Val);
69  x1hiVal = _mm256_unpackhi_ps(x1Val, x1Val);
70 
71  // TODO: it may be possible to rearrange swizzling to better pipeline data
72  b0Val = _mm256_permute2f128_ps(x0loVal, x0hiVal, 0x20); // t0|t0|t1|t1|t2|t2|t3|t3
73  b1Val = _mm256_permute2f128_ps(x0loVal, x0hiVal, 0x31); // t4|t4|t5|t5|t6|t6|t7|t7
74  b2Val = _mm256_permute2f128_ps(x1loVal, x1hiVal, 0x20);
75  b3Val = _mm256_permute2f128_ps(x1loVal, x1hiVal, 0x31);
76 
77  c0Val = _mm256_mul_ps(a0Val, b0Val);
78  c1Val = _mm256_mul_ps(a1Val, b1Val);
79  c2Val = _mm256_mul_ps(a2Val, b2Val);
80  c3Val = _mm256_mul_ps(a3Val, b3Val);
81 
82  dotProdVal0 = _mm256_add_ps(c0Val, dotProdVal0);
83  dotProdVal1 = _mm256_add_ps(c1Val, dotProdVal1);
84  dotProdVal2 = _mm256_add_ps(c2Val, dotProdVal2);
85  dotProdVal3 = _mm256_add_ps(c3Val, dotProdVal3);
86 
87  aPtr += 32;
88  bPtr += 16;
89  }
90 
91  dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal1);
92  dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal2);
93  dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal3);
94 
95  __VOLK_ATTR_ALIGNED(32) float dotProductVector[8];
96 
97  _mm256_store_ps(dotProductVector,dotProdVal0); // Store the results back into the dot product vector
98 
99  *realpt = dotProductVector[0];
100  *imagpt = dotProductVector[1];
101  *realpt += dotProductVector[2];
102  *imagpt += dotProductVector[3];
103  *realpt += dotProductVector[4];
104  *imagpt += dotProductVector[5];
105  *realpt += dotProductVector[6];
106  *imagpt += dotProductVector[7];
107 
108  number = sixteenthPoints*16;
109  for(;number < num_points; number++){
110  *realpt += ((*aPtr++) * (*bPtr));
111  *imagpt += ((*aPtr++) * (*bPtr++));
112  }
113 
114  *result = *(lv_32fc_t*)(&res[0]);
115 }
116 
117 #endif /*LV_HAVE_AVX*/
118 
119 
120 
121 
122 #ifdef LV_HAVE_SSE
123 
124 
125 static inline void volk_32fc_32f_dot_prod_32fc_a_sse( lv_32fc_t* result, const lv_32fc_t* input, const float* taps, unsigned int num_points) {
126 
127  unsigned int number = 0;
128  const unsigned int sixteenthPoints = num_points / 8;
129 
130  float res[2];
131  float *realpt = &res[0], *imagpt = &res[1];
132  const float* aPtr = (float*)input;
133  const float* bPtr = taps;
134 
135  __m128 a0Val, a1Val, a2Val, a3Val;
136  __m128 b0Val, b1Val, b2Val, b3Val;
137  __m128 x0Val, x1Val, x2Val, x3Val;
138  __m128 c0Val, c1Val, c2Val, c3Val;
139 
140  __m128 dotProdVal0 = _mm_setzero_ps();
141  __m128 dotProdVal1 = _mm_setzero_ps();
142  __m128 dotProdVal2 = _mm_setzero_ps();
143  __m128 dotProdVal3 = _mm_setzero_ps();
144 
145  for(;number < sixteenthPoints; number++){
146 
147  a0Val = _mm_load_ps(aPtr);
148  a1Val = _mm_load_ps(aPtr+4);
149  a2Val = _mm_load_ps(aPtr+8);
150  a3Val = _mm_load_ps(aPtr+12);
151 
152  x0Val = _mm_load_ps(bPtr);
153  x1Val = _mm_load_ps(bPtr);
154  x2Val = _mm_load_ps(bPtr+4);
155  x3Val = _mm_load_ps(bPtr+4);
156  b0Val = _mm_unpacklo_ps(x0Val, x1Val);
157  b1Val = _mm_unpackhi_ps(x0Val, x1Val);
158  b2Val = _mm_unpacklo_ps(x2Val, x3Val);
159  b3Val = _mm_unpackhi_ps(x2Val, x3Val);
160 
161  c0Val = _mm_mul_ps(a0Val, b0Val);
162  c1Val = _mm_mul_ps(a1Val, b1Val);
163  c2Val = _mm_mul_ps(a2Val, b2Val);
164  c3Val = _mm_mul_ps(a3Val, b3Val);
165 
166  dotProdVal0 = _mm_add_ps(c0Val, dotProdVal0);
167  dotProdVal1 = _mm_add_ps(c1Val, dotProdVal1);
168  dotProdVal2 = _mm_add_ps(c2Val, dotProdVal2);
169  dotProdVal3 = _mm_add_ps(c3Val, dotProdVal3);
170 
171  aPtr += 16;
172  bPtr += 8;
173  }
174 
175  dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal1);
176  dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal2);
177  dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal3);
178 
179  __VOLK_ATTR_ALIGNED(16) float dotProductVector[4];
180 
181  _mm_store_ps(dotProductVector,dotProdVal0); // Store the results back into the dot product vector
182 
183  *realpt = dotProductVector[0];
184  *imagpt = dotProductVector[1];
185  *realpt += dotProductVector[2];
186  *imagpt += dotProductVector[3];
187 
188  number = sixteenthPoints*8;
189  for(;number < num_points; number++){
190  *realpt += ((*aPtr++) * (*bPtr));
191  *imagpt += ((*aPtr++) * (*bPtr++));
192  }
193 
194  *result = *(lv_32fc_t*)(&res[0]);
195 }
196 
197 #endif /*LV_HAVE_SSE*/
198 
199 
200 
201 #ifdef LV_HAVE_AVX
202 
203 #include <immintrin.h>
204 
205 static inline void volk_32fc_32f_dot_prod_32fc_u_avx( lv_32fc_t* result, const lv_32fc_t* input, const float* taps, unsigned int num_points) {
206 
207  unsigned int number = 0;
208  const unsigned int sixteenthPoints = num_points / 16;
209 
210  float res[2];
211  float *realpt = &res[0], *imagpt = &res[1];
212  const float* aPtr = (float*)input;
213  const float* bPtr = taps;
214 
215  __m256 a0Val, a1Val, a2Val, a3Val;
216  __m256 b0Val, b1Val, b2Val, b3Val;
217  __m256 x0Val, x1Val, x0loVal, x0hiVal, x1loVal, x1hiVal;
218  __m256 c0Val, c1Val, c2Val, c3Val;
219 
220  __m256 dotProdVal0 = _mm256_setzero_ps();
221  __m256 dotProdVal1 = _mm256_setzero_ps();
222  __m256 dotProdVal2 = _mm256_setzero_ps();
223  __m256 dotProdVal3 = _mm256_setzero_ps();
224 
225  for(;number < sixteenthPoints; number++){
226 
227  a0Val = _mm256_loadu_ps(aPtr);
228  a1Val = _mm256_loadu_ps(aPtr+8);
229  a2Val = _mm256_loadu_ps(aPtr+16);
230  a3Val = _mm256_loadu_ps(aPtr+24);
231 
232  x0Val = _mm256_loadu_ps(bPtr); // t0|t1|t2|t3|t4|t5|t6|t7
233  x1Val = _mm256_loadu_ps(bPtr+8);
234  x0loVal = _mm256_unpacklo_ps(x0Val, x0Val); // t0|t0|t1|t1|t4|t4|t5|t5
235  x0hiVal = _mm256_unpackhi_ps(x0Val, x0Val); // t2|t2|t3|t3|t6|t6|t7|t7
236  x1loVal = _mm256_unpacklo_ps(x1Val, x1Val);
237  x1hiVal = _mm256_unpackhi_ps(x1Val, x1Val);
238 
239  // TODO: it may be possible to rearrange swizzling to better pipeline data
240  b0Val = _mm256_permute2f128_ps(x0loVal, x0hiVal, 0x20); // t0|t0|t1|t1|t2|t2|t3|t3
241  b1Val = _mm256_permute2f128_ps(x0loVal, x0hiVal, 0x31); // t4|t4|t5|t5|t6|t6|t7|t7
242  b2Val = _mm256_permute2f128_ps(x1loVal, x1hiVal, 0x20);
243  b3Val = _mm256_permute2f128_ps(x1loVal, x1hiVal, 0x31);
244 
245  c0Val = _mm256_mul_ps(a0Val, b0Val);
246  c1Val = _mm256_mul_ps(a1Val, b1Val);
247  c2Val = _mm256_mul_ps(a2Val, b2Val);
248  c3Val = _mm256_mul_ps(a3Val, b3Val);
249 
250  dotProdVal0 = _mm256_add_ps(c0Val, dotProdVal0);
251  dotProdVal1 = _mm256_add_ps(c1Val, dotProdVal1);
252  dotProdVal2 = _mm256_add_ps(c2Val, dotProdVal2);
253  dotProdVal3 = _mm256_add_ps(c3Val, dotProdVal3);
254 
255  aPtr += 32;
256  bPtr += 16;
257  }
258 
259  dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal1);
260  dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal2);
261  dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal3);
262 
263  __VOLK_ATTR_ALIGNED(32) float dotProductVector[8];
264 
265  _mm256_store_ps(dotProductVector,dotProdVal0); // Store the results back into the dot product vector
266 
267  *realpt = dotProductVector[0];
268  *imagpt = dotProductVector[1];
269  *realpt += dotProductVector[2];
270  *imagpt += dotProductVector[3];
271  *realpt += dotProductVector[4];
272  *imagpt += dotProductVector[5];
273  *realpt += dotProductVector[6];
274  *imagpt += dotProductVector[7];
275 
276  number = sixteenthPoints*16;
277  for(;number < num_points; number++){
278  *realpt += ((*aPtr++) * (*bPtr));
279  *imagpt += ((*aPtr++) * (*bPtr++));
280  }
281 
282  *result = *(lv_32fc_t*)(&res[0]);
283 }
284 #endif /*LV_HAVE_AVX*/
285 
286 
287 #ifdef LV_HAVE_SSE
288 
289 static inline void volk_32fc_32f_dot_prod_32fc_u_sse( lv_32fc_t* result, const lv_32fc_t* input, const float* taps, unsigned int num_points) {
290 
291  unsigned int number = 0;
292  const unsigned int sixteenthPoints = num_points / 8;
293 
294  float res[2];
295  float *realpt = &res[0], *imagpt = &res[1];
296  const float* aPtr = (float*)input;
297  const float* bPtr = taps;
298 
299  __m128 a0Val, a1Val, a2Val, a3Val;
300  __m128 b0Val, b1Val, b2Val, b3Val;
301  __m128 x0Val, x1Val, x2Val, x3Val;
302  __m128 c0Val, c1Val, c2Val, c3Val;
303 
304  __m128 dotProdVal0 = _mm_setzero_ps();
305  __m128 dotProdVal1 = _mm_setzero_ps();
306  __m128 dotProdVal2 = _mm_setzero_ps();
307  __m128 dotProdVal3 = _mm_setzero_ps();
308 
309  for(;number < sixteenthPoints; number++){
310 
311  a0Val = _mm_loadu_ps(aPtr);
312  a1Val = _mm_loadu_ps(aPtr+4);
313  a2Val = _mm_loadu_ps(aPtr+8);
314  a3Val = _mm_loadu_ps(aPtr+12);
315 
316  x0Val = _mm_loadu_ps(bPtr);
317  x1Val = _mm_loadu_ps(bPtr);
318  x2Val = _mm_loadu_ps(bPtr+4);
319  x3Val = _mm_loadu_ps(bPtr+4);
320  b0Val = _mm_unpacklo_ps(x0Val, x1Val);
321  b1Val = _mm_unpackhi_ps(x0Val, x1Val);
322  b2Val = _mm_unpacklo_ps(x2Val, x3Val);
323  b3Val = _mm_unpackhi_ps(x2Val, x3Val);
324 
325  c0Val = _mm_mul_ps(a0Val, b0Val);
326  c1Val = _mm_mul_ps(a1Val, b1Val);
327  c2Val = _mm_mul_ps(a2Val, b2Val);
328  c3Val = _mm_mul_ps(a3Val, b3Val);
329 
330  dotProdVal0 = _mm_add_ps(c0Val, dotProdVal0);
331  dotProdVal1 = _mm_add_ps(c1Val, dotProdVal1);
332  dotProdVal2 = _mm_add_ps(c2Val, dotProdVal2);
333  dotProdVal3 = _mm_add_ps(c3Val, dotProdVal3);
334 
335  aPtr += 16;
336  bPtr += 8;
337  }
338 
339  dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal1);
340  dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal2);
341  dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal3);
342 
343  __VOLK_ATTR_ALIGNED(16) float dotProductVector[4];
344 
345  _mm_store_ps(dotProductVector,dotProdVal0); // Store the results back into the dot product vector
346 
347  *realpt = dotProductVector[0];
348  *imagpt = dotProductVector[1];
349  *realpt += dotProductVector[2];
350  *imagpt += dotProductVector[3];
351 
352  number = sixteenthPoints*8;
353  for(;number < num_points; number++){
354  *realpt += ((*aPtr++) * (*bPtr));
355  *imagpt += ((*aPtr++) * (*bPtr++));
356  }
357 
358  *result = *(lv_32fc_t*)(&res[0]);
359 }
360 
361 #endif /*LV_HAVE_SSE*/
362 
363 
364 #endif /*INCLUDED_volk_32fc_32f_dot_prod_32fc_H*/
__VOLK_ATTR_ALIGNED
#define __VOLK_ATTR_ALIGNED(x)
Definition: volk_common.h:27
taps
static const float taps[NSTEPS+1][NTAPS]
Definition: interpolator_taps.h:9
lv_32fc_t
float complex lv_32fc_t
Definition: volk_complex.h:56
volk_common.h