Context Navigation

← Previous Revision
Next Revision →
Blame
Revision Log

source: tags/ms_r17q3/TEMPLATES/valgrind.h

Visit:

Last change on this file was 6574, checked in by westram, 15 years ago
added valgrind.h (from valgrind 3.4.1) if running under valgrind, skip callback in GBK_raises_SIGSEGV() and always assume it does raise SIGSEGV
File size: 226.9 KB

Line
1	/* -- c --
2	----------------------------------------------------------------
3
4	Notice that the following BSD-style license applies to this one
5	file (valgrind.h) only. The rest of Valgrind is licensed under the
6	terms of the GNU General Public License, version 2, unless
7	otherwise indicated. See the COPYING file in the source
8	distribution for details.
9
10	----------------------------------------------------------------
11
12	This file is part of Valgrind, a dynamic binary instrumentation
13	framework.
14
15	Copyright (C) 2000-2008 Julian Seward. All rights reserved.
16
17	Redistribution and use in source and binary forms, with or without
18	modification, are permitted provided that the following conditions
19	are met:
20
21	1. Redistributions of source code must retain the above copyright
22	notice, this list of conditions and the following disclaimer.
23
24	2. The origin of this software must not be misrepresented; you must
25	not claim that you wrote the original software. If you use this
26	software in a product, an acknowledgment in the product
27	documentation would be appreciated but is not required.
28
29	3. Altered source versions must be plainly marked as such, and must
30	not be misrepresented as being the original software.
31
32	4. The name of the author may not be used to endorse or promote
33	products derived from this software without specific prior written
34	permission.
35
36	THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
37	OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
38	WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
39	ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
40	DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
41	DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
42	GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
43	INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
44	WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
45	NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
46	SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47
48	----------------------------------------------------------------
49
50	Notice that the above BSD-style license applies to this one file
51	(valgrind.h) only. The entire rest of Valgrind is licensed under
52	the terms of the GNU General Public License, version 2. See the
53	COPYING file in the source distribution for details.
54
55	----------------------------------------------------------------
56	*/
57
58
59	/* This file is for inclusion into client (your!) code.
60
61	You can use these macros to manipulate and query Valgrind's
62	execution inside your own programs.
63
64	The resulting executables will still run without Valgrind, just a
65	little bit more slowly than they otherwise would, but otherwise
66	unchanged. When not running on valgrind, each client request
67	consumes very few (eg. 7) instructions, so the resulting performance
68	loss is negligible unless you plan to execute client requests
69	millions of times per second. Nevertheless, if that is still a
70	problem, you can compile with the NVALGRIND symbol defined (gcc
71	-DNVALGRIND) so that client requests are not even compiled in. */
72
73	#ifndef __VALGRIND_H
74	#define __VALGRIND_H
75
76	#include <stdarg.h>
77
78	/* Nb: this file might be included in a file compiled with -ansi. So
79	we can't use C++ style "//" comments nor the "asm" keyword (instead
80	use "__asm__"). */
81
82	/* Derive some tags indicating what the target platform is. Note
83	that in this file we're using the compiler's CPP symbols for
84	identifying architectures, which are different to the ones we use
85	within the rest of Valgrind. Note, __powerpc__ is active for both
86	32 and 64-bit PPC, whereas __powerpc64__ is only active for the
87	latter (on Linux, that is). */
88	#undef PLAT_x86_linux
89	#undef PLAT_amd64_linux
90	#undef PLAT_ppc32_linux
91	#undef PLAT_ppc64_linux
92	#undef PLAT_ppc32_aix5
93	#undef PLAT_ppc64_aix5
94
95	#if !defined(_AIX) && defined(__i386__)
96	# define PLAT_x86_linux 1
97	#elif !defined(_AIX) && defined(__x86_64__)
98	# define PLAT_amd64_linux 1
99	#elif !defined(_AIX) && defined(__powerpc__) && !defined(__powerpc64__)
100	# define PLAT_ppc32_linux 1
101	#elif !defined(_AIX) && defined(__powerpc__) && defined(__powerpc64__)
102	# define PLAT_ppc64_linux 1
103	#elif defined(_AIX) && defined(__64BIT__)
104	# define PLAT_ppc64_aix5 1
105	#elif defined(_AIX) && !defined(__64BIT__)
106	# define PLAT_ppc32_aix5 1
107	#endif
108
109
110	/* If we're not compiling for our target platform, don't generate
111	any inline asms. */
112	#if !defined(PLAT_x86_linux) && !defined(PLAT_amd64_linux) \
113	&& !defined(PLAT_ppc32_linux) && !defined(PLAT_ppc64_linux) \
114	&& !defined(PLAT_ppc32_aix5) && !defined(PLAT_ppc64_aix5)
115	# if !defined(NVALGRIND)
116	# define NVALGRIND 1
117	# endif
118	#endif
119
120
121	/* ------------------------------------------------------------------ */
122	/* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS. There is nothing */
123	/* in here of use to end-users -- skip to the next section. */
124	/* ------------------------------------------------------------------ */
125
126	#if defined(NVALGRIND)
127
128	/* Define NVALGRIND to completely remove the Valgrind magic sequence
129	from the compiled code (analogous to NDEBUG's effects on
130	assert()) */
131	#define VALGRIND_DO_CLIENT_REQUEST( \
132	_zzq_rlval, _zzq_default, _zzq_request, \
133	_zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
134	{ \
135	(_zzq_rlval) = (_zzq_default); \
136	}
137
138	#else /* ! NVALGRIND */
139
140	/* The following defines the magic code sequences which the JITter
141	spots and handles magically. Don't look too closely at them as
142	they will rot your brain.
143
144	The assembly code sequences for all architectures is in this one
145	file. This is because this file must be stand-alone, and we don't
146	want to have multiple files.
147
148	For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
149	value gets put in the return slot, so that everything works when
150	this is executed not under Valgrind. Args are passed in a memory
151	block, and so there's no intrinsic limit to the number that could
152	be passed, but it's currently five.
153
154	The macro args are:
155	_zzq_rlval result lvalue
156	_zzq_default default value (result returned when running on real CPU)
157	_zzq_request request code
158	_zzq_arg1..5 request params
159
160	The other two macros are used to support function wrapping, and are
161	a lot simpler. VALGRIND_GET_NR_CONTEXT returns the value of the
162	guest's NRADDR pseudo-register and whatever other information is
163	needed to safely run the call original from the wrapper: on
164	ppc64-linux, the R2 value at the divert point is also needed. This
165	information is abstracted into a user-visible type, OrigFn.
166
167	VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
168	guest, but guarantees that the branch instruction will not be
169	redirected: x86: call %eax, amd64: call %rax, ppc32/ppc64:
170	branch-and-link-to-r11. VALGRIND_CALL_NOREDIR is just text, not a
171	complete inline asm, since it needs to be combined with more magic
172	inline asm stuff to be useful.
173	*/
174
175	/* ------------------------- x86-linux ------------------------- */
176
177	#if defined(PLAT_x86_linux)
178
179	typedef
180	struct {
181	unsigned int nraddr; /* where's the code? */
182	}
183	OrigFn;
184
185	#define __SPECIAL_INSTRUCTION_PREAMBLE \
186	"roll $3, %%edi ; roll $13, %%edi\n\t" \
187	"roll $29, %%edi ; roll $19, %%edi\n\t"
188
189	#define VALGRIND_DO_CLIENT_REQUEST( \
190	_zzq_rlval, _zzq_default, _zzq_request, \
191	_zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
192	{ volatile unsigned int _zzq_args[6]; \
193	volatile unsigned int _zzq_result; \
194	_zzq_args[0] = (unsigned int)(_zzq_request); \
195	_zzq_args[1] = (unsigned int)(_zzq_arg1); \
196	_zzq_args[2] = (unsigned int)(_zzq_arg2); \
197	_zzq_args[3] = (unsigned int)(_zzq_arg3); \
198	_zzq_args[4] = (unsigned int)(_zzq_arg4); \
199	_zzq_args[5] = (unsigned int)(_zzq_arg5); \
200	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
201	/* %EDX = client_request ( %EAX ) */ \
202	"xchgl %%ebx,%%ebx" \
203	: "=d" (_zzq_result) \
204	: "a" (&_zzq_args[0]), "0" (_zzq_default) \
205	: "cc", "memory" \
206	); \
207	_zzq_rlval = _zzq_result; \
208	}
209
210	#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
211	{ volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
212	volatile unsigned int __addr; \
213	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
214	/* %EAX = guest_NRADDR */ \
215	"xchgl %%ecx,%%ecx" \
216	: "=a" (__addr) \
217	: \
218	: "cc", "memory" \
219	); \
220	_zzq_orig->nraddr = __addr; \
221	}
222
223	#define VALGRIND_CALL_NOREDIR_EAX \
224	__SPECIAL_INSTRUCTION_PREAMBLE \
225	/* call-noredir %EAX / \
226	"xchgl %%edx,%%edx\n\t"
227	#endif /* PLAT_x86_linux */
228
229	/* ------------------------ amd64-linux ------------------------ */
230
231	#if defined(PLAT_amd64_linux)
232
233	typedef
234	struct {
235	unsigned long long int nraddr; /* where's the code? */
236	}
237	OrigFn;
238
239	#define __SPECIAL_INSTRUCTION_PREAMBLE \
240	"rolq $3, %%rdi ; rolq $13, %%rdi\n\t" \
241	"rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
242
243	#define VALGRIND_DO_CLIENT_REQUEST( \
244	_zzq_rlval, _zzq_default, _zzq_request, \
245	_zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
246	{ volatile unsigned long long int _zzq_args[6]; \
247	volatile unsigned long long int _zzq_result; \
248	_zzq_args[0] = (unsigned long long int)(_zzq_request); \
249	_zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
250	_zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
251	_zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
252	_zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
253	_zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
254	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
255	/* %RDX = client_request ( %RAX ) */ \
256	"xchgq %%rbx,%%rbx" \
257	: "=d" (_zzq_result) \
258	: "a" (&_zzq_args[0]), "0" (_zzq_default) \
259	: "cc", "memory" \
260	); \
261	_zzq_rlval = _zzq_result; \
262	}
263
264	#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
265	{ volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
266	volatile unsigned long long int __addr; \
267	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
268	/* %RAX = guest_NRADDR */ \
269	"xchgq %%rcx,%%rcx" \
270	: "=a" (__addr) \
271	: \
272	: "cc", "memory" \
273	); \
274	_zzq_orig->nraddr = __addr; \
275	}
276
277	#define VALGRIND_CALL_NOREDIR_RAX \
278	__SPECIAL_INSTRUCTION_PREAMBLE \
279	/* call-noredir %RAX / \
280	"xchgq %%rdx,%%rdx\n\t"
281	#endif /* PLAT_amd64_linux */
282
283	/* ------------------------ ppc32-linux ------------------------ */
284
285	#if defined(PLAT_ppc32_linux)
286
287	typedef
288	struct {
289	unsigned int nraddr; /* where's the code? */
290	}
291	OrigFn;
292
293	#define __SPECIAL_INSTRUCTION_PREAMBLE \
294	"rlwinm 0,0,3,0,0 ; rlwinm 0,0,13,0,0\n\t" \
295	"rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
296
297	#define VALGRIND_DO_CLIENT_REQUEST( \
298	_zzq_rlval, _zzq_default, _zzq_request, \
299	_zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
300	\
301	{ unsigned int _zzq_args[6]; \
302	unsigned int _zzq_result; \
303	unsigned int* _zzq_ptr; \
304	_zzq_args[0] = (unsigned int)(_zzq_request); \
305	_zzq_args[1] = (unsigned int)(_zzq_arg1); \
306	_zzq_args[2] = (unsigned int)(_zzq_arg2); \
307	_zzq_args[3] = (unsigned int)(_zzq_arg3); \
308	_zzq_args[4] = (unsigned int)(_zzq_arg4); \
309	_zzq_args[5] = (unsigned int)(_zzq_arg5); \
310	_zzq_ptr = _zzq_args; \
311	__asm__ volatile("mr 3,%1\n\t" /default/ \
312	"mr 4,%2\n\t" /ptr/ \
313	__SPECIAL_INSTRUCTION_PREAMBLE \
314	/* %R3 = client_request ( %R4 ) */ \
315	"or 1,1,1\n\t" \
316	"mr %0,3" /result/ \
317	: "=b" (_zzq_result) \
318	: "b" (_zzq_default), "b" (_zzq_ptr) \
319	: "cc", "memory", "r3", "r4"); \
320	_zzq_rlval = _zzq_result; \
321	}
322
323	#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
324	{ volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
325	unsigned int __addr; \
326	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
327	/* %R3 = guest_NRADDR */ \
328	"or 2,2,2\n\t" \
329	"mr %0,3" \
330	: "=b" (__addr) \
331	: \
332	: "cc", "memory", "r3" \
333	); \
334	_zzq_orig->nraddr = __addr; \
335	}
336
337	#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
338	__SPECIAL_INSTRUCTION_PREAMBLE \
339	/* branch-and-link-to-noredir %R11 / \
340	"or 3,3,3\n\t"
341	#endif /* PLAT_ppc32_linux */
342
343	/* ------------------------ ppc64-linux ------------------------ */
344
345	#if defined(PLAT_ppc64_linux)
346
347	typedef
348	struct {
349	unsigned long long int nraddr; /* where's the code? */
350	unsigned long long int r2; /* what tocptr do we need? */
351	}
352	OrigFn;
353
354	#define __SPECIAL_INSTRUCTION_PREAMBLE \
355	"rotldi 0,0,3 ; rotldi 0,0,13\n\t" \
356	"rotldi 0,0,61 ; rotldi 0,0,51\n\t"
357
358	#define VALGRIND_DO_CLIENT_REQUEST( \
359	_zzq_rlval, _zzq_default, _zzq_request, \
360	_zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
361	\
362	{ unsigned long long int _zzq_args[6]; \
363	register unsigned long long int _zzq_result __asm__("r3"); \
364	register unsigned long long int* _zzq_ptr __asm__("r4"); \
365	_zzq_args[0] = (unsigned long long int)(_zzq_request); \
366	_zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
367	_zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
368	_zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
369	_zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
370	_zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
371	_zzq_ptr = _zzq_args; \
372	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
373	/* %R3 = client_request ( %R4 ) */ \
374	"or 1,1,1" \
375	: "=r" (_zzq_result) \
376	: "0" (_zzq_default), "r" (_zzq_ptr) \
377	: "cc", "memory"); \
378	_zzq_rlval = _zzq_result; \
379	}
380
381	#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
382	{ volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
383	register unsigned long long int __addr __asm__("r3"); \
384	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
385	/* %R3 = guest_NRADDR */ \
386	"or 2,2,2" \
387	: "=r" (__addr) \
388	: \
389	: "cc", "memory" \
390	); \
391	_zzq_orig->nraddr = __addr; \
392	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
393	/* %R3 = guest_NRADDR_GPR2 */ \
394	"or 4,4,4" \
395	: "=r" (__addr) \
396	: \
397	: "cc", "memory" \
398	); \
399	_zzq_orig->r2 = __addr; \
400	}
401
402	#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
403	__SPECIAL_INSTRUCTION_PREAMBLE \
404	/* branch-and-link-to-noredir %R11 / \
405	"or 3,3,3\n\t"
406
407	#endif /* PLAT_ppc64_linux */
408
409	/* ------------------------ ppc32-aix5 ------------------------- */
410
411	#if defined(PLAT_ppc32_aix5)
412
413	typedef
414	struct {
415	unsigned int nraddr; /* where's the code? */
416	unsigned int r2; /* what tocptr do we need? */
417	}
418	OrigFn;
419
420	#define __SPECIAL_INSTRUCTION_PREAMBLE \
421	"rlwinm 0,0,3,0,0 ; rlwinm 0,0,13,0,0\n\t" \
422	"rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
423
424	#define VALGRIND_DO_CLIENT_REQUEST( \
425	_zzq_rlval, _zzq_default, _zzq_request, \
426	_zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
427	\
428	{ unsigned int _zzq_args[7]; \
429	register unsigned int _zzq_result; \
430	register unsigned int* _zzq_ptr; \
431	_zzq_args[0] = (unsigned int)(_zzq_request); \
432	_zzq_args[1] = (unsigned int)(_zzq_arg1); \
433	_zzq_args[2] = (unsigned int)(_zzq_arg2); \
434	_zzq_args[3] = (unsigned int)(_zzq_arg3); \
435	_zzq_args[4] = (unsigned int)(_zzq_arg4); \
436	_zzq_args[5] = (unsigned int)(_zzq_arg5); \
437	_zzq_args[6] = (unsigned int)(_zzq_default); \
438	_zzq_ptr = _zzq_args; \
439	__asm__ volatile("mr 4,%1\n\t" \
440	"lwz 3, 24(4)\n\t" \
441	__SPECIAL_INSTRUCTION_PREAMBLE \
442	/* %R3 = client_request ( %R4 ) */ \
443	"or 1,1,1\n\t" \
444	"mr %0,3" \
445	: "=b" (_zzq_result) \
446	: "b" (_zzq_ptr) \
447	: "r3", "r4", "cc", "memory"); \
448	_zzq_rlval = _zzq_result; \
449	}
450
451	#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
452	{ volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
453	register unsigned int __addr; \
454	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
455	/* %R3 = guest_NRADDR */ \
456	"or 2,2,2\n\t" \
457	"mr %0,3" \
458	: "=b" (__addr) \
459	: \
460	: "r3", "cc", "memory" \
461	); \
462	_zzq_orig->nraddr = __addr; \
463	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
464	/* %R3 = guest_NRADDR_GPR2 */ \
465	"or 4,4,4\n\t" \
466	"mr %0,3" \
467	: "=b" (__addr) \
468	: \
469	: "r3", "cc", "memory" \
470	); \
471	_zzq_orig->r2 = __addr; \
472	}
473
474	#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
475	__SPECIAL_INSTRUCTION_PREAMBLE \
476	/* branch-and-link-to-noredir %R11 / \
477	"or 3,3,3\n\t"
478
479	#endif /* PLAT_ppc32_aix5 */
480
481	/* ------------------------ ppc64-aix5 ------------------------- */
482
483	#if defined(PLAT_ppc64_aix5)
484
485	typedef
486	struct {
487	unsigned long long int nraddr; /* where's the code? */
488	unsigned long long int r2; /* what tocptr do we need? */
489	}
490	OrigFn;
491
492	#define __SPECIAL_INSTRUCTION_PREAMBLE \
493	"rotldi 0,0,3 ; rotldi 0,0,13\n\t" \
494	"rotldi 0,0,61 ; rotldi 0,0,51\n\t"
495
496	#define VALGRIND_DO_CLIENT_REQUEST( \
497	_zzq_rlval, _zzq_default, _zzq_request, \
498	_zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
499	\
500	{ unsigned long long int _zzq_args[7]; \
501	register unsigned long long int _zzq_result; \
502	register unsigned long long int* _zzq_ptr; \
503	_zzq_args[0] = (unsigned int long long)(_zzq_request); \
504	_zzq_args[1] = (unsigned int long long)(_zzq_arg1); \
505	_zzq_args[2] = (unsigned int long long)(_zzq_arg2); \
506	_zzq_args[3] = (unsigned int long long)(_zzq_arg3); \
507	_zzq_args[4] = (unsigned int long long)(_zzq_arg4); \
508	_zzq_args[5] = (unsigned int long long)(_zzq_arg5); \
509	_zzq_args[6] = (unsigned int long long)(_zzq_default); \
510	_zzq_ptr = _zzq_args; \
511	__asm__ volatile("mr 4,%1\n\t" \
512	"ld 3, 48(4)\n\t" \
513	__SPECIAL_INSTRUCTION_PREAMBLE \
514	/* %R3 = client_request ( %R4 ) */ \
515	"or 1,1,1\n\t" \
516	"mr %0,3" \
517	: "=b" (_zzq_result) \
518	: "b" (_zzq_ptr) \
519	: "r3", "r4", "cc", "memory"); \
520	_zzq_rlval = _zzq_result; \
521	}
522
523	#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
524	{ volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
525	register unsigned long long int __addr; \
526	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
527	/* %R3 = guest_NRADDR */ \
528	"or 2,2,2\n\t" \
529	"mr %0,3" \
530	: "=b" (__addr) \
531	: \
532	: "r3", "cc", "memory" \
533	); \
534	_zzq_orig->nraddr = __addr; \
535	__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
536	/* %R3 = guest_NRADDR_GPR2 */ \
537	"or 4,4,4\n\t" \
538	"mr %0,3" \
539	: "=b" (__addr) \
540	: \
541	: "r3", "cc", "memory" \
542	); \
543	_zzq_orig->r2 = __addr; \
544	}
545
546	#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
547	__SPECIAL_INSTRUCTION_PREAMBLE \
548	/* branch-and-link-to-noredir %R11 / \
549	"or 3,3,3\n\t"
550
551	#endif /* PLAT_ppc64_aix5 */
552
553	/* Insert assembly code for other platforms here... */
554
555	#endif /* NVALGRIND */
556
557
558	/* ------------------------------------------------------------------ */
559	/* PLATFORM SPECIFICS for FUNCTION WRAPPING. This is all very */
560	/* ugly. It's the least-worst tradeoff I can think of. */
561	/* ------------------------------------------------------------------ */
562
563	/* This section defines magic (a.k.a appalling-hack) macros for doing
564	guaranteed-no-redirection macros, so as to get from function
565	wrappers to the functions they are wrapping. The whole point is to
566	construct standard call sequences, but to do the call itself with a
567	special no-redirect call pseudo-instruction that the JIT
568	understands and handles specially. This section is long and
569	repetitious, and I can't see a way to make it shorter.
570
571	The naming scheme is as follows:
572
573	CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
574
575	'W' stands for "word" and 'v' for "void". Hence there are
576	different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
577	and for each, the possibility of returning a word-typed result, or
578	no result.
579	*/
580
581	/* Use these to write the name of your wrapper. NOTE: duplicates
582	VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
583
584	#define I_WRAP_SONAME_FNNAME_ZU(soname,fnname) \
585	_vgwZU_##soname##_##fnname
586
587	#define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname) \
588	_vgwZZ_##soname##_##fnname
589
590	/* Use this macro from within a wrapper function to collect the
591	context (address and possibly other info) of the original function.
592	Once you have that you can then use it in one of the CALL_FN_
593	macros. The type of the argument _lval is OrigFn. */
594	#define VALGRIND_GET_ORIG_FN(_lval) VALGRIND_GET_NR_CONTEXT(_lval)
595
596	/* Derivatives of the main macros below, for calling functions
597	returning void. */
598
599	#define CALL_FN_v_v(fnptr) \
600	do { volatile unsigned long _junk; \
601	CALL_FN_W_v(_junk,fnptr); } while (0)
602
603	#define CALL_FN_v_W(fnptr, arg1) \
604	do { volatile unsigned long _junk; \
605	CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
606
607	#define CALL_FN_v_WW(fnptr, arg1,arg2) \
608	do { volatile unsigned long _junk; \
609	CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
610
611	#define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3) \
612	do { volatile unsigned long _junk; \
613	CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
614
615	/* ------------------------- x86-linux ------------------------- */
616
617	#if defined(PLAT_x86_linux)
618
619	/* These regs are trashed by the hidden call. No need to mention eax
620	as gcc can already see that, plus causes gcc to bomb. */
621	#define __CALLER_SAVED_REGS /"eax"/ "ecx", "edx"
622
623	/* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
624	long) == 4. */
625
626	#define CALL_FN_W_v(lval, orig) \
627	do { \
628	volatile OrigFn _orig = (orig); \
629	volatile unsigned long _argvec[1]; \
630	volatile unsigned long _res; \
631	_argvec[0] = (unsigned long)_orig.nraddr; \
632	__asm__ volatile( \
633	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
634	VALGRIND_CALL_NOREDIR_EAX \
635	: /out/ "=a" (_res) \
636	: /in/ "a" (&_argvec[0]) \
637	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
638	); \
639	lval = (__typeof__(lval)) _res; \
640	} while (0)
641
642	#define CALL_FN_W_W(lval, orig, arg1) \
643	do { \
644	volatile OrigFn _orig = (orig); \
645	volatile unsigned long _argvec[2]; \
646	volatile unsigned long _res; \
647	_argvec[0] = (unsigned long)_orig.nraddr; \
648	_argvec[1] = (unsigned long)(arg1); \
649	__asm__ volatile( \
650	"pushl 4(%%eax)\n\t" \
651	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
652	VALGRIND_CALL_NOREDIR_EAX \
653	"addl $4, %%esp\n" \
654	: /out/ "=a" (_res) \
655	: /in/ "a" (&_argvec[0]) \
656	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
657	); \
658	lval = (__typeof__(lval)) _res; \
659	} while (0)
660
661	#define CALL_FN_W_WW(lval, orig, arg1,arg2) \
662	do { \
663	volatile OrigFn _orig = (orig); \
664	volatile unsigned long _argvec[3]; \
665	volatile unsigned long _res; \
666	_argvec[0] = (unsigned long)_orig.nraddr; \
667	_argvec[1] = (unsigned long)(arg1); \
668	_argvec[2] = (unsigned long)(arg2); \
669	__asm__ volatile( \
670	"pushl 8(%%eax)\n\t" \
671	"pushl 4(%%eax)\n\t" \
672	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
673	VALGRIND_CALL_NOREDIR_EAX \
674	"addl $8, %%esp\n" \
675	: /out/ "=a" (_res) \
676	: /in/ "a" (&_argvec[0]) \
677	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
678	); \
679	lval = (__typeof__(lval)) _res; \
680	} while (0)
681
682	#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
683	do { \
684	volatile OrigFn _orig = (orig); \
685	volatile unsigned long _argvec[4]; \
686	volatile unsigned long _res; \
687	_argvec[0] = (unsigned long)_orig.nraddr; \
688	_argvec[1] = (unsigned long)(arg1); \
689	_argvec[2] = (unsigned long)(arg2); \
690	_argvec[3] = (unsigned long)(arg3); \
691	__asm__ volatile( \
692	"pushl 12(%%eax)\n\t" \
693	"pushl 8(%%eax)\n\t" \
694	"pushl 4(%%eax)\n\t" \
695	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
696	VALGRIND_CALL_NOREDIR_EAX \
697	"addl $12, %%esp\n" \
698	: /out/ "=a" (_res) \
699	: /in/ "a" (&_argvec[0]) \
700	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
701	); \
702	lval = (__typeof__(lval)) _res; \
703	} while (0)
704
705	#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
706	do { \
707	volatile OrigFn _orig = (orig); \
708	volatile unsigned long _argvec[5]; \
709	volatile unsigned long _res; \
710	_argvec[0] = (unsigned long)_orig.nraddr; \
711	_argvec[1] = (unsigned long)(arg1); \
712	_argvec[2] = (unsigned long)(arg2); \
713	_argvec[3] = (unsigned long)(arg3); \
714	_argvec[4] = (unsigned long)(arg4); \
715	__asm__ volatile( \
716	"pushl 16(%%eax)\n\t" \
717	"pushl 12(%%eax)\n\t" \
718	"pushl 8(%%eax)\n\t" \
719	"pushl 4(%%eax)\n\t" \
720	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
721	VALGRIND_CALL_NOREDIR_EAX \
722	"addl $16, %%esp\n" \
723	: /out/ "=a" (_res) \
724	: /in/ "a" (&_argvec[0]) \
725	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
726	); \
727	lval = (__typeof__(lval)) _res; \
728	} while (0)
729
730	#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
731	do { \
732	volatile OrigFn _orig = (orig); \
733	volatile unsigned long _argvec[6]; \
734	volatile unsigned long _res; \
735	_argvec[0] = (unsigned long)_orig.nraddr; \
736	_argvec[1] = (unsigned long)(arg1); \
737	_argvec[2] = (unsigned long)(arg2); \
738	_argvec[3] = (unsigned long)(arg3); \
739	_argvec[4] = (unsigned long)(arg4); \
740	_argvec[5] = (unsigned long)(arg5); \
741	__asm__ volatile( \
742	"pushl 20(%%eax)\n\t" \
743	"pushl 16(%%eax)\n\t" \
744	"pushl 12(%%eax)\n\t" \
745	"pushl 8(%%eax)\n\t" \
746	"pushl 4(%%eax)\n\t" \
747	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
748	VALGRIND_CALL_NOREDIR_EAX \
749	"addl $20, %%esp\n" \
750	: /out/ "=a" (_res) \
751	: /in/ "a" (&_argvec[0]) \
752	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
753	); \
754	lval = (__typeof__(lval)) _res; \
755	} while (0)
756
757	#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
758	do { \
759	volatile OrigFn _orig = (orig); \
760	volatile unsigned long _argvec[7]; \
761	volatile unsigned long _res; \
762	_argvec[0] = (unsigned long)_orig.nraddr; \
763	_argvec[1] = (unsigned long)(arg1); \
764	_argvec[2] = (unsigned long)(arg2); \
765	_argvec[3] = (unsigned long)(arg3); \
766	_argvec[4] = (unsigned long)(arg4); \
767	_argvec[5] = (unsigned long)(arg5); \
768	_argvec[6] = (unsigned long)(arg6); \
769	__asm__ volatile( \
770	"pushl 24(%%eax)\n\t" \
771	"pushl 20(%%eax)\n\t" \
772	"pushl 16(%%eax)\n\t" \
773	"pushl 12(%%eax)\n\t" \
774	"pushl 8(%%eax)\n\t" \
775	"pushl 4(%%eax)\n\t" \
776	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
777	VALGRIND_CALL_NOREDIR_EAX \
778	"addl $24, %%esp\n" \
779	: /out/ "=a" (_res) \
780	: /in/ "a" (&_argvec[0]) \
781	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
782	); \
783	lval = (__typeof__(lval)) _res; \
784	} while (0)
785
786	#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
787	arg7) \
788	do { \
789	volatile OrigFn _orig = (orig); \
790	volatile unsigned long _argvec[8]; \
791	volatile unsigned long _res; \
792	_argvec[0] = (unsigned long)_orig.nraddr; \
793	_argvec[1] = (unsigned long)(arg1); \
794	_argvec[2] = (unsigned long)(arg2); \
795	_argvec[3] = (unsigned long)(arg3); \
796	_argvec[4] = (unsigned long)(arg4); \
797	_argvec[5] = (unsigned long)(arg5); \
798	_argvec[6] = (unsigned long)(arg6); \
799	_argvec[7] = (unsigned long)(arg7); \
800	__asm__ volatile( \
801	"pushl 28(%%eax)\n\t" \
802	"pushl 24(%%eax)\n\t" \
803	"pushl 20(%%eax)\n\t" \
804	"pushl 16(%%eax)\n\t" \
805	"pushl 12(%%eax)\n\t" \
806	"pushl 8(%%eax)\n\t" \
807	"pushl 4(%%eax)\n\t" \
808	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
809	VALGRIND_CALL_NOREDIR_EAX \
810	"addl $28, %%esp\n" \
811	: /out/ "=a" (_res) \
812	: /in/ "a" (&_argvec[0]) \
813	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
814	); \
815	lval = (__typeof__(lval)) _res; \
816	} while (0)
817
818	#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
819	arg7,arg8) \
820	do { \
821	volatile OrigFn _orig = (orig); \
822	volatile unsigned long _argvec[9]; \
823	volatile unsigned long _res; \
824	_argvec[0] = (unsigned long)_orig.nraddr; \
825	_argvec[1] = (unsigned long)(arg1); \
826	_argvec[2] = (unsigned long)(arg2); \
827	_argvec[3] = (unsigned long)(arg3); \
828	_argvec[4] = (unsigned long)(arg4); \
829	_argvec[5] = (unsigned long)(arg5); \
830	_argvec[6] = (unsigned long)(arg6); \
831	_argvec[7] = (unsigned long)(arg7); \
832	_argvec[8] = (unsigned long)(arg8); \
833	__asm__ volatile( \
834	"pushl 32(%%eax)\n\t" \
835	"pushl 28(%%eax)\n\t" \
836	"pushl 24(%%eax)\n\t" \
837	"pushl 20(%%eax)\n\t" \
838	"pushl 16(%%eax)\n\t" \
839	"pushl 12(%%eax)\n\t" \
840	"pushl 8(%%eax)\n\t" \
841	"pushl 4(%%eax)\n\t" \
842	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
843	VALGRIND_CALL_NOREDIR_EAX \
844	"addl $32, %%esp\n" \
845	: /out/ "=a" (_res) \
846	: /in/ "a" (&_argvec[0]) \
847	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
848	); \
849	lval = (__typeof__(lval)) _res; \
850	} while (0)
851
852	#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
853	arg7,arg8,arg9) \
854	do { \
855	volatile OrigFn _orig = (orig); \
856	volatile unsigned long _argvec[10]; \
857	volatile unsigned long _res; \
858	_argvec[0] = (unsigned long)_orig.nraddr; \
859	_argvec[1] = (unsigned long)(arg1); \
860	_argvec[2] = (unsigned long)(arg2); \
861	_argvec[3] = (unsigned long)(arg3); \
862	_argvec[4] = (unsigned long)(arg4); \
863	_argvec[5] = (unsigned long)(arg5); \
864	_argvec[6] = (unsigned long)(arg6); \
865	_argvec[7] = (unsigned long)(arg7); \
866	_argvec[8] = (unsigned long)(arg8); \
867	_argvec[9] = (unsigned long)(arg9); \
868	__asm__ volatile( \
869	"pushl 36(%%eax)\n\t" \
870	"pushl 32(%%eax)\n\t" \
871	"pushl 28(%%eax)\n\t" \
872	"pushl 24(%%eax)\n\t" \
873	"pushl 20(%%eax)\n\t" \
874	"pushl 16(%%eax)\n\t" \
875	"pushl 12(%%eax)\n\t" \
876	"pushl 8(%%eax)\n\t" \
877	"pushl 4(%%eax)\n\t" \
878	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
879	VALGRIND_CALL_NOREDIR_EAX \
880	"addl $36, %%esp\n" \
881	: /out/ "=a" (_res) \
882	: /in/ "a" (&_argvec[0]) \
883	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
884	); \
885	lval = (__typeof__(lval)) _res; \
886	} while (0)
887
888	#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
889	arg7,arg8,arg9,arg10) \
890	do { \
891	volatile OrigFn _orig = (orig); \
892	volatile unsigned long _argvec[11]; \
893	volatile unsigned long _res; \
894	_argvec[0] = (unsigned long)_orig.nraddr; \
895	_argvec[1] = (unsigned long)(arg1); \
896	_argvec[2] = (unsigned long)(arg2); \
897	_argvec[3] = (unsigned long)(arg3); \
898	_argvec[4] = (unsigned long)(arg4); \
899	_argvec[5] = (unsigned long)(arg5); \
900	_argvec[6] = (unsigned long)(arg6); \
901	_argvec[7] = (unsigned long)(arg7); \
902	_argvec[8] = (unsigned long)(arg8); \
903	_argvec[9] = (unsigned long)(arg9); \
904	_argvec[10] = (unsigned long)(arg10); \
905	__asm__ volatile( \
906	"pushl 40(%%eax)\n\t" \
907	"pushl 36(%%eax)\n\t" \
908	"pushl 32(%%eax)\n\t" \
909	"pushl 28(%%eax)\n\t" \
910	"pushl 24(%%eax)\n\t" \
911	"pushl 20(%%eax)\n\t" \
912	"pushl 16(%%eax)\n\t" \
913	"pushl 12(%%eax)\n\t" \
914	"pushl 8(%%eax)\n\t" \
915	"pushl 4(%%eax)\n\t" \
916	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
917	VALGRIND_CALL_NOREDIR_EAX \
918	"addl $40, %%esp\n" \
919	: /out/ "=a" (_res) \
920	: /in/ "a" (&_argvec[0]) \
921	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
922	); \
923	lval = (__typeof__(lval)) _res; \
924	} while (0)
925
926	#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
927	arg6,arg7,arg8,arg9,arg10, \
928	arg11) \
929	do { \
930	volatile OrigFn _orig = (orig); \
931	volatile unsigned long _argvec[12]; \
932	volatile unsigned long _res; \
933	_argvec[0] = (unsigned long)_orig.nraddr; \
934	_argvec[1] = (unsigned long)(arg1); \
935	_argvec[2] = (unsigned long)(arg2); \
936	_argvec[3] = (unsigned long)(arg3); \
937	_argvec[4] = (unsigned long)(arg4); \
938	_argvec[5] = (unsigned long)(arg5); \
939	_argvec[6] = (unsigned long)(arg6); \
940	_argvec[7] = (unsigned long)(arg7); \
941	_argvec[8] = (unsigned long)(arg8); \
942	_argvec[9] = (unsigned long)(arg9); \
943	_argvec[10] = (unsigned long)(arg10); \
944	_argvec[11] = (unsigned long)(arg11); \
945	__asm__ volatile( \
946	"pushl 44(%%eax)\n\t" \
947	"pushl 40(%%eax)\n\t" \
948	"pushl 36(%%eax)\n\t" \
949	"pushl 32(%%eax)\n\t" \
950	"pushl 28(%%eax)\n\t" \
951	"pushl 24(%%eax)\n\t" \
952	"pushl 20(%%eax)\n\t" \
953	"pushl 16(%%eax)\n\t" \
954	"pushl 12(%%eax)\n\t" \
955	"pushl 8(%%eax)\n\t" \
956	"pushl 4(%%eax)\n\t" \
957	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
958	VALGRIND_CALL_NOREDIR_EAX \
959	"addl $44, %%esp\n" \
960	: /out/ "=a" (_res) \
961	: /in/ "a" (&_argvec[0]) \
962	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
963	); \
964	lval = (__typeof__(lval)) _res; \
965	} while (0)
966
967	#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
968	arg6,arg7,arg8,arg9,arg10, \
969	arg11,arg12) \
970	do { \
971	volatile OrigFn _orig = (orig); \
972	volatile unsigned long _argvec[13]; \
973	volatile unsigned long _res; \
974	_argvec[0] = (unsigned long)_orig.nraddr; \
975	_argvec[1] = (unsigned long)(arg1); \
976	_argvec[2] = (unsigned long)(arg2); \
977	_argvec[3] = (unsigned long)(arg3); \
978	_argvec[4] = (unsigned long)(arg4); \
979	_argvec[5] = (unsigned long)(arg5); \
980	_argvec[6] = (unsigned long)(arg6); \
981	_argvec[7] = (unsigned long)(arg7); \
982	_argvec[8] = (unsigned long)(arg8); \
983	_argvec[9] = (unsigned long)(arg9); \
984	_argvec[10] = (unsigned long)(arg10); \
985	_argvec[11] = (unsigned long)(arg11); \
986	_argvec[12] = (unsigned long)(arg12); \
987	__asm__ volatile( \
988	"pushl 48(%%eax)\n\t" \
989	"pushl 44(%%eax)\n\t" \
990	"pushl 40(%%eax)\n\t" \
991	"pushl 36(%%eax)\n\t" \
992	"pushl 32(%%eax)\n\t" \
993	"pushl 28(%%eax)\n\t" \
994	"pushl 24(%%eax)\n\t" \
995	"pushl 20(%%eax)\n\t" \
996	"pushl 16(%%eax)\n\t" \
997	"pushl 12(%%eax)\n\t" \
998	"pushl 8(%%eax)\n\t" \
999	"pushl 4(%%eax)\n\t" \
1000	"movl (%%eax), %%eax\n\t" /* target->%eax */ \
1001	VALGRIND_CALL_NOREDIR_EAX \
1002	"addl $48, %%esp\n" \
1003	: /out/ "=a" (_res) \
1004	: /in/ "a" (&_argvec[0]) \
1005	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1006	); \
1007	lval = (__typeof__(lval)) _res; \
1008	} while (0)
1009
1010	#endif /* PLAT_x86_linux */
1011
1012	/* ------------------------ amd64-linux ------------------------ */
1013
1014	#if defined(PLAT_amd64_linux)
1015
1016	/* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1017
1018	/* These regs are trashed by the hidden call. */
1019	#define __CALLER_SAVED_REGS /"rax",/ "rcx", "rdx", "rsi", \
1020	"rdi", "r8", "r9", "r10", "r11"
1021
1022	/* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1023	long) == 8. */
1024
1025	/* NB 9 Sept 07. There is a nasty kludge here in all these CALL_FN_
1026	macros. In order not to trash the stack redzone, we need to drop
1027	%rsp by 128 before the hidden call, and restore afterwards. The
1028	nastyness is that it is only by luck that the stack still appears
1029	to be unwindable during the hidden call - since then the behaviour
1030	of any routine using this macro does not match what the CFI data
1031	says. Sigh.
1032
1033	Why is this important? Imagine that a wrapper has a stack
1034	allocated local, and passes to the hidden call, a pointer to it.
1035	Because gcc does not know about the hidden call, it may allocate
1036	that local in the redzone. Unfortunately the hidden call may then
1037	trash it before it comes to use it. So we must step clear of the
1038	redzone, for the duration of the hidden call, to make it safe.
1039
1040	Probably the same problem afflicts the other redzone-style ABIs too
1041	(ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1042	self describing (none of this CFI nonsense) so at least messing
1043	with the stack pointer doesn't give a danger of non-unwindable
1044	stack. */
1045
1046	#define CALL_FN_W_v(lval, orig) \
1047	do { \
1048	volatile OrigFn _orig = (orig); \
1049	volatile unsigned long _argvec[1]; \
1050	volatile unsigned long _res; \
1051	_argvec[0] = (unsigned long)_orig.nraddr; \
1052	__asm__ volatile( \
1053	"subq $128,%%rsp\n\t" \
1054	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1055	VALGRIND_CALL_NOREDIR_RAX \
1056	"addq $128,%%rsp\n\t" \
1057	: /out/ "=a" (_res) \
1058	: /in/ "a" (&_argvec[0]) \
1059	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1060	); \
1061	lval = (__typeof__(lval)) _res; \
1062	} while (0)
1063
1064	#define CALL_FN_W_W(lval, orig, arg1) \
1065	do { \
1066	volatile OrigFn _orig = (orig); \
1067	volatile unsigned long _argvec[2]; \
1068	volatile unsigned long _res; \
1069	_argvec[0] = (unsigned long)_orig.nraddr; \
1070	_argvec[1] = (unsigned long)(arg1); \
1071	__asm__ volatile( \
1072	"subq $128,%%rsp\n\t" \
1073	"movq 8(%%rax), %%rdi\n\t" \
1074	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1075	VALGRIND_CALL_NOREDIR_RAX \
1076	"addq $128,%%rsp\n\t" \
1077	: /out/ "=a" (_res) \
1078	: /in/ "a" (&_argvec[0]) \
1079	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1080	); \
1081	lval = (__typeof__(lval)) _res; \
1082	} while (0)
1083
1084	#define CALL_FN_W_WW(lval, orig, arg1,arg2) \
1085	do { \
1086	volatile OrigFn _orig = (orig); \
1087	volatile unsigned long _argvec[3]; \
1088	volatile unsigned long _res; \
1089	_argvec[0] = (unsigned long)_orig.nraddr; \
1090	_argvec[1] = (unsigned long)(arg1); \
1091	_argvec[2] = (unsigned long)(arg2); \
1092	__asm__ volatile( \
1093	"subq $128,%%rsp\n\t" \
1094	"movq 16(%%rax), %%rsi\n\t" \
1095	"movq 8(%%rax), %%rdi\n\t" \
1096	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1097	VALGRIND_CALL_NOREDIR_RAX \
1098	"addq $128,%%rsp\n\t" \
1099	: /out/ "=a" (_res) \
1100	: /in/ "a" (&_argvec[0]) \
1101	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1102	); \
1103	lval = (__typeof__(lval)) _res; \
1104	} while (0)
1105
1106	#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
1107	do { \
1108	volatile OrigFn _orig = (orig); \
1109	volatile unsigned long _argvec[4]; \
1110	volatile unsigned long _res; \
1111	_argvec[0] = (unsigned long)_orig.nraddr; \
1112	_argvec[1] = (unsigned long)(arg1); \
1113	_argvec[2] = (unsigned long)(arg2); \
1114	_argvec[3] = (unsigned long)(arg3); \
1115	__asm__ volatile( \
1116	"subq $128,%%rsp\n\t" \
1117	"movq 24(%%rax), %%rdx\n\t" \
1118	"movq 16(%%rax), %%rsi\n\t" \
1119	"movq 8(%%rax), %%rdi\n\t" \
1120	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1121	VALGRIND_CALL_NOREDIR_RAX \
1122	"addq $128,%%rsp\n\t" \
1123	: /out/ "=a" (_res) \
1124	: /in/ "a" (&_argvec[0]) \
1125	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1126	); \
1127	lval = (__typeof__(lval)) _res; \
1128	} while (0)
1129
1130	#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
1131	do { \
1132	volatile OrigFn _orig = (orig); \
1133	volatile unsigned long _argvec[5]; \
1134	volatile unsigned long _res; \
1135	_argvec[0] = (unsigned long)_orig.nraddr; \
1136	_argvec[1] = (unsigned long)(arg1); \
1137	_argvec[2] = (unsigned long)(arg2); \
1138	_argvec[3] = (unsigned long)(arg3); \
1139	_argvec[4] = (unsigned long)(arg4); \
1140	__asm__ volatile( \
1141	"subq $128,%%rsp\n\t" \
1142	"movq 32(%%rax), %%rcx\n\t" \
1143	"movq 24(%%rax), %%rdx\n\t" \
1144	"movq 16(%%rax), %%rsi\n\t" \
1145	"movq 8(%%rax), %%rdi\n\t" \
1146	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1147	VALGRIND_CALL_NOREDIR_RAX \
1148	"addq $128,%%rsp\n\t" \
1149	: /out/ "=a" (_res) \
1150	: /in/ "a" (&_argvec[0]) \
1151	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1152	); \
1153	lval = (__typeof__(lval)) _res; \
1154	} while (0)
1155
1156	#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
1157	do { \
1158	volatile OrigFn _orig = (orig); \
1159	volatile unsigned long _argvec[6]; \
1160	volatile unsigned long _res; \
1161	_argvec[0] = (unsigned long)_orig.nraddr; \
1162	_argvec[1] = (unsigned long)(arg1); \
1163	_argvec[2] = (unsigned long)(arg2); \
1164	_argvec[3] = (unsigned long)(arg3); \
1165	_argvec[4] = (unsigned long)(arg4); \
1166	_argvec[5] = (unsigned long)(arg5); \
1167	__asm__ volatile( \
1168	"subq $128,%%rsp\n\t" \
1169	"movq 40(%%rax), %%r8\n\t" \
1170	"movq 32(%%rax), %%rcx\n\t" \
1171	"movq 24(%%rax), %%rdx\n\t" \
1172	"movq 16(%%rax), %%rsi\n\t" \
1173	"movq 8(%%rax), %%rdi\n\t" \
1174	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1175	VALGRIND_CALL_NOREDIR_RAX \
1176	"addq $128,%%rsp\n\t" \
1177	: /out/ "=a" (_res) \
1178	: /in/ "a" (&_argvec[0]) \
1179	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1180	); \
1181	lval = (__typeof__(lval)) _res; \
1182	} while (0)
1183
1184	#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
1185	do { \
1186	volatile OrigFn _orig = (orig); \
1187	volatile unsigned long _argvec[7]; \
1188	volatile unsigned long _res; \
1189	_argvec[0] = (unsigned long)_orig.nraddr; \
1190	_argvec[1] = (unsigned long)(arg1); \
1191	_argvec[2] = (unsigned long)(arg2); \
1192	_argvec[3] = (unsigned long)(arg3); \
1193	_argvec[4] = (unsigned long)(arg4); \
1194	_argvec[5] = (unsigned long)(arg5); \
1195	_argvec[6] = (unsigned long)(arg6); \
1196	__asm__ volatile( \
1197	"subq $128,%%rsp\n\t" \
1198	"movq 48(%%rax), %%r9\n\t" \
1199	"movq 40(%%rax), %%r8\n\t" \
1200	"movq 32(%%rax), %%rcx\n\t" \
1201	"movq 24(%%rax), %%rdx\n\t" \
1202	"movq 16(%%rax), %%rsi\n\t" \
1203	"movq 8(%%rax), %%rdi\n\t" \
1204	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1205	"addq $128,%%rsp\n\t" \
1206	VALGRIND_CALL_NOREDIR_RAX \
1207	: /out/ "=a" (_res) \
1208	: /in/ "a" (&_argvec[0]) \
1209	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1210	); \
1211	lval = (__typeof__(lval)) _res; \
1212	} while (0)
1213
1214	#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1215	arg7) \
1216	do { \
1217	volatile OrigFn _orig = (orig); \
1218	volatile unsigned long _argvec[8]; \
1219	volatile unsigned long _res; \
1220	_argvec[0] = (unsigned long)_orig.nraddr; \
1221	_argvec[1] = (unsigned long)(arg1); \
1222	_argvec[2] = (unsigned long)(arg2); \
1223	_argvec[3] = (unsigned long)(arg3); \
1224	_argvec[4] = (unsigned long)(arg4); \
1225	_argvec[5] = (unsigned long)(arg5); \
1226	_argvec[6] = (unsigned long)(arg6); \
1227	_argvec[7] = (unsigned long)(arg7); \
1228	__asm__ volatile( \
1229	"subq $128,%%rsp\n\t" \
1230	"pushq 56(%%rax)\n\t" \
1231	"movq 48(%%rax), %%r9\n\t" \
1232	"movq 40(%%rax), %%r8\n\t" \
1233	"movq 32(%%rax), %%rcx\n\t" \
1234	"movq 24(%%rax), %%rdx\n\t" \
1235	"movq 16(%%rax), %%rsi\n\t" \
1236	"movq 8(%%rax), %%rdi\n\t" \
1237	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1238	VALGRIND_CALL_NOREDIR_RAX \
1239	"addq $8, %%rsp\n" \
1240	"addq $128,%%rsp\n\t" \
1241	: /out/ "=a" (_res) \
1242	: /in/ "a" (&_argvec[0]) \
1243	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1244	); \
1245	lval = (__typeof__(lval)) _res; \
1246	} while (0)
1247
1248	#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1249	arg7,arg8) \
1250	do { \
1251	volatile OrigFn _orig = (orig); \
1252	volatile unsigned long _argvec[9]; \
1253	volatile unsigned long _res; \
1254	_argvec[0] = (unsigned long)_orig.nraddr; \
1255	_argvec[1] = (unsigned long)(arg1); \
1256	_argvec[2] = (unsigned long)(arg2); \
1257	_argvec[3] = (unsigned long)(arg3); \
1258	_argvec[4] = (unsigned long)(arg4); \
1259	_argvec[5] = (unsigned long)(arg5); \
1260	_argvec[6] = (unsigned long)(arg6); \
1261	_argvec[7] = (unsigned long)(arg7); \
1262	_argvec[8] = (unsigned long)(arg8); \
1263	__asm__ volatile( \
1264	"subq $128,%%rsp\n\t" \
1265	"pushq 64(%%rax)\n\t" \
1266	"pushq 56(%%rax)\n\t" \
1267	"movq 48(%%rax), %%r9\n\t" \
1268	"movq 40(%%rax), %%r8\n\t" \
1269	"movq 32(%%rax), %%rcx\n\t" \
1270	"movq 24(%%rax), %%rdx\n\t" \
1271	"movq 16(%%rax), %%rsi\n\t" \
1272	"movq 8(%%rax), %%rdi\n\t" \
1273	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1274	VALGRIND_CALL_NOREDIR_RAX \
1275	"addq $16, %%rsp\n" \
1276	"addq $128,%%rsp\n\t" \
1277	: /out/ "=a" (_res) \
1278	: /in/ "a" (&_argvec[0]) \
1279	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1280	); \
1281	lval = (__typeof__(lval)) _res; \
1282	} while (0)
1283
1284	#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1285	arg7,arg8,arg9) \
1286	do { \
1287	volatile OrigFn _orig = (orig); \
1288	volatile unsigned long _argvec[10]; \
1289	volatile unsigned long _res; \
1290	_argvec[0] = (unsigned long)_orig.nraddr; \
1291	_argvec[1] = (unsigned long)(arg1); \
1292	_argvec[2] = (unsigned long)(arg2); \
1293	_argvec[3] = (unsigned long)(arg3); \
1294	_argvec[4] = (unsigned long)(arg4); \
1295	_argvec[5] = (unsigned long)(arg5); \
1296	_argvec[6] = (unsigned long)(arg6); \
1297	_argvec[7] = (unsigned long)(arg7); \
1298	_argvec[8] = (unsigned long)(arg8); \
1299	_argvec[9] = (unsigned long)(arg9); \
1300	__asm__ volatile( \
1301	"subq $128,%%rsp\n\t" \
1302	"pushq 72(%%rax)\n\t" \
1303	"pushq 64(%%rax)\n\t" \
1304	"pushq 56(%%rax)\n\t" \
1305	"movq 48(%%rax), %%r9\n\t" \
1306	"movq 40(%%rax), %%r8\n\t" \
1307	"movq 32(%%rax), %%rcx\n\t" \
1308	"movq 24(%%rax), %%rdx\n\t" \
1309	"movq 16(%%rax), %%rsi\n\t" \
1310	"movq 8(%%rax), %%rdi\n\t" \
1311	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1312	VALGRIND_CALL_NOREDIR_RAX \
1313	"addq $24, %%rsp\n" \
1314	"addq $128,%%rsp\n\t" \
1315	: /out/ "=a" (_res) \
1316	: /in/ "a" (&_argvec[0]) \
1317	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1318	); \
1319	lval = (__typeof__(lval)) _res; \
1320	} while (0)
1321
1322	#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1323	arg7,arg8,arg9,arg10) \
1324	do { \
1325	volatile OrigFn _orig = (orig); \
1326	volatile unsigned long _argvec[11]; \
1327	volatile unsigned long _res; \
1328	_argvec[0] = (unsigned long)_orig.nraddr; \
1329	_argvec[1] = (unsigned long)(arg1); \
1330	_argvec[2] = (unsigned long)(arg2); \
1331	_argvec[3] = (unsigned long)(arg3); \
1332	_argvec[4] = (unsigned long)(arg4); \
1333	_argvec[5] = (unsigned long)(arg5); \
1334	_argvec[6] = (unsigned long)(arg6); \
1335	_argvec[7] = (unsigned long)(arg7); \
1336	_argvec[8] = (unsigned long)(arg8); \
1337	_argvec[9] = (unsigned long)(arg9); \
1338	_argvec[10] = (unsigned long)(arg10); \
1339	__asm__ volatile( \
1340	"subq $128,%%rsp\n\t" \
1341	"pushq 80(%%rax)\n\t" \
1342	"pushq 72(%%rax)\n\t" \
1343	"pushq 64(%%rax)\n\t" \
1344	"pushq 56(%%rax)\n\t" \
1345	"movq 48(%%rax), %%r9\n\t" \
1346	"movq 40(%%rax), %%r8\n\t" \
1347	"movq 32(%%rax), %%rcx\n\t" \
1348	"movq 24(%%rax), %%rdx\n\t" \
1349	"movq 16(%%rax), %%rsi\n\t" \
1350	"movq 8(%%rax), %%rdi\n\t" \
1351	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1352	VALGRIND_CALL_NOREDIR_RAX \
1353	"addq $32, %%rsp\n" \
1354	"addq $128,%%rsp\n\t" \
1355	: /out/ "=a" (_res) \
1356	: /in/ "a" (&_argvec[0]) \
1357	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1358	); \
1359	lval = (__typeof__(lval)) _res; \
1360	} while (0)
1361
1362	#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1363	arg7,arg8,arg9,arg10,arg11) \
1364	do { \
1365	volatile OrigFn _orig = (orig); \
1366	volatile unsigned long _argvec[12]; \
1367	volatile unsigned long _res; \
1368	_argvec[0] = (unsigned long)_orig.nraddr; \
1369	_argvec[1] = (unsigned long)(arg1); \
1370	_argvec[2] = (unsigned long)(arg2); \
1371	_argvec[3] = (unsigned long)(arg3); \
1372	_argvec[4] = (unsigned long)(arg4); \
1373	_argvec[5] = (unsigned long)(arg5); \
1374	_argvec[6] = (unsigned long)(arg6); \
1375	_argvec[7] = (unsigned long)(arg7); \
1376	_argvec[8] = (unsigned long)(arg8); \
1377	_argvec[9] = (unsigned long)(arg9); \
1378	_argvec[10] = (unsigned long)(arg10); \
1379	_argvec[11] = (unsigned long)(arg11); \
1380	__asm__ volatile( \
1381	"subq $128,%%rsp\n\t" \
1382	"pushq 88(%%rax)\n\t" \
1383	"pushq 80(%%rax)\n\t" \
1384	"pushq 72(%%rax)\n\t" \
1385	"pushq 64(%%rax)\n\t" \
1386	"pushq 56(%%rax)\n\t" \
1387	"movq 48(%%rax), %%r9\n\t" \
1388	"movq 40(%%rax), %%r8\n\t" \
1389	"movq 32(%%rax), %%rcx\n\t" \
1390	"movq 24(%%rax), %%rdx\n\t" \
1391	"movq 16(%%rax), %%rsi\n\t" \
1392	"movq 8(%%rax), %%rdi\n\t" \
1393	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1394	VALGRIND_CALL_NOREDIR_RAX \
1395	"addq $40, %%rsp\n" \
1396	"addq $128,%%rsp\n\t" \
1397	: /out/ "=a" (_res) \
1398	: /in/ "a" (&_argvec[0]) \
1399	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1400	); \
1401	lval = (__typeof__(lval)) _res; \
1402	} while (0)
1403
1404	#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1405	arg7,arg8,arg9,arg10,arg11,arg12) \
1406	do { \
1407	volatile OrigFn _orig = (orig); \
1408	volatile unsigned long _argvec[13]; \
1409	volatile unsigned long _res; \
1410	_argvec[0] = (unsigned long)_orig.nraddr; \
1411	_argvec[1] = (unsigned long)(arg1); \
1412	_argvec[2] = (unsigned long)(arg2); \
1413	_argvec[3] = (unsigned long)(arg3); \
1414	_argvec[4] = (unsigned long)(arg4); \
1415	_argvec[5] = (unsigned long)(arg5); \
1416	_argvec[6] = (unsigned long)(arg6); \
1417	_argvec[7] = (unsigned long)(arg7); \
1418	_argvec[8] = (unsigned long)(arg8); \
1419	_argvec[9] = (unsigned long)(arg9); \
1420	_argvec[10] = (unsigned long)(arg10); \
1421	_argvec[11] = (unsigned long)(arg11); \
1422	_argvec[12] = (unsigned long)(arg12); \
1423	__asm__ volatile( \
1424	"subq $128,%%rsp\n\t" \
1425	"pushq 96(%%rax)\n\t" \
1426	"pushq 88(%%rax)\n\t" \
1427	"pushq 80(%%rax)\n\t" \
1428	"pushq 72(%%rax)\n\t" \
1429	"pushq 64(%%rax)\n\t" \
1430	"pushq 56(%%rax)\n\t" \
1431	"movq 48(%%rax), %%r9\n\t" \
1432	"movq 40(%%rax), %%r8\n\t" \
1433	"movq 32(%%rax), %%rcx\n\t" \
1434	"movq 24(%%rax), %%rdx\n\t" \
1435	"movq 16(%%rax), %%rsi\n\t" \
1436	"movq 8(%%rax), %%rdi\n\t" \
1437	"movq (%%rax), %%rax\n\t" /* target->%rax */ \
1438	VALGRIND_CALL_NOREDIR_RAX \
1439	"addq $48, %%rsp\n" \
1440	"addq $128,%%rsp\n\t" \
1441	: /out/ "=a" (_res) \
1442	: /in/ "a" (&_argvec[0]) \
1443	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1444	); \
1445	lval = (__typeof__(lval)) _res; \
1446	} while (0)
1447
1448	#endif /* PLAT_amd64_linux */
1449
1450	/* ------------------------ ppc32-linux ------------------------ */
1451
1452	#if defined(PLAT_ppc32_linux)
1453
1454	/* This is useful for finding out about the on-stack stuff:
1455
1456	extern int f9 ( int,int,int,int,int,int,int,int,int );
1457	extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1458	extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1459	extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1460
1461	int g9 ( void ) {
1462	return f9(11,22,33,44,55,66,77,88,99);
1463	}
1464	int g10 ( void ) {
1465	return f10(11,22,33,44,55,66,77,88,99,110);
1466	}
1467	int g11 ( void ) {
1468	return f11(11,22,33,44,55,66,77,88,99,110,121);
1469	}
1470	int g12 ( void ) {
1471	return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1472	}
1473	*/
1474
1475	/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1476
1477	/* These regs are trashed by the hidden call. */
1478	#define __CALLER_SAVED_REGS \
1479	"lr", "ctr", "xer", \
1480	"cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \
1481	"r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \
1482	"r11", "r12", "r13"
1483
1484	/* These CALL_FN_ macros assume that on ppc32-linux,
1485	sizeof(unsigned long) == 4. */
1486
1487	#define CALL_FN_W_v(lval, orig) \
1488	do { \
1489	volatile OrigFn _orig = (orig); \
1490	volatile unsigned long _argvec[1]; \
1491	volatile unsigned long _res; \
1492	_argvec[0] = (unsigned long)_orig.nraddr; \
1493	__asm__ volatile( \
1494	"mr 11,%1\n\t" \
1495	"lwz 11,0(11)\n\t" /* target->r11 */ \
1496	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1497	"mr %0,3" \
1498	: /out/ "=r" (_res) \
1499	: /in/ "r" (&_argvec[0]) \
1500	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1501	); \
1502	lval = (__typeof__(lval)) _res; \
1503	} while (0)
1504
1505	#define CALL_FN_W_W(lval, orig, arg1) \
1506	do { \
1507	volatile OrigFn _orig = (orig); \
1508	volatile unsigned long _argvec[2]; \
1509	volatile unsigned long _res; \
1510	_argvec[0] = (unsigned long)_orig.nraddr; \
1511	_argvec[1] = (unsigned long)arg1; \
1512	__asm__ volatile( \
1513	"mr 11,%1\n\t" \
1514	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1515	"lwz 11,0(11)\n\t" /* target->r11 */ \
1516	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1517	"mr %0,3" \
1518	: /out/ "=r" (_res) \
1519	: /in/ "r" (&_argvec[0]) \
1520	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1521	); \
1522	lval = (__typeof__(lval)) _res; \
1523	} while (0)
1524
1525	#define CALL_FN_W_WW(lval, orig, arg1,arg2) \
1526	do { \
1527	volatile OrigFn _orig = (orig); \
1528	volatile unsigned long _argvec[3]; \
1529	volatile unsigned long _res; \
1530	_argvec[0] = (unsigned long)_orig.nraddr; \
1531	_argvec[1] = (unsigned long)arg1; \
1532	_argvec[2] = (unsigned long)arg2; \
1533	__asm__ volatile( \
1534	"mr 11,%1\n\t" \
1535	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1536	"lwz 4,8(11)\n\t" \
1537	"lwz 11,0(11)\n\t" /* target->r11 */ \
1538	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1539	"mr %0,3" \
1540	: /out/ "=r" (_res) \
1541	: /in/ "r" (&_argvec[0]) \
1542	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1543	); \
1544	lval = (__typeof__(lval)) _res; \
1545	} while (0)
1546
1547	#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
1548	do { \
1549	volatile OrigFn _orig = (orig); \
1550	volatile unsigned long _argvec[4]; \
1551	volatile unsigned long _res; \
1552	_argvec[0] = (unsigned long)_orig.nraddr; \
1553	_argvec[1] = (unsigned long)arg1; \
1554	_argvec[2] = (unsigned long)arg2; \
1555	_argvec[3] = (unsigned long)arg3; \
1556	__asm__ volatile( \
1557	"mr 11,%1\n\t" \
1558	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1559	"lwz 4,8(11)\n\t" \
1560	"lwz 5,12(11)\n\t" \
1561	"lwz 11,0(11)\n\t" /* target->r11 */ \
1562	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1563	"mr %0,3" \
1564	: /out/ "=r" (_res) \
1565	: /in/ "r" (&_argvec[0]) \
1566	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1567	); \
1568	lval = (__typeof__(lval)) _res; \
1569	} while (0)
1570
1571	#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
1572	do { \
1573	volatile OrigFn _orig = (orig); \
1574	volatile unsigned long _argvec[5]; \
1575	volatile unsigned long _res; \
1576	_argvec[0] = (unsigned long)_orig.nraddr; \
1577	_argvec[1] = (unsigned long)arg1; \
1578	_argvec[2] = (unsigned long)arg2; \
1579	_argvec[3] = (unsigned long)arg3; \
1580	_argvec[4] = (unsigned long)arg4; \
1581	__asm__ volatile( \
1582	"mr 11,%1\n\t" \
1583	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1584	"lwz 4,8(11)\n\t" \
1585	"lwz 5,12(11)\n\t" \
1586	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1587	"lwz 11,0(11)\n\t" /* target->r11 */ \
1588	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1589	"mr %0,3" \
1590	: /out/ "=r" (_res) \
1591	: /in/ "r" (&_argvec[0]) \
1592	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1593	); \
1594	lval = (__typeof__(lval)) _res; \
1595	} while (0)
1596
1597	#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
1598	do { \
1599	volatile OrigFn _orig = (orig); \
1600	volatile unsigned long _argvec[6]; \
1601	volatile unsigned long _res; \
1602	_argvec[0] = (unsigned long)_orig.nraddr; \
1603	_argvec[1] = (unsigned long)arg1; \
1604	_argvec[2] = (unsigned long)arg2; \
1605	_argvec[3] = (unsigned long)arg3; \
1606	_argvec[4] = (unsigned long)arg4; \
1607	_argvec[5] = (unsigned long)arg5; \
1608	__asm__ volatile( \
1609	"mr 11,%1\n\t" \
1610	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1611	"lwz 4,8(11)\n\t" \
1612	"lwz 5,12(11)\n\t" \
1613	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1614	"lwz 7,20(11)\n\t" \
1615	"lwz 11,0(11)\n\t" /* target->r11 */ \
1616	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1617	"mr %0,3" \
1618	: /out/ "=r" (_res) \
1619	: /in/ "r" (&_argvec[0]) \
1620	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1621	); \
1622	lval = (__typeof__(lval)) _res; \
1623	} while (0)
1624
1625	#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
1626	do { \
1627	volatile OrigFn _orig = (orig); \
1628	volatile unsigned long _argvec[7]; \
1629	volatile unsigned long _res; \
1630	_argvec[0] = (unsigned long)_orig.nraddr; \
1631	_argvec[1] = (unsigned long)arg1; \
1632	_argvec[2] = (unsigned long)arg2; \
1633	_argvec[3] = (unsigned long)arg3; \
1634	_argvec[4] = (unsigned long)arg4; \
1635	_argvec[5] = (unsigned long)arg5; \
1636	_argvec[6] = (unsigned long)arg6; \
1637	__asm__ volatile( \
1638	"mr 11,%1\n\t" \
1639	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1640	"lwz 4,8(11)\n\t" \
1641	"lwz 5,12(11)\n\t" \
1642	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1643	"lwz 7,20(11)\n\t" \
1644	"lwz 8,24(11)\n\t" \
1645	"lwz 11,0(11)\n\t" /* target->r11 */ \
1646	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1647	"mr %0,3" \
1648	: /out/ "=r" (_res) \
1649	: /in/ "r" (&_argvec[0]) \
1650	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1651	); \
1652	lval = (__typeof__(lval)) _res; \
1653	} while (0)
1654
1655	#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1656	arg7) \
1657	do { \
1658	volatile OrigFn _orig = (orig); \
1659	volatile unsigned long _argvec[8]; \
1660	volatile unsigned long _res; \
1661	_argvec[0] = (unsigned long)_orig.nraddr; \
1662	_argvec[1] = (unsigned long)arg1; \
1663	_argvec[2] = (unsigned long)arg2; \
1664	_argvec[3] = (unsigned long)arg3; \
1665	_argvec[4] = (unsigned long)arg4; \
1666	_argvec[5] = (unsigned long)arg5; \
1667	_argvec[6] = (unsigned long)arg6; \
1668	_argvec[7] = (unsigned long)arg7; \
1669	__asm__ volatile( \
1670	"mr 11,%1\n\t" \
1671	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1672	"lwz 4,8(11)\n\t" \
1673	"lwz 5,12(11)\n\t" \
1674	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1675	"lwz 7,20(11)\n\t" \
1676	"lwz 8,24(11)\n\t" \
1677	"lwz 9,28(11)\n\t" \
1678	"lwz 11,0(11)\n\t" /* target->r11 */ \
1679	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1680	"mr %0,3" \
1681	: /out/ "=r" (_res) \
1682	: /in/ "r" (&_argvec[0]) \
1683	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1684	); \
1685	lval = (__typeof__(lval)) _res; \
1686	} while (0)
1687
1688	#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1689	arg7,arg8) \
1690	do { \
1691	volatile OrigFn _orig = (orig); \
1692	volatile unsigned long _argvec[9]; \
1693	volatile unsigned long _res; \
1694	_argvec[0] = (unsigned long)_orig.nraddr; \
1695	_argvec[1] = (unsigned long)arg1; \
1696	_argvec[2] = (unsigned long)arg2; \
1697	_argvec[3] = (unsigned long)arg3; \
1698	_argvec[4] = (unsigned long)arg4; \
1699	_argvec[5] = (unsigned long)arg5; \
1700	_argvec[6] = (unsigned long)arg6; \
1701	_argvec[7] = (unsigned long)arg7; \
1702	_argvec[8] = (unsigned long)arg8; \
1703	__asm__ volatile( \
1704	"mr 11,%1\n\t" \
1705	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1706	"lwz 4,8(11)\n\t" \
1707	"lwz 5,12(11)\n\t" \
1708	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1709	"lwz 7,20(11)\n\t" \
1710	"lwz 8,24(11)\n\t" \
1711	"lwz 9,28(11)\n\t" \
1712	"lwz 10,32(11)\n\t" /* arg8->r10 */ \
1713	"lwz 11,0(11)\n\t" /* target->r11 */ \
1714	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1715	"mr %0,3" \
1716	: /out/ "=r" (_res) \
1717	: /in/ "r" (&_argvec[0]) \
1718	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1719	); \
1720	lval = (__typeof__(lval)) _res; \
1721	} while (0)
1722
1723	#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1724	arg7,arg8,arg9) \
1725	do { \
1726	volatile OrigFn _orig = (orig); \
1727	volatile unsigned long _argvec[10]; \
1728	volatile unsigned long _res; \
1729	_argvec[0] = (unsigned long)_orig.nraddr; \
1730	_argvec[1] = (unsigned long)arg1; \
1731	_argvec[2] = (unsigned long)arg2; \
1732	_argvec[3] = (unsigned long)arg3; \
1733	_argvec[4] = (unsigned long)arg4; \
1734	_argvec[5] = (unsigned long)arg5; \
1735	_argvec[6] = (unsigned long)arg6; \
1736	_argvec[7] = (unsigned long)arg7; \
1737	_argvec[8] = (unsigned long)arg8; \
1738	_argvec[9] = (unsigned long)arg9; \
1739	__asm__ volatile( \
1740	"mr 11,%1\n\t" \
1741	"addi 1,1,-16\n\t" \
1742	/* arg9 */ \
1743	"lwz 3,36(11)\n\t" \
1744	"stw 3,8(1)\n\t" \
1745	/* args1-8 */ \
1746	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1747	"lwz 4,8(11)\n\t" \
1748	"lwz 5,12(11)\n\t" \
1749	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1750	"lwz 7,20(11)\n\t" \
1751	"lwz 8,24(11)\n\t" \
1752	"lwz 9,28(11)\n\t" \
1753	"lwz 10,32(11)\n\t" /* arg8->r10 */ \
1754	"lwz 11,0(11)\n\t" /* target->r11 */ \
1755	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1756	"addi 1,1,16\n\t" \
1757	"mr %0,3" \
1758	: /out/ "=r" (_res) \
1759	: /in/ "r" (&_argvec[0]) \
1760	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1761	); \
1762	lval = (__typeof__(lval)) _res; \
1763	} while (0)
1764
1765	#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1766	arg7,arg8,arg9,arg10) \
1767	do { \
1768	volatile OrigFn _orig = (orig); \
1769	volatile unsigned long _argvec[11]; \
1770	volatile unsigned long _res; \
1771	_argvec[0] = (unsigned long)_orig.nraddr; \
1772	_argvec[1] = (unsigned long)arg1; \
1773	_argvec[2] = (unsigned long)arg2; \
1774	_argvec[3] = (unsigned long)arg3; \
1775	_argvec[4] = (unsigned long)arg4; \
1776	_argvec[5] = (unsigned long)arg5; \
1777	_argvec[6] = (unsigned long)arg6; \
1778	_argvec[7] = (unsigned long)arg7; \
1779	_argvec[8] = (unsigned long)arg8; \
1780	_argvec[9] = (unsigned long)arg9; \
1781	_argvec[10] = (unsigned long)arg10; \
1782	__asm__ volatile( \
1783	"mr 11,%1\n\t" \
1784	"addi 1,1,-16\n\t" \
1785	/* arg10 */ \
1786	"lwz 3,40(11)\n\t" \
1787	"stw 3,12(1)\n\t" \
1788	/* arg9 */ \
1789	"lwz 3,36(11)\n\t" \
1790	"stw 3,8(1)\n\t" \
1791	/* args1-8 */ \
1792	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1793	"lwz 4,8(11)\n\t" \
1794	"lwz 5,12(11)\n\t" \
1795	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1796	"lwz 7,20(11)\n\t" \
1797	"lwz 8,24(11)\n\t" \
1798	"lwz 9,28(11)\n\t" \
1799	"lwz 10,32(11)\n\t" /* arg8->r10 */ \
1800	"lwz 11,0(11)\n\t" /* target->r11 */ \
1801	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1802	"addi 1,1,16\n\t" \
1803	"mr %0,3" \
1804	: /out/ "=r" (_res) \
1805	: /in/ "r" (&_argvec[0]) \
1806	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1807	); \
1808	lval = (__typeof__(lval)) _res; \
1809	} while (0)
1810
1811	#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1812	arg7,arg8,arg9,arg10,arg11) \
1813	do { \
1814	volatile OrigFn _orig = (orig); \
1815	volatile unsigned long _argvec[12]; \
1816	volatile unsigned long _res; \
1817	_argvec[0] = (unsigned long)_orig.nraddr; \
1818	_argvec[1] = (unsigned long)arg1; \
1819	_argvec[2] = (unsigned long)arg2; \
1820	_argvec[3] = (unsigned long)arg3; \
1821	_argvec[4] = (unsigned long)arg4; \
1822	_argvec[5] = (unsigned long)arg5; \
1823	_argvec[6] = (unsigned long)arg6; \
1824	_argvec[7] = (unsigned long)arg7; \
1825	_argvec[8] = (unsigned long)arg8; \
1826	_argvec[9] = (unsigned long)arg9; \
1827	_argvec[10] = (unsigned long)arg10; \
1828	_argvec[11] = (unsigned long)arg11; \
1829	__asm__ volatile( \
1830	"mr 11,%1\n\t" \
1831	"addi 1,1,-32\n\t" \
1832	/* arg11 */ \
1833	"lwz 3,44(11)\n\t" \
1834	"stw 3,16(1)\n\t" \
1835	/* arg10 */ \
1836	"lwz 3,40(11)\n\t" \
1837	"stw 3,12(1)\n\t" \
1838	/* arg9 */ \
1839	"lwz 3,36(11)\n\t" \
1840	"stw 3,8(1)\n\t" \
1841	/* args1-8 */ \
1842	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1843	"lwz 4,8(11)\n\t" \
1844	"lwz 5,12(11)\n\t" \
1845	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1846	"lwz 7,20(11)\n\t" \
1847	"lwz 8,24(11)\n\t" \
1848	"lwz 9,28(11)\n\t" \
1849	"lwz 10,32(11)\n\t" /* arg8->r10 */ \
1850	"lwz 11,0(11)\n\t" /* target->r11 */ \
1851	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1852	"addi 1,1,32\n\t" \
1853	"mr %0,3" \
1854	: /out/ "=r" (_res) \
1855	: /in/ "r" (&_argvec[0]) \
1856	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1857	); \
1858	lval = (__typeof__(lval)) _res; \
1859	} while (0)
1860
1861	#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1862	arg7,arg8,arg9,arg10,arg11,arg12) \
1863	do { \
1864	volatile OrigFn _orig = (orig); \
1865	volatile unsigned long _argvec[13]; \
1866	volatile unsigned long _res; \
1867	_argvec[0] = (unsigned long)_orig.nraddr; \
1868	_argvec[1] = (unsigned long)arg1; \
1869	_argvec[2] = (unsigned long)arg2; \
1870	_argvec[3] = (unsigned long)arg3; \
1871	_argvec[4] = (unsigned long)arg4; \
1872	_argvec[5] = (unsigned long)arg5; \
1873	_argvec[6] = (unsigned long)arg6; \
1874	_argvec[7] = (unsigned long)arg7; \
1875	_argvec[8] = (unsigned long)arg8; \
1876	_argvec[9] = (unsigned long)arg9; \
1877	_argvec[10] = (unsigned long)arg10; \
1878	_argvec[11] = (unsigned long)arg11; \
1879	_argvec[12] = (unsigned long)arg12; \
1880	__asm__ volatile( \
1881	"mr 11,%1\n\t" \
1882	"addi 1,1,-32\n\t" \
1883	/* arg12 */ \
1884	"lwz 3,48(11)\n\t" \
1885	"stw 3,20(1)\n\t" \
1886	/* arg11 */ \
1887	"lwz 3,44(11)\n\t" \
1888	"stw 3,16(1)\n\t" \
1889	/* arg10 */ \
1890	"lwz 3,40(11)\n\t" \
1891	"stw 3,12(1)\n\t" \
1892	/* arg9 */ \
1893	"lwz 3,36(11)\n\t" \
1894	"stw 3,8(1)\n\t" \
1895	/* args1-8 */ \
1896	"lwz 3,4(11)\n\t" /* arg1->r3 */ \
1897	"lwz 4,8(11)\n\t" \
1898	"lwz 5,12(11)\n\t" \
1899	"lwz 6,16(11)\n\t" /* arg4->r6 */ \
1900	"lwz 7,20(11)\n\t" \
1901	"lwz 8,24(11)\n\t" \
1902	"lwz 9,28(11)\n\t" \
1903	"lwz 10,32(11)\n\t" /* arg8->r10 */ \
1904	"lwz 11,0(11)\n\t" /* target->r11 */ \
1905	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1906	"addi 1,1,32\n\t" \
1907	"mr %0,3" \
1908	: /out/ "=r" (_res) \
1909	: /in/ "r" (&_argvec[0]) \
1910	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1911	); \
1912	lval = (__typeof__(lval)) _res; \
1913	} while (0)
1914
1915	#endif /* PLAT_ppc32_linux */
1916
1917	/* ------------------------ ppc64-linux ------------------------ */
1918
1919	#if defined(PLAT_ppc64_linux)
1920
1921	/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1922
1923	/* These regs are trashed by the hidden call. */
1924	#define __CALLER_SAVED_REGS \
1925	"lr", "ctr", "xer", \
1926	"cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \
1927	"r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \
1928	"r11", "r12", "r13"
1929
1930	/* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
1931	long) == 8. */
1932
1933	#define CALL_FN_W_v(lval, orig) \
1934	do { \
1935	volatile OrigFn _orig = (orig); \
1936	volatile unsigned long _argvec[3+0]; \
1937	volatile unsigned long _res; \
1938	/* _argvec[0] holds current r2 across the call */ \
1939	_argvec[1] = (unsigned long)_orig.r2; \
1940	_argvec[2] = (unsigned long)_orig.nraddr; \
1941	__asm__ volatile( \
1942	"mr 11,%1\n\t" \
1943	"std 2,-16(11)\n\t" /* save tocptr */ \
1944	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
1945	"ld 11, 0(11)\n\t" /* target->r11 */ \
1946	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1947	"mr 11,%1\n\t" \
1948	"mr %0,3\n\t" \
1949	"ld 2,-16(11)" /* restore tocptr */ \
1950	: /out/ "=r" (_res) \
1951	: /in/ "r" (&_argvec[2]) \
1952	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1953	); \
1954	lval = (__typeof__(lval)) _res; \
1955	} while (0)
1956
1957	#define CALL_FN_W_W(lval, orig, arg1) \
1958	do { \
1959	volatile OrigFn _orig = (orig); \
1960	volatile unsigned long _argvec[3+1]; \
1961	volatile unsigned long _res; \
1962	/* _argvec[0] holds current r2 across the call */ \
1963	_argvec[1] = (unsigned long)_orig.r2; \
1964	_argvec[2] = (unsigned long)_orig.nraddr; \
1965	_argvec[2+1] = (unsigned long)arg1; \
1966	__asm__ volatile( \
1967	"mr 11,%1\n\t" \
1968	"std 2,-16(11)\n\t" /* save tocptr */ \
1969	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
1970	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
1971	"ld 11, 0(11)\n\t" /* target->r11 */ \
1972	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
1973	"mr 11,%1\n\t" \
1974	"mr %0,3\n\t" \
1975	"ld 2,-16(11)" /* restore tocptr */ \
1976	: /out/ "=r" (_res) \
1977	: /in/ "r" (&_argvec[2]) \
1978	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
1979	); \
1980	lval = (__typeof__(lval)) _res; \
1981	} while (0)
1982
1983	#define CALL_FN_W_WW(lval, orig, arg1,arg2) \
1984	do { \
1985	volatile OrigFn _orig = (orig); \
1986	volatile unsigned long _argvec[3+2]; \
1987	volatile unsigned long _res; \
1988	/* _argvec[0] holds current r2 across the call */ \
1989	_argvec[1] = (unsigned long)_orig.r2; \
1990	_argvec[2] = (unsigned long)_orig.nraddr; \
1991	_argvec[2+1] = (unsigned long)arg1; \
1992	_argvec[2+2] = (unsigned long)arg2; \
1993	__asm__ volatile( \
1994	"mr 11,%1\n\t" \
1995	"std 2,-16(11)\n\t" /* save tocptr */ \
1996	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
1997	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
1998	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
1999	"ld 11, 0(11)\n\t" /* target->r11 */ \
2000	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2001	"mr 11,%1\n\t" \
2002	"mr %0,3\n\t" \
2003	"ld 2,-16(11)" /* restore tocptr */ \
2004	: /out/ "=r" (_res) \
2005	: /in/ "r" (&_argvec[2]) \
2006	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2007	); \
2008	lval = (__typeof__(lval)) _res; \
2009	} while (0)
2010
2011	#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
2012	do { \
2013	volatile OrigFn _orig = (orig); \
2014	volatile unsigned long _argvec[3+3]; \
2015	volatile unsigned long _res; \
2016	/* _argvec[0] holds current r2 across the call */ \
2017	_argvec[1] = (unsigned long)_orig.r2; \
2018	_argvec[2] = (unsigned long)_orig.nraddr; \
2019	_argvec[2+1] = (unsigned long)arg1; \
2020	_argvec[2+2] = (unsigned long)arg2; \
2021	_argvec[2+3] = (unsigned long)arg3; \
2022	__asm__ volatile( \
2023	"mr 11,%1\n\t" \
2024	"std 2,-16(11)\n\t" /* save tocptr */ \
2025	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2026	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2027	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2028	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2029	"ld 11, 0(11)\n\t" /* target->r11 */ \
2030	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2031	"mr 11,%1\n\t" \
2032	"mr %0,3\n\t" \
2033	"ld 2,-16(11)" /* restore tocptr */ \
2034	: /out/ "=r" (_res) \
2035	: /in/ "r" (&_argvec[2]) \
2036	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2037	); \
2038	lval = (__typeof__(lval)) _res; \
2039	} while (0)
2040
2041	#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
2042	do { \
2043	volatile OrigFn _orig = (orig); \
2044	volatile unsigned long _argvec[3+4]; \
2045	volatile unsigned long _res; \
2046	/* _argvec[0] holds current r2 across the call */ \
2047	_argvec[1] = (unsigned long)_orig.r2; \
2048	_argvec[2] = (unsigned long)_orig.nraddr; \
2049	_argvec[2+1] = (unsigned long)arg1; \
2050	_argvec[2+2] = (unsigned long)arg2; \
2051	_argvec[2+3] = (unsigned long)arg3; \
2052	_argvec[2+4] = (unsigned long)arg4; \
2053	__asm__ volatile( \
2054	"mr 11,%1\n\t" \
2055	"std 2,-16(11)\n\t" /* save tocptr */ \
2056	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2057	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2058	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2059	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2060	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2061	"ld 11, 0(11)\n\t" /* target->r11 */ \
2062	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2063	"mr 11,%1\n\t" \
2064	"mr %0,3\n\t" \
2065	"ld 2,-16(11)" /* restore tocptr */ \
2066	: /out/ "=r" (_res) \
2067	: /in/ "r" (&_argvec[2]) \
2068	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2069	); \
2070	lval = (__typeof__(lval)) _res; \
2071	} while (0)
2072
2073	#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
2074	do { \
2075	volatile OrigFn _orig = (orig); \
2076	volatile unsigned long _argvec[3+5]; \
2077	volatile unsigned long _res; \
2078	/* _argvec[0] holds current r2 across the call */ \
2079	_argvec[1] = (unsigned long)_orig.r2; \
2080	_argvec[2] = (unsigned long)_orig.nraddr; \
2081	_argvec[2+1] = (unsigned long)arg1; \
2082	_argvec[2+2] = (unsigned long)arg2; \
2083	_argvec[2+3] = (unsigned long)arg3; \
2084	_argvec[2+4] = (unsigned long)arg4; \
2085	_argvec[2+5] = (unsigned long)arg5; \
2086	__asm__ volatile( \
2087	"mr 11,%1\n\t" \
2088	"std 2,-16(11)\n\t" /* save tocptr */ \
2089	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2090	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2091	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2092	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2093	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2094	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2095	"ld 11, 0(11)\n\t" /* target->r11 */ \
2096	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2097	"mr 11,%1\n\t" \
2098	"mr %0,3\n\t" \
2099	"ld 2,-16(11)" /* restore tocptr */ \
2100	: /out/ "=r" (_res) \
2101	: /in/ "r" (&_argvec[2]) \
2102	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2103	); \
2104	lval = (__typeof__(lval)) _res; \
2105	} while (0)
2106
2107	#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
2108	do { \
2109	volatile OrigFn _orig = (orig); \
2110	volatile unsigned long _argvec[3+6]; \
2111	volatile unsigned long _res; \
2112	/* _argvec[0] holds current r2 across the call */ \
2113	_argvec[1] = (unsigned long)_orig.r2; \
2114	_argvec[2] = (unsigned long)_orig.nraddr; \
2115	_argvec[2+1] = (unsigned long)arg1; \
2116	_argvec[2+2] = (unsigned long)arg2; \
2117	_argvec[2+3] = (unsigned long)arg3; \
2118	_argvec[2+4] = (unsigned long)arg4; \
2119	_argvec[2+5] = (unsigned long)arg5; \
2120	_argvec[2+6] = (unsigned long)arg6; \
2121	__asm__ volatile( \
2122	"mr 11,%1\n\t" \
2123	"std 2,-16(11)\n\t" /* save tocptr */ \
2124	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2125	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2126	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2127	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2128	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2129	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2130	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
2131	"ld 11, 0(11)\n\t" /* target->r11 */ \
2132	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2133	"mr 11,%1\n\t" \
2134	"mr %0,3\n\t" \
2135	"ld 2,-16(11)" /* restore tocptr */ \
2136	: /out/ "=r" (_res) \
2137	: /in/ "r" (&_argvec[2]) \
2138	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2139	); \
2140	lval = (__typeof__(lval)) _res; \
2141	} while (0)
2142
2143	#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2144	arg7) \
2145	do { \
2146	volatile OrigFn _orig = (orig); \
2147	volatile unsigned long _argvec[3+7]; \
2148	volatile unsigned long _res; \
2149	/* _argvec[0] holds current r2 across the call */ \
2150	_argvec[1] = (unsigned long)_orig.r2; \
2151	_argvec[2] = (unsigned long)_orig.nraddr; \
2152	_argvec[2+1] = (unsigned long)arg1; \
2153	_argvec[2+2] = (unsigned long)arg2; \
2154	_argvec[2+3] = (unsigned long)arg3; \
2155	_argvec[2+4] = (unsigned long)arg4; \
2156	_argvec[2+5] = (unsigned long)arg5; \
2157	_argvec[2+6] = (unsigned long)arg6; \
2158	_argvec[2+7] = (unsigned long)arg7; \
2159	__asm__ volatile( \
2160	"mr 11,%1\n\t" \
2161	"std 2,-16(11)\n\t" /* save tocptr */ \
2162	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2163	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2164	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2165	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2166	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2167	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2168	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
2169	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
2170	"ld 11, 0(11)\n\t" /* target->r11 */ \
2171	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2172	"mr 11,%1\n\t" \
2173	"mr %0,3\n\t" \
2174	"ld 2,-16(11)" /* restore tocptr */ \
2175	: /out/ "=r" (_res) \
2176	: /in/ "r" (&_argvec[2]) \
2177	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2178	); \
2179	lval = (__typeof__(lval)) _res; \
2180	} while (0)
2181
2182	#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2183	arg7,arg8) \
2184	do { \
2185	volatile OrigFn _orig = (orig); \
2186	volatile unsigned long _argvec[3+8]; \
2187	volatile unsigned long _res; \
2188	/* _argvec[0] holds current r2 across the call */ \
2189	_argvec[1] = (unsigned long)_orig.r2; \
2190	_argvec[2] = (unsigned long)_orig.nraddr; \
2191	_argvec[2+1] = (unsigned long)arg1; \
2192	_argvec[2+2] = (unsigned long)arg2; \
2193	_argvec[2+3] = (unsigned long)arg3; \
2194	_argvec[2+4] = (unsigned long)arg4; \
2195	_argvec[2+5] = (unsigned long)arg5; \
2196	_argvec[2+6] = (unsigned long)arg6; \
2197	_argvec[2+7] = (unsigned long)arg7; \
2198	_argvec[2+8] = (unsigned long)arg8; \
2199	__asm__ volatile( \
2200	"mr 11,%1\n\t" \
2201	"std 2,-16(11)\n\t" /* save tocptr */ \
2202	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2203	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2204	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2205	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2206	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2207	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2208	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
2209	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
2210	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
2211	"ld 11, 0(11)\n\t" /* target->r11 */ \
2212	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2213	"mr 11,%1\n\t" \
2214	"mr %0,3\n\t" \
2215	"ld 2,-16(11)" /* restore tocptr */ \
2216	: /out/ "=r" (_res) \
2217	: /in/ "r" (&_argvec[2]) \
2218	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2219	); \
2220	lval = (__typeof__(lval)) _res; \
2221	} while (0)
2222
2223	#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2224	arg7,arg8,arg9) \
2225	do { \
2226	volatile OrigFn _orig = (orig); \
2227	volatile unsigned long _argvec[3+9]; \
2228	volatile unsigned long _res; \
2229	/* _argvec[0] holds current r2 across the call */ \
2230	_argvec[1] = (unsigned long)_orig.r2; \
2231	_argvec[2] = (unsigned long)_orig.nraddr; \
2232	_argvec[2+1] = (unsigned long)arg1; \
2233	_argvec[2+2] = (unsigned long)arg2; \
2234	_argvec[2+3] = (unsigned long)arg3; \
2235	_argvec[2+4] = (unsigned long)arg4; \
2236	_argvec[2+5] = (unsigned long)arg5; \
2237	_argvec[2+6] = (unsigned long)arg6; \
2238	_argvec[2+7] = (unsigned long)arg7; \
2239	_argvec[2+8] = (unsigned long)arg8; \
2240	_argvec[2+9] = (unsigned long)arg9; \
2241	__asm__ volatile( \
2242	"mr 11,%1\n\t" \
2243	"std 2,-16(11)\n\t" /* save tocptr */ \
2244	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2245	"addi 1,1,-128\n\t" /* expand stack frame */ \
2246	/* arg9 */ \
2247	"ld 3,72(11)\n\t" \
2248	"std 3,112(1)\n\t" \
2249	/* args1-8 */ \
2250	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2251	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2252	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2253	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2254	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2255	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
2256	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
2257	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
2258	"ld 11, 0(11)\n\t" /* target->r11 */ \
2259	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2260	"mr 11,%1\n\t" \
2261	"mr %0,3\n\t" \
2262	"ld 2,-16(11)\n\t" /* restore tocptr */ \
2263	"addi 1,1,128" /* restore frame */ \
2264	: /out/ "=r" (_res) \
2265	: /in/ "r" (&_argvec[2]) \
2266	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2267	); \
2268	lval = (__typeof__(lval)) _res; \
2269	} while (0)
2270
2271	#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2272	arg7,arg8,arg9,arg10) \
2273	do { \
2274	volatile OrigFn _orig = (orig); \
2275	volatile unsigned long _argvec[3+10]; \
2276	volatile unsigned long _res; \
2277	/* _argvec[0] holds current r2 across the call */ \
2278	_argvec[1] = (unsigned long)_orig.r2; \
2279	_argvec[2] = (unsigned long)_orig.nraddr; \
2280	_argvec[2+1] = (unsigned long)arg1; \
2281	_argvec[2+2] = (unsigned long)arg2; \
2282	_argvec[2+3] = (unsigned long)arg3; \
2283	_argvec[2+4] = (unsigned long)arg4; \
2284	_argvec[2+5] = (unsigned long)arg5; \
2285	_argvec[2+6] = (unsigned long)arg6; \
2286	_argvec[2+7] = (unsigned long)arg7; \
2287	_argvec[2+8] = (unsigned long)arg8; \
2288	_argvec[2+9] = (unsigned long)arg9; \
2289	_argvec[2+10] = (unsigned long)arg10; \
2290	__asm__ volatile( \
2291	"mr 11,%1\n\t" \
2292	"std 2,-16(11)\n\t" /* save tocptr */ \
2293	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2294	"addi 1,1,-128\n\t" /* expand stack frame */ \
2295	/* arg10 */ \
2296	"ld 3,80(11)\n\t" \
2297	"std 3,120(1)\n\t" \
2298	/* arg9 */ \
2299	"ld 3,72(11)\n\t" \
2300	"std 3,112(1)\n\t" \
2301	/* args1-8 */ \
2302	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2303	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2304	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2305	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2306	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2307	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
2308	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
2309	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
2310	"ld 11, 0(11)\n\t" /* target->r11 */ \
2311	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2312	"mr 11,%1\n\t" \
2313	"mr %0,3\n\t" \
2314	"ld 2,-16(11)\n\t" /* restore tocptr */ \
2315	"addi 1,1,128" /* restore frame */ \
2316	: /out/ "=r" (_res) \
2317	: /in/ "r" (&_argvec[2]) \
2318	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2319	); \
2320	lval = (__typeof__(lval)) _res; \
2321	} while (0)
2322
2323	#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2324	arg7,arg8,arg9,arg10,arg11) \
2325	do { \
2326	volatile OrigFn _orig = (orig); \
2327	volatile unsigned long _argvec[3+11]; \
2328	volatile unsigned long _res; \
2329	/* _argvec[0] holds current r2 across the call */ \
2330	_argvec[1] = (unsigned long)_orig.r2; \
2331	_argvec[2] = (unsigned long)_orig.nraddr; \
2332	_argvec[2+1] = (unsigned long)arg1; \
2333	_argvec[2+2] = (unsigned long)arg2; \
2334	_argvec[2+3] = (unsigned long)arg3; \
2335	_argvec[2+4] = (unsigned long)arg4; \
2336	_argvec[2+5] = (unsigned long)arg5; \
2337	_argvec[2+6] = (unsigned long)arg6; \
2338	_argvec[2+7] = (unsigned long)arg7; \
2339	_argvec[2+8] = (unsigned long)arg8; \
2340	_argvec[2+9] = (unsigned long)arg9; \
2341	_argvec[2+10] = (unsigned long)arg10; \
2342	_argvec[2+11] = (unsigned long)arg11; \
2343	__asm__ volatile( \
2344	"mr 11,%1\n\t" \
2345	"std 2,-16(11)\n\t" /* save tocptr */ \
2346	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2347	"addi 1,1,-144\n\t" /* expand stack frame */ \
2348	/* arg11 */ \
2349	"ld 3,88(11)\n\t" \
2350	"std 3,128(1)\n\t" \
2351	/* arg10 */ \
2352	"ld 3,80(11)\n\t" \
2353	"std 3,120(1)\n\t" \
2354	/* arg9 */ \
2355	"ld 3,72(11)\n\t" \
2356	"std 3,112(1)\n\t" \
2357	/* args1-8 */ \
2358	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2359	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2360	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2361	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2362	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2363	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
2364	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
2365	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
2366	"ld 11, 0(11)\n\t" /* target->r11 */ \
2367	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2368	"mr 11,%1\n\t" \
2369	"mr %0,3\n\t" \
2370	"ld 2,-16(11)\n\t" /* restore tocptr */ \
2371	"addi 1,1,144" /* restore frame */ \
2372	: /out/ "=r" (_res) \
2373	: /in/ "r" (&_argvec[2]) \
2374	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2375	); \
2376	lval = (__typeof__(lval)) _res; \
2377	} while (0)
2378
2379	#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2380	arg7,arg8,arg9,arg10,arg11,arg12) \
2381	do { \
2382	volatile OrigFn _orig = (orig); \
2383	volatile unsigned long _argvec[3+12]; \
2384	volatile unsigned long _res; \
2385	/* _argvec[0] holds current r2 across the call */ \
2386	_argvec[1] = (unsigned long)_orig.r2; \
2387	_argvec[2] = (unsigned long)_orig.nraddr; \
2388	_argvec[2+1] = (unsigned long)arg1; \
2389	_argvec[2+2] = (unsigned long)arg2; \
2390	_argvec[2+3] = (unsigned long)arg3; \
2391	_argvec[2+4] = (unsigned long)arg4; \
2392	_argvec[2+5] = (unsigned long)arg5; \
2393	_argvec[2+6] = (unsigned long)arg6; \
2394	_argvec[2+7] = (unsigned long)arg7; \
2395	_argvec[2+8] = (unsigned long)arg8; \
2396	_argvec[2+9] = (unsigned long)arg9; \
2397	_argvec[2+10] = (unsigned long)arg10; \
2398	_argvec[2+11] = (unsigned long)arg11; \
2399	_argvec[2+12] = (unsigned long)arg12; \
2400	__asm__ volatile( \
2401	"mr 11,%1\n\t" \
2402	"std 2,-16(11)\n\t" /* save tocptr */ \
2403	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2404	"addi 1,1,-144\n\t" /* expand stack frame */ \
2405	/* arg12 */ \
2406	"ld 3,96(11)\n\t" \
2407	"std 3,136(1)\n\t" \
2408	/* arg11 */ \
2409	"ld 3,88(11)\n\t" \
2410	"std 3,128(1)\n\t" \
2411	/* arg10 */ \
2412	"ld 3,80(11)\n\t" \
2413	"std 3,120(1)\n\t" \
2414	/* arg9 */ \
2415	"ld 3,72(11)\n\t" \
2416	"std 3,112(1)\n\t" \
2417	/* args1-8 */ \
2418	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
2419	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
2420	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
2421	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
2422	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
2423	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
2424	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
2425	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
2426	"ld 11, 0(11)\n\t" /* target->r11 */ \
2427	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2428	"mr 11,%1\n\t" \
2429	"mr %0,3\n\t" \
2430	"ld 2,-16(11)\n\t" /* restore tocptr */ \
2431	"addi 1,1,144" /* restore frame */ \
2432	: /out/ "=r" (_res) \
2433	: /in/ "r" (&_argvec[2]) \
2434	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2435	); \
2436	lval = (__typeof__(lval)) _res; \
2437	} while (0)
2438
2439	#endif /* PLAT_ppc64_linux */
2440
2441	/* ------------------------ ppc32-aix5 ------------------------- */
2442
2443	#if defined(PLAT_ppc32_aix5)
2444
2445	/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2446
2447	/* These regs are trashed by the hidden call. */
2448	#define __CALLER_SAVED_REGS \
2449	"lr", "ctr", "xer", \
2450	"cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \
2451	"r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \
2452	"r11", "r12", "r13"
2453
2454	/* Expand the stack frame, copying enough info that unwinding
2455	still works. Trashes r3. */
2456
2457	#define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr) \
2458	"addi 1,1,-" #_n_fr "\n\t" \
2459	"lwz 3," #_n_fr "(1)\n\t" \
2460	"stw 3,0(1)\n\t"
2461
2462	#define VG_CONTRACT_FRAME_BY(_n_fr) \
2463	"addi 1,1," #_n_fr "\n\t"
2464
2465	/* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
2466	long) == 4. */
2467
2468	#define CALL_FN_W_v(lval, orig) \
2469	do { \
2470	volatile OrigFn _orig = (orig); \
2471	volatile unsigned long _argvec[3+0]; \
2472	volatile unsigned long _res; \
2473	/* _argvec[0] holds current r2 across the call */ \
2474	_argvec[1] = (unsigned long)_orig.r2; \
2475	_argvec[2] = (unsigned long)_orig.nraddr; \
2476	__asm__ volatile( \
2477	"mr 11,%1\n\t" \
2478	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2479	"stw 2,-8(11)\n\t" /* save tocptr */ \
2480	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2481	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2482	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2483	"mr 11,%1\n\t" \
2484	"mr %0,3\n\t" \
2485	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2486	VG_CONTRACT_FRAME_BY(512) \
2487	: /out/ "=r" (_res) \
2488	: /in/ "r" (&_argvec[2]) \
2489	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2490	); \
2491	lval = (__typeof__(lval)) _res; \
2492	} while (0)
2493
2494	#define CALL_FN_W_W(lval, orig, arg1) \
2495	do { \
2496	volatile OrigFn _orig = (orig); \
2497	volatile unsigned long _argvec[3+1]; \
2498	volatile unsigned long _res; \
2499	/* _argvec[0] holds current r2 across the call */ \
2500	_argvec[1] = (unsigned long)_orig.r2; \
2501	_argvec[2] = (unsigned long)_orig.nraddr; \
2502	_argvec[2+1] = (unsigned long)arg1; \
2503	__asm__ volatile( \
2504	"mr 11,%1\n\t" \
2505	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2506	"stw 2,-8(11)\n\t" /* save tocptr */ \
2507	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2508	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2509	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2510	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2511	"mr 11,%1\n\t" \
2512	"mr %0,3\n\t" \
2513	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2514	VG_CONTRACT_FRAME_BY(512) \
2515	: /out/ "=r" (_res) \
2516	: /in/ "r" (&_argvec[2]) \
2517	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2518	); \
2519	lval = (__typeof__(lval)) _res; \
2520	} while (0)
2521
2522	#define CALL_FN_W_WW(lval, orig, arg1,arg2) \
2523	do { \
2524	volatile OrigFn _orig = (orig); \
2525	volatile unsigned long _argvec[3+2]; \
2526	volatile unsigned long _res; \
2527	/* _argvec[0] holds current r2 across the call */ \
2528	_argvec[1] = (unsigned long)_orig.r2; \
2529	_argvec[2] = (unsigned long)_orig.nraddr; \
2530	_argvec[2+1] = (unsigned long)arg1; \
2531	_argvec[2+2] = (unsigned long)arg2; \
2532	__asm__ volatile( \
2533	"mr 11,%1\n\t" \
2534	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2535	"stw 2,-8(11)\n\t" /* save tocptr */ \
2536	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2537	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2538	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2539	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2540	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2541	"mr 11,%1\n\t" \
2542	"mr %0,3\n\t" \
2543	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2544	VG_CONTRACT_FRAME_BY(512) \
2545	: /out/ "=r" (_res) \
2546	: /in/ "r" (&_argvec[2]) \
2547	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2548	); \
2549	lval = (__typeof__(lval)) _res; \
2550	} while (0)
2551
2552	#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
2553	do { \
2554	volatile OrigFn _orig = (orig); \
2555	volatile unsigned long _argvec[3+3]; \
2556	volatile unsigned long _res; \
2557	/* _argvec[0] holds current r2 across the call */ \
2558	_argvec[1] = (unsigned long)_orig.r2; \
2559	_argvec[2] = (unsigned long)_orig.nraddr; \
2560	_argvec[2+1] = (unsigned long)arg1; \
2561	_argvec[2+2] = (unsigned long)arg2; \
2562	_argvec[2+3] = (unsigned long)arg3; \
2563	__asm__ volatile( \
2564	"mr 11,%1\n\t" \
2565	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2566	"stw 2,-8(11)\n\t" /* save tocptr */ \
2567	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2568	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2569	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2570	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2571	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2572	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2573	"mr 11,%1\n\t" \
2574	"mr %0,3\n\t" \
2575	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2576	VG_CONTRACT_FRAME_BY(512) \
2577	: /out/ "=r" (_res) \
2578	: /in/ "r" (&_argvec[2]) \
2579	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2580	); \
2581	lval = (__typeof__(lval)) _res; \
2582	} while (0)
2583
2584	#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
2585	do { \
2586	volatile OrigFn _orig = (orig); \
2587	volatile unsigned long _argvec[3+4]; \
2588	volatile unsigned long _res; \
2589	/* _argvec[0] holds current r2 across the call */ \
2590	_argvec[1] = (unsigned long)_orig.r2; \
2591	_argvec[2] = (unsigned long)_orig.nraddr; \
2592	_argvec[2+1] = (unsigned long)arg1; \
2593	_argvec[2+2] = (unsigned long)arg2; \
2594	_argvec[2+3] = (unsigned long)arg3; \
2595	_argvec[2+4] = (unsigned long)arg4; \
2596	__asm__ volatile( \
2597	"mr 11,%1\n\t" \
2598	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2599	"stw 2,-8(11)\n\t" /* save tocptr */ \
2600	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2601	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2602	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2603	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2604	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2605	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2606	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2607	"mr 11,%1\n\t" \
2608	"mr %0,3\n\t" \
2609	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2610	VG_CONTRACT_FRAME_BY(512) \
2611	: /out/ "=r" (_res) \
2612	: /in/ "r" (&_argvec[2]) \
2613	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2614	); \
2615	lval = (__typeof__(lval)) _res; \
2616	} while (0)
2617
2618	#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
2619	do { \
2620	volatile OrigFn _orig = (orig); \
2621	volatile unsigned long _argvec[3+5]; \
2622	volatile unsigned long _res; \
2623	/* _argvec[0] holds current r2 across the call */ \
2624	_argvec[1] = (unsigned long)_orig.r2; \
2625	_argvec[2] = (unsigned long)_orig.nraddr; \
2626	_argvec[2+1] = (unsigned long)arg1; \
2627	_argvec[2+2] = (unsigned long)arg2; \
2628	_argvec[2+3] = (unsigned long)arg3; \
2629	_argvec[2+4] = (unsigned long)arg4; \
2630	_argvec[2+5] = (unsigned long)arg5; \
2631	__asm__ volatile( \
2632	"mr 11,%1\n\t" \
2633	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2634	"stw 2,-8(11)\n\t" /* save tocptr */ \
2635	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2636	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2637	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2638	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2639	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2640	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2641	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2642	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2643	"mr 11,%1\n\t" \
2644	"mr %0,3\n\t" \
2645	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2646	VG_CONTRACT_FRAME_BY(512) \
2647	: /out/ "=r" (_res) \
2648	: /in/ "r" (&_argvec[2]) \
2649	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2650	); \
2651	lval = (__typeof__(lval)) _res; \
2652	} while (0)
2653
2654	#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
2655	do { \
2656	volatile OrigFn _orig = (orig); \
2657	volatile unsigned long _argvec[3+6]; \
2658	volatile unsigned long _res; \
2659	/* _argvec[0] holds current r2 across the call */ \
2660	_argvec[1] = (unsigned long)_orig.r2; \
2661	_argvec[2] = (unsigned long)_orig.nraddr; \
2662	_argvec[2+1] = (unsigned long)arg1; \
2663	_argvec[2+2] = (unsigned long)arg2; \
2664	_argvec[2+3] = (unsigned long)arg3; \
2665	_argvec[2+4] = (unsigned long)arg4; \
2666	_argvec[2+5] = (unsigned long)arg5; \
2667	_argvec[2+6] = (unsigned long)arg6; \
2668	__asm__ volatile( \
2669	"mr 11,%1\n\t" \
2670	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2671	"stw 2,-8(11)\n\t" /* save tocptr */ \
2672	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2673	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2674	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2675	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2676	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2677	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2678	"lwz 8, 24(11)\n\t" /* arg6->r8 */ \
2679	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2680	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2681	"mr 11,%1\n\t" \
2682	"mr %0,3\n\t" \
2683	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2684	VG_CONTRACT_FRAME_BY(512) \
2685	: /out/ "=r" (_res) \
2686	: /in/ "r" (&_argvec[2]) \
2687	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2688	); \
2689	lval = (__typeof__(lval)) _res; \
2690	} while (0)
2691
2692	#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2693	arg7) \
2694	do { \
2695	volatile OrigFn _orig = (orig); \
2696	volatile unsigned long _argvec[3+7]; \
2697	volatile unsigned long _res; \
2698	/* _argvec[0] holds current r2 across the call */ \
2699	_argvec[1] = (unsigned long)_orig.r2; \
2700	_argvec[2] = (unsigned long)_orig.nraddr; \
2701	_argvec[2+1] = (unsigned long)arg1; \
2702	_argvec[2+2] = (unsigned long)arg2; \
2703	_argvec[2+3] = (unsigned long)arg3; \
2704	_argvec[2+4] = (unsigned long)arg4; \
2705	_argvec[2+5] = (unsigned long)arg5; \
2706	_argvec[2+6] = (unsigned long)arg6; \
2707	_argvec[2+7] = (unsigned long)arg7; \
2708	__asm__ volatile( \
2709	"mr 11,%1\n\t" \
2710	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2711	"stw 2,-8(11)\n\t" /* save tocptr */ \
2712	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2713	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2714	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2715	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2716	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2717	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2718	"lwz 8, 24(11)\n\t" /* arg6->r8 */ \
2719	"lwz 9, 28(11)\n\t" /* arg7->r9 */ \
2720	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2721	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2722	"mr 11,%1\n\t" \
2723	"mr %0,3\n\t" \
2724	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2725	VG_CONTRACT_FRAME_BY(512) \
2726	: /out/ "=r" (_res) \
2727	: /in/ "r" (&_argvec[2]) \
2728	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2729	); \
2730	lval = (__typeof__(lval)) _res; \
2731	} while (0)
2732
2733	#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2734	arg7,arg8) \
2735	do { \
2736	volatile OrigFn _orig = (orig); \
2737	volatile unsigned long _argvec[3+8]; \
2738	volatile unsigned long _res; \
2739	/* _argvec[0] holds current r2 across the call */ \
2740	_argvec[1] = (unsigned long)_orig.r2; \
2741	_argvec[2] = (unsigned long)_orig.nraddr; \
2742	_argvec[2+1] = (unsigned long)arg1; \
2743	_argvec[2+2] = (unsigned long)arg2; \
2744	_argvec[2+3] = (unsigned long)arg3; \
2745	_argvec[2+4] = (unsigned long)arg4; \
2746	_argvec[2+5] = (unsigned long)arg5; \
2747	_argvec[2+6] = (unsigned long)arg6; \
2748	_argvec[2+7] = (unsigned long)arg7; \
2749	_argvec[2+8] = (unsigned long)arg8; \
2750	__asm__ volatile( \
2751	"mr 11,%1\n\t" \
2752	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2753	"stw 2,-8(11)\n\t" /* save tocptr */ \
2754	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2755	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2756	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2757	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2758	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2759	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2760	"lwz 8, 24(11)\n\t" /* arg6->r8 */ \
2761	"lwz 9, 28(11)\n\t" /* arg7->r9 */ \
2762	"lwz 10, 32(11)\n\t" /* arg8->r10 */ \
2763	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2764	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2765	"mr 11,%1\n\t" \
2766	"mr %0,3\n\t" \
2767	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2768	VG_CONTRACT_FRAME_BY(512) \
2769	: /out/ "=r" (_res) \
2770	: /in/ "r" (&_argvec[2]) \
2771	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2772	); \
2773	lval = (__typeof__(lval)) _res; \
2774	} while (0)
2775
2776	#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2777	arg7,arg8,arg9) \
2778	do { \
2779	volatile OrigFn _orig = (orig); \
2780	volatile unsigned long _argvec[3+9]; \
2781	volatile unsigned long _res; \
2782	/* _argvec[0] holds current r2 across the call */ \
2783	_argvec[1] = (unsigned long)_orig.r2; \
2784	_argvec[2] = (unsigned long)_orig.nraddr; \
2785	_argvec[2+1] = (unsigned long)arg1; \
2786	_argvec[2+2] = (unsigned long)arg2; \
2787	_argvec[2+3] = (unsigned long)arg3; \
2788	_argvec[2+4] = (unsigned long)arg4; \
2789	_argvec[2+5] = (unsigned long)arg5; \
2790	_argvec[2+6] = (unsigned long)arg6; \
2791	_argvec[2+7] = (unsigned long)arg7; \
2792	_argvec[2+8] = (unsigned long)arg8; \
2793	_argvec[2+9] = (unsigned long)arg9; \
2794	__asm__ volatile( \
2795	"mr 11,%1\n\t" \
2796	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2797	"stw 2,-8(11)\n\t" /* save tocptr */ \
2798	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2799	VG_EXPAND_FRAME_BY_trashes_r3(64) \
2800	/* arg9 */ \
2801	"lwz 3,36(11)\n\t" \
2802	"stw 3,56(1)\n\t" \
2803	/* args1-8 */ \
2804	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2805	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2806	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2807	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2808	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2809	"lwz 8, 24(11)\n\t" /* arg6->r8 */ \
2810	"lwz 9, 28(11)\n\t" /* arg7->r9 */ \
2811	"lwz 10, 32(11)\n\t" /* arg8->r10 */ \
2812	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2813	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2814	"mr 11,%1\n\t" \
2815	"mr %0,3\n\t" \
2816	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2817	VG_CONTRACT_FRAME_BY(64) \
2818	VG_CONTRACT_FRAME_BY(512) \
2819	: /out/ "=r" (_res) \
2820	: /in/ "r" (&_argvec[2]) \
2821	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2822	); \
2823	lval = (__typeof__(lval)) _res; \
2824	} while (0)
2825
2826	#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2827	arg7,arg8,arg9,arg10) \
2828	do { \
2829	volatile OrigFn _orig = (orig); \
2830	volatile unsigned long _argvec[3+10]; \
2831	volatile unsigned long _res; \
2832	/* _argvec[0] holds current r2 across the call */ \
2833	_argvec[1] = (unsigned long)_orig.r2; \
2834	_argvec[2] = (unsigned long)_orig.nraddr; \
2835	_argvec[2+1] = (unsigned long)arg1; \
2836	_argvec[2+2] = (unsigned long)arg2; \
2837	_argvec[2+3] = (unsigned long)arg3; \
2838	_argvec[2+4] = (unsigned long)arg4; \
2839	_argvec[2+5] = (unsigned long)arg5; \
2840	_argvec[2+6] = (unsigned long)arg6; \
2841	_argvec[2+7] = (unsigned long)arg7; \
2842	_argvec[2+8] = (unsigned long)arg8; \
2843	_argvec[2+9] = (unsigned long)arg9; \
2844	_argvec[2+10] = (unsigned long)arg10; \
2845	__asm__ volatile( \
2846	"mr 11,%1\n\t" \
2847	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2848	"stw 2,-8(11)\n\t" /* save tocptr */ \
2849	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2850	VG_EXPAND_FRAME_BY_trashes_r3(64) \
2851	/* arg10 */ \
2852	"lwz 3,40(11)\n\t" \
2853	"stw 3,60(1)\n\t" \
2854	/* arg9 */ \
2855	"lwz 3,36(11)\n\t" \
2856	"stw 3,56(1)\n\t" \
2857	/* args1-8 */ \
2858	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2859	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2860	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2861	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2862	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2863	"lwz 8, 24(11)\n\t" /* arg6->r8 */ \
2864	"lwz 9, 28(11)\n\t" /* arg7->r9 */ \
2865	"lwz 10, 32(11)\n\t" /* arg8->r10 */ \
2866	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2867	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2868	"mr 11,%1\n\t" \
2869	"mr %0,3\n\t" \
2870	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2871	VG_CONTRACT_FRAME_BY(64) \
2872	VG_CONTRACT_FRAME_BY(512) \
2873	: /out/ "=r" (_res) \
2874	: /in/ "r" (&_argvec[2]) \
2875	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2876	); \
2877	lval = (__typeof__(lval)) _res; \
2878	} while (0)
2879
2880	#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2881	arg7,arg8,arg9,arg10,arg11) \
2882	do { \
2883	volatile OrigFn _orig = (orig); \
2884	volatile unsigned long _argvec[3+11]; \
2885	volatile unsigned long _res; \
2886	/* _argvec[0] holds current r2 across the call */ \
2887	_argvec[1] = (unsigned long)_orig.r2; \
2888	_argvec[2] = (unsigned long)_orig.nraddr; \
2889	_argvec[2+1] = (unsigned long)arg1; \
2890	_argvec[2+2] = (unsigned long)arg2; \
2891	_argvec[2+3] = (unsigned long)arg3; \
2892	_argvec[2+4] = (unsigned long)arg4; \
2893	_argvec[2+5] = (unsigned long)arg5; \
2894	_argvec[2+6] = (unsigned long)arg6; \
2895	_argvec[2+7] = (unsigned long)arg7; \
2896	_argvec[2+8] = (unsigned long)arg8; \
2897	_argvec[2+9] = (unsigned long)arg9; \
2898	_argvec[2+10] = (unsigned long)arg10; \
2899	_argvec[2+11] = (unsigned long)arg11; \
2900	__asm__ volatile( \
2901	"mr 11,%1\n\t" \
2902	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2903	"stw 2,-8(11)\n\t" /* save tocptr */ \
2904	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2905	VG_EXPAND_FRAME_BY_trashes_r3(72) \
2906	/* arg11 */ \
2907	"lwz 3,44(11)\n\t" \
2908	"stw 3,64(1)\n\t" \
2909	/* arg10 */ \
2910	"lwz 3,40(11)\n\t" \
2911	"stw 3,60(1)\n\t" \
2912	/* arg9 */ \
2913	"lwz 3,36(11)\n\t" \
2914	"stw 3,56(1)\n\t" \
2915	/* args1-8 */ \
2916	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2917	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2918	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2919	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2920	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2921	"lwz 8, 24(11)\n\t" /* arg6->r8 */ \
2922	"lwz 9, 28(11)\n\t" /* arg7->r9 */ \
2923	"lwz 10, 32(11)\n\t" /* arg8->r10 */ \
2924	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2925	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2926	"mr 11,%1\n\t" \
2927	"mr %0,3\n\t" \
2928	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2929	VG_CONTRACT_FRAME_BY(72) \
2930	VG_CONTRACT_FRAME_BY(512) \
2931	: /out/ "=r" (_res) \
2932	: /in/ "r" (&_argvec[2]) \
2933	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2934	); \
2935	lval = (__typeof__(lval)) _res; \
2936	} while (0)
2937
2938	#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2939	arg7,arg8,arg9,arg10,arg11,arg12) \
2940	do { \
2941	volatile OrigFn _orig = (orig); \
2942	volatile unsigned long _argvec[3+12]; \
2943	volatile unsigned long _res; \
2944	/* _argvec[0] holds current r2 across the call */ \
2945	_argvec[1] = (unsigned long)_orig.r2; \
2946	_argvec[2] = (unsigned long)_orig.nraddr; \
2947	_argvec[2+1] = (unsigned long)arg1; \
2948	_argvec[2+2] = (unsigned long)arg2; \
2949	_argvec[2+3] = (unsigned long)arg3; \
2950	_argvec[2+4] = (unsigned long)arg4; \
2951	_argvec[2+5] = (unsigned long)arg5; \
2952	_argvec[2+6] = (unsigned long)arg6; \
2953	_argvec[2+7] = (unsigned long)arg7; \
2954	_argvec[2+8] = (unsigned long)arg8; \
2955	_argvec[2+9] = (unsigned long)arg9; \
2956	_argvec[2+10] = (unsigned long)arg10; \
2957	_argvec[2+11] = (unsigned long)arg11; \
2958	_argvec[2+12] = (unsigned long)arg12; \
2959	__asm__ volatile( \
2960	"mr 11,%1\n\t" \
2961	VG_EXPAND_FRAME_BY_trashes_r3(512) \
2962	"stw 2,-8(11)\n\t" /* save tocptr */ \
2963	"lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \
2964	VG_EXPAND_FRAME_BY_trashes_r3(72) \
2965	/* arg12 */ \
2966	"lwz 3,48(11)\n\t" \
2967	"stw 3,68(1)\n\t" \
2968	/* arg11 */ \
2969	"lwz 3,44(11)\n\t" \
2970	"stw 3,64(1)\n\t" \
2971	/* arg10 */ \
2972	"lwz 3,40(11)\n\t" \
2973	"stw 3,60(1)\n\t" \
2974	/* arg9 */ \
2975	"lwz 3,36(11)\n\t" \
2976	"stw 3,56(1)\n\t" \
2977	/* args1-8 */ \
2978	"lwz 3, 4(11)\n\t" /* arg1->r3 */ \
2979	"lwz 4, 8(11)\n\t" /* arg2->r4 */ \
2980	"lwz 5, 12(11)\n\t" /* arg3->r5 */ \
2981	"lwz 6, 16(11)\n\t" /* arg4->r6 */ \
2982	"lwz 7, 20(11)\n\t" /* arg5->r7 */ \
2983	"lwz 8, 24(11)\n\t" /* arg6->r8 */ \
2984	"lwz 9, 28(11)\n\t" /* arg7->r9 */ \
2985	"lwz 10, 32(11)\n\t" /* arg8->r10 */ \
2986	"lwz 11, 0(11)\n\t" /* target->r11 */ \
2987	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2988	"mr 11,%1\n\t" \
2989	"mr %0,3\n\t" \
2990	"lwz 2,-8(11)\n\t" /* restore tocptr */ \
2991	VG_CONTRACT_FRAME_BY(72) \
2992	VG_CONTRACT_FRAME_BY(512) \
2993	: /out/ "=r" (_res) \
2994	: /in/ "r" (&_argvec[2]) \
2995	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
2996	); \
2997	lval = (__typeof__(lval)) _res; \
2998	} while (0)
2999
3000	#endif /* PLAT_ppc32_aix5 */
3001
3002	/* ------------------------ ppc64-aix5 ------------------------- */
3003
3004	#if defined(PLAT_ppc64_aix5)
3005
3006	/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3007
3008	/* These regs are trashed by the hidden call. */
3009	#define __CALLER_SAVED_REGS \
3010	"lr", "ctr", "xer", \
3011	"cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \
3012	"r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \
3013	"r11", "r12", "r13"
3014
3015	/* Expand the stack frame, copying enough info that unwinding
3016	still works. Trashes r3. */
3017
3018	#define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr) \
3019	"addi 1,1,-" #_n_fr "\n\t" \
3020	"ld 3," #_n_fr "(1)\n\t" \
3021	"std 3,0(1)\n\t"
3022
3023	#define VG_CONTRACT_FRAME_BY(_n_fr) \
3024	"addi 1,1," #_n_fr "\n\t"
3025
3026	/* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3027	long) == 8. */
3028
3029	#define CALL_FN_W_v(lval, orig) \
3030	do { \
3031	volatile OrigFn _orig = (orig); \
3032	volatile unsigned long _argvec[3+0]; \
3033	volatile unsigned long _res; \
3034	/* _argvec[0] holds current r2 across the call */ \
3035	_argvec[1] = (unsigned long)_orig.r2; \
3036	_argvec[2] = (unsigned long)_orig.nraddr; \
3037	__asm__ volatile( \
3038	"mr 11,%1\n\t" \
3039	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3040	"std 2,-16(11)\n\t" /* save tocptr */ \
3041	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3042	"ld 11, 0(11)\n\t" /* target->r11 */ \
3043	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3044	"mr 11,%1\n\t" \
3045	"mr %0,3\n\t" \
3046	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3047	VG_CONTRACT_FRAME_BY(512) \
3048	: /out/ "=r" (_res) \
3049	: /in/ "r" (&_argvec[2]) \
3050	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3051	); \
3052	lval = (__typeof__(lval)) _res; \
3053	} while (0)
3054
3055	#define CALL_FN_W_W(lval, orig, arg1) \
3056	do { \
3057	volatile OrigFn _orig = (orig); \
3058	volatile unsigned long _argvec[3+1]; \
3059	volatile unsigned long _res; \
3060	/* _argvec[0] holds current r2 across the call */ \
3061	_argvec[1] = (unsigned long)_orig.r2; \
3062	_argvec[2] = (unsigned long)_orig.nraddr; \
3063	_argvec[2+1] = (unsigned long)arg1; \
3064	__asm__ volatile( \
3065	"mr 11,%1\n\t" \
3066	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3067	"std 2,-16(11)\n\t" /* save tocptr */ \
3068	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3069	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3070	"ld 11, 0(11)\n\t" /* target->r11 */ \
3071	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3072	"mr 11,%1\n\t" \
3073	"mr %0,3\n\t" \
3074	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3075	VG_CONTRACT_FRAME_BY(512) \
3076	: /out/ "=r" (_res) \
3077	: /in/ "r" (&_argvec[2]) \
3078	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3079	); \
3080	lval = (__typeof__(lval)) _res; \
3081	} while (0)
3082
3083	#define CALL_FN_W_WW(lval, orig, arg1,arg2) \
3084	do { \
3085	volatile OrigFn _orig = (orig); \
3086	volatile unsigned long _argvec[3+2]; \
3087	volatile unsigned long _res; \
3088	/* _argvec[0] holds current r2 across the call */ \
3089	_argvec[1] = (unsigned long)_orig.r2; \
3090	_argvec[2] = (unsigned long)_orig.nraddr; \
3091	_argvec[2+1] = (unsigned long)arg1; \
3092	_argvec[2+2] = (unsigned long)arg2; \
3093	__asm__ volatile( \
3094	"mr 11,%1\n\t" \
3095	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3096	"std 2,-16(11)\n\t" /* save tocptr */ \
3097	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3098	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3099	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3100	"ld 11, 0(11)\n\t" /* target->r11 */ \
3101	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3102	"mr 11,%1\n\t" \
3103	"mr %0,3\n\t" \
3104	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3105	VG_CONTRACT_FRAME_BY(512) \
3106	: /out/ "=r" (_res) \
3107	: /in/ "r" (&_argvec[2]) \
3108	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3109	); \
3110	lval = (__typeof__(lval)) _res; \
3111	} while (0)
3112
3113	#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
3114	do { \
3115	volatile OrigFn _orig = (orig); \
3116	volatile unsigned long _argvec[3+3]; \
3117	volatile unsigned long _res; \
3118	/* _argvec[0] holds current r2 across the call */ \
3119	_argvec[1] = (unsigned long)_orig.r2; \
3120	_argvec[2] = (unsigned long)_orig.nraddr; \
3121	_argvec[2+1] = (unsigned long)arg1; \
3122	_argvec[2+2] = (unsigned long)arg2; \
3123	_argvec[2+3] = (unsigned long)arg3; \
3124	__asm__ volatile( \
3125	"mr 11,%1\n\t" \
3126	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3127	"std 2,-16(11)\n\t" /* save tocptr */ \
3128	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3129	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3130	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3131	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3132	"ld 11, 0(11)\n\t" /* target->r11 */ \
3133	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3134	"mr 11,%1\n\t" \
3135	"mr %0,3\n\t" \
3136	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3137	VG_CONTRACT_FRAME_BY(512) \
3138	: /out/ "=r" (_res) \
3139	: /in/ "r" (&_argvec[2]) \
3140	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3141	); \
3142	lval = (__typeof__(lval)) _res; \
3143	} while (0)
3144
3145	#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
3146	do { \
3147	volatile OrigFn _orig = (orig); \
3148	volatile unsigned long _argvec[3+4]; \
3149	volatile unsigned long _res; \
3150	/* _argvec[0] holds current r2 across the call */ \
3151	_argvec[1] = (unsigned long)_orig.r2; \
3152	_argvec[2] = (unsigned long)_orig.nraddr; \
3153	_argvec[2+1] = (unsigned long)arg1; \
3154	_argvec[2+2] = (unsigned long)arg2; \
3155	_argvec[2+3] = (unsigned long)arg3; \
3156	_argvec[2+4] = (unsigned long)arg4; \
3157	__asm__ volatile( \
3158	"mr 11,%1\n\t" \
3159	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3160	"std 2,-16(11)\n\t" /* save tocptr */ \
3161	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3162	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3163	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3164	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3165	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3166	"ld 11, 0(11)\n\t" /* target->r11 */ \
3167	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3168	"mr 11,%1\n\t" \
3169	"mr %0,3\n\t" \
3170	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3171	VG_CONTRACT_FRAME_BY(512) \
3172	: /out/ "=r" (_res) \
3173	: /in/ "r" (&_argvec[2]) \
3174	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3175	); \
3176	lval = (__typeof__(lval)) _res; \
3177	} while (0)
3178
3179	#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
3180	do { \
3181	volatile OrigFn _orig = (orig); \
3182	volatile unsigned long _argvec[3+5]; \
3183	volatile unsigned long _res; \
3184	/* _argvec[0] holds current r2 across the call */ \
3185	_argvec[1] = (unsigned long)_orig.r2; \
3186	_argvec[2] = (unsigned long)_orig.nraddr; \
3187	_argvec[2+1] = (unsigned long)arg1; \
3188	_argvec[2+2] = (unsigned long)arg2; \
3189	_argvec[2+3] = (unsigned long)arg3; \
3190	_argvec[2+4] = (unsigned long)arg4; \
3191	_argvec[2+5] = (unsigned long)arg5; \
3192	__asm__ volatile( \
3193	"mr 11,%1\n\t" \
3194	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3195	"std 2,-16(11)\n\t" /* save tocptr */ \
3196	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3197	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3198	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3199	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3200	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3201	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3202	"ld 11, 0(11)\n\t" /* target->r11 */ \
3203	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3204	"mr 11,%1\n\t" \
3205	"mr %0,3\n\t" \
3206	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3207	VG_CONTRACT_FRAME_BY(512) \
3208	: /out/ "=r" (_res) \
3209	: /in/ "r" (&_argvec[2]) \
3210	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3211	); \
3212	lval = (__typeof__(lval)) _res; \
3213	} while (0)
3214
3215	#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
3216	do { \
3217	volatile OrigFn _orig = (orig); \
3218	volatile unsigned long _argvec[3+6]; \
3219	volatile unsigned long _res; \
3220	/* _argvec[0] holds current r2 across the call */ \
3221	_argvec[1] = (unsigned long)_orig.r2; \
3222	_argvec[2] = (unsigned long)_orig.nraddr; \
3223	_argvec[2+1] = (unsigned long)arg1; \
3224	_argvec[2+2] = (unsigned long)arg2; \
3225	_argvec[2+3] = (unsigned long)arg3; \
3226	_argvec[2+4] = (unsigned long)arg4; \
3227	_argvec[2+5] = (unsigned long)arg5; \
3228	_argvec[2+6] = (unsigned long)arg6; \
3229	__asm__ volatile( \
3230	"mr 11,%1\n\t" \
3231	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3232	"std 2,-16(11)\n\t" /* save tocptr */ \
3233	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3234	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3235	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3236	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3237	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3238	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3239	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
3240	"ld 11, 0(11)\n\t" /* target->r11 */ \
3241	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3242	"mr 11,%1\n\t" \
3243	"mr %0,3\n\t" \
3244	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3245	VG_CONTRACT_FRAME_BY(512) \
3246	: /out/ "=r" (_res) \
3247	: /in/ "r" (&_argvec[2]) \
3248	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3249	); \
3250	lval = (__typeof__(lval)) _res; \
3251	} while (0)
3252
3253	#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3254	arg7) \
3255	do { \
3256	volatile OrigFn _orig = (orig); \
3257	volatile unsigned long _argvec[3+7]; \
3258	volatile unsigned long _res; \
3259	/* _argvec[0] holds current r2 across the call */ \
3260	_argvec[1] = (unsigned long)_orig.r2; \
3261	_argvec[2] = (unsigned long)_orig.nraddr; \
3262	_argvec[2+1] = (unsigned long)arg1; \
3263	_argvec[2+2] = (unsigned long)arg2; \
3264	_argvec[2+3] = (unsigned long)arg3; \
3265	_argvec[2+4] = (unsigned long)arg4; \
3266	_argvec[2+5] = (unsigned long)arg5; \
3267	_argvec[2+6] = (unsigned long)arg6; \
3268	_argvec[2+7] = (unsigned long)arg7; \
3269	__asm__ volatile( \
3270	"mr 11,%1\n\t" \
3271	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3272	"std 2,-16(11)\n\t" /* save tocptr */ \
3273	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3274	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3275	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3276	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3277	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3278	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3279	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
3280	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
3281	"ld 11, 0(11)\n\t" /* target->r11 */ \
3282	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3283	"mr 11,%1\n\t" \
3284	"mr %0,3\n\t" \
3285	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3286	VG_CONTRACT_FRAME_BY(512) \
3287	: /out/ "=r" (_res) \
3288	: /in/ "r" (&_argvec[2]) \
3289	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3290	); \
3291	lval = (__typeof__(lval)) _res; \
3292	} while (0)
3293
3294	#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3295	arg7,arg8) \
3296	do { \
3297	volatile OrigFn _orig = (orig); \
3298	volatile unsigned long _argvec[3+8]; \
3299	volatile unsigned long _res; \
3300	/* _argvec[0] holds current r2 across the call */ \
3301	_argvec[1] = (unsigned long)_orig.r2; \
3302	_argvec[2] = (unsigned long)_orig.nraddr; \
3303	_argvec[2+1] = (unsigned long)arg1; \
3304	_argvec[2+2] = (unsigned long)arg2; \
3305	_argvec[2+3] = (unsigned long)arg3; \
3306	_argvec[2+4] = (unsigned long)arg4; \
3307	_argvec[2+5] = (unsigned long)arg5; \
3308	_argvec[2+6] = (unsigned long)arg6; \
3309	_argvec[2+7] = (unsigned long)arg7; \
3310	_argvec[2+8] = (unsigned long)arg8; \
3311	__asm__ volatile( \
3312	"mr 11,%1\n\t" \
3313	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3314	"std 2,-16(11)\n\t" /* save tocptr */ \
3315	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3316	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3317	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3318	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3319	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3320	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3321	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
3322	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
3323	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
3324	"ld 11, 0(11)\n\t" /* target->r11 */ \
3325	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3326	"mr 11,%1\n\t" \
3327	"mr %0,3\n\t" \
3328	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3329	VG_CONTRACT_FRAME_BY(512) \
3330	: /out/ "=r" (_res) \
3331	: /in/ "r" (&_argvec[2]) \
3332	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3333	); \
3334	lval = (__typeof__(lval)) _res; \
3335	} while (0)
3336
3337	#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3338	arg7,arg8,arg9) \
3339	do { \
3340	volatile OrigFn _orig = (orig); \
3341	volatile unsigned long _argvec[3+9]; \
3342	volatile unsigned long _res; \
3343	/* _argvec[0] holds current r2 across the call */ \
3344	_argvec[1] = (unsigned long)_orig.r2; \
3345	_argvec[2] = (unsigned long)_orig.nraddr; \
3346	_argvec[2+1] = (unsigned long)arg1; \
3347	_argvec[2+2] = (unsigned long)arg2; \
3348	_argvec[2+3] = (unsigned long)arg3; \
3349	_argvec[2+4] = (unsigned long)arg4; \
3350	_argvec[2+5] = (unsigned long)arg5; \
3351	_argvec[2+6] = (unsigned long)arg6; \
3352	_argvec[2+7] = (unsigned long)arg7; \
3353	_argvec[2+8] = (unsigned long)arg8; \
3354	_argvec[2+9] = (unsigned long)arg9; \
3355	__asm__ volatile( \
3356	"mr 11,%1\n\t" \
3357	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3358	"std 2,-16(11)\n\t" /* save tocptr */ \
3359	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3360	VG_EXPAND_FRAME_BY_trashes_r3(128) \
3361	/* arg9 */ \
3362	"ld 3,72(11)\n\t" \
3363	"std 3,112(1)\n\t" \
3364	/* args1-8 */ \
3365	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3366	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3367	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3368	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3369	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3370	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
3371	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
3372	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
3373	"ld 11, 0(11)\n\t" /* target->r11 */ \
3374	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3375	"mr 11,%1\n\t" \
3376	"mr %0,3\n\t" \
3377	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3378	VG_CONTRACT_FRAME_BY(128) \
3379	VG_CONTRACT_FRAME_BY(512) \
3380	: /out/ "=r" (_res) \
3381	: /in/ "r" (&_argvec[2]) \
3382	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3383	); \
3384	lval = (__typeof__(lval)) _res; \
3385	} while (0)
3386
3387	#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3388	arg7,arg8,arg9,arg10) \
3389	do { \
3390	volatile OrigFn _orig = (orig); \
3391	volatile unsigned long _argvec[3+10]; \
3392	volatile unsigned long _res; \
3393	/* _argvec[0] holds current r2 across the call */ \
3394	_argvec[1] = (unsigned long)_orig.r2; \
3395	_argvec[2] = (unsigned long)_orig.nraddr; \
3396	_argvec[2+1] = (unsigned long)arg1; \
3397	_argvec[2+2] = (unsigned long)arg2; \
3398	_argvec[2+3] = (unsigned long)arg3; \
3399	_argvec[2+4] = (unsigned long)arg4; \
3400	_argvec[2+5] = (unsigned long)arg5; \
3401	_argvec[2+6] = (unsigned long)arg6; \
3402	_argvec[2+7] = (unsigned long)arg7; \
3403	_argvec[2+8] = (unsigned long)arg8; \
3404	_argvec[2+9] = (unsigned long)arg9; \
3405	_argvec[2+10] = (unsigned long)arg10; \
3406	__asm__ volatile( \
3407	"mr 11,%1\n\t" \
3408	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3409	"std 2,-16(11)\n\t" /* save tocptr */ \
3410	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3411	VG_EXPAND_FRAME_BY_trashes_r3(128) \
3412	/* arg10 */ \
3413	"ld 3,80(11)\n\t" \
3414	"std 3,120(1)\n\t" \
3415	/* arg9 */ \
3416	"ld 3,72(11)\n\t" \
3417	"std 3,112(1)\n\t" \
3418	/* args1-8 */ \
3419	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3420	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3421	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3422	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3423	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3424	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
3425	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
3426	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
3427	"ld 11, 0(11)\n\t" /* target->r11 */ \
3428	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3429	"mr 11,%1\n\t" \
3430	"mr %0,3\n\t" \
3431	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3432	VG_CONTRACT_FRAME_BY(128) \
3433	VG_CONTRACT_FRAME_BY(512) \
3434	: /out/ "=r" (_res) \
3435	: /in/ "r" (&_argvec[2]) \
3436	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3437	); \
3438	lval = (__typeof__(lval)) _res; \
3439	} while (0)
3440
3441	#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3442	arg7,arg8,arg9,arg10,arg11) \
3443	do { \
3444	volatile OrigFn _orig = (orig); \
3445	volatile unsigned long _argvec[3+11]; \
3446	volatile unsigned long _res; \
3447	/* _argvec[0] holds current r2 across the call */ \
3448	_argvec[1] = (unsigned long)_orig.r2; \
3449	_argvec[2] = (unsigned long)_orig.nraddr; \
3450	_argvec[2+1] = (unsigned long)arg1; \
3451	_argvec[2+2] = (unsigned long)arg2; \
3452	_argvec[2+3] = (unsigned long)arg3; \
3453	_argvec[2+4] = (unsigned long)arg4; \
3454	_argvec[2+5] = (unsigned long)arg5; \
3455	_argvec[2+6] = (unsigned long)arg6; \
3456	_argvec[2+7] = (unsigned long)arg7; \
3457	_argvec[2+8] = (unsigned long)arg8; \
3458	_argvec[2+9] = (unsigned long)arg9; \
3459	_argvec[2+10] = (unsigned long)arg10; \
3460	_argvec[2+11] = (unsigned long)arg11; \
3461	__asm__ volatile( \
3462	"mr 11,%1\n\t" \
3463	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3464	"std 2,-16(11)\n\t" /* save tocptr */ \
3465	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3466	VG_EXPAND_FRAME_BY_trashes_r3(144) \
3467	/* arg11 */ \
3468	"ld 3,88(11)\n\t" \
3469	"std 3,128(1)\n\t" \
3470	/* arg10 */ \
3471	"ld 3,80(11)\n\t" \
3472	"std 3,120(1)\n\t" \
3473	/* arg9 */ \
3474	"ld 3,72(11)\n\t" \
3475	"std 3,112(1)\n\t" \
3476	/* args1-8 */ \
3477	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3478	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3479	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3480	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3481	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3482	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
3483	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
3484	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
3485	"ld 11, 0(11)\n\t" /* target->r11 */ \
3486	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3487	"mr 11,%1\n\t" \
3488	"mr %0,3\n\t" \
3489	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3490	VG_CONTRACT_FRAME_BY(144) \
3491	VG_CONTRACT_FRAME_BY(512) \
3492	: /out/ "=r" (_res) \
3493	: /in/ "r" (&_argvec[2]) \
3494	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3495	); \
3496	lval = (__typeof__(lval)) _res; \
3497	} while (0)
3498
3499	#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3500	arg7,arg8,arg9,arg10,arg11,arg12) \
3501	do { \
3502	volatile OrigFn _orig = (orig); \
3503	volatile unsigned long _argvec[3+12]; \
3504	volatile unsigned long _res; \
3505	/* _argvec[0] holds current r2 across the call */ \
3506	_argvec[1] = (unsigned long)_orig.r2; \
3507	_argvec[2] = (unsigned long)_orig.nraddr; \
3508	_argvec[2+1] = (unsigned long)arg1; \
3509	_argvec[2+2] = (unsigned long)arg2; \
3510	_argvec[2+3] = (unsigned long)arg3; \
3511	_argvec[2+4] = (unsigned long)arg4; \
3512	_argvec[2+5] = (unsigned long)arg5; \
3513	_argvec[2+6] = (unsigned long)arg6; \
3514	_argvec[2+7] = (unsigned long)arg7; \
3515	_argvec[2+8] = (unsigned long)arg8; \
3516	_argvec[2+9] = (unsigned long)arg9; \
3517	_argvec[2+10] = (unsigned long)arg10; \
3518	_argvec[2+11] = (unsigned long)arg11; \
3519	_argvec[2+12] = (unsigned long)arg12; \
3520	__asm__ volatile( \
3521	"mr 11,%1\n\t" \
3522	VG_EXPAND_FRAME_BY_trashes_r3(512) \
3523	"std 2,-16(11)\n\t" /* save tocptr */ \
3524	"ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3525	VG_EXPAND_FRAME_BY_trashes_r3(144) \
3526	/* arg12 */ \
3527	"ld 3,96(11)\n\t" \
3528	"std 3,136(1)\n\t" \
3529	/* arg11 */ \
3530	"ld 3,88(11)\n\t" \
3531	"std 3,128(1)\n\t" \
3532	/* arg10 */ \
3533	"ld 3,80(11)\n\t" \
3534	"std 3,120(1)\n\t" \
3535	/* arg9 */ \
3536	"ld 3,72(11)\n\t" \
3537	"std 3,112(1)\n\t" \
3538	/* args1-8 */ \
3539	"ld 3, 8(11)\n\t" /* arg1->r3 */ \
3540	"ld 4, 16(11)\n\t" /* arg2->r4 */ \
3541	"ld 5, 24(11)\n\t" /* arg3->r5 */ \
3542	"ld 6, 32(11)\n\t" /* arg4->r6 */ \
3543	"ld 7, 40(11)\n\t" /* arg5->r7 */ \
3544	"ld 8, 48(11)\n\t" /* arg6->r8 */ \
3545	"ld 9, 56(11)\n\t" /* arg7->r9 */ \
3546	"ld 10, 64(11)\n\t" /* arg8->r10 */ \
3547	"ld 11, 0(11)\n\t" /* target->r11 */ \
3548	VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3549	"mr 11,%1\n\t" \
3550	"mr %0,3\n\t" \
3551	"ld 2,-16(11)\n\t" /* restore tocptr */ \
3552	VG_CONTRACT_FRAME_BY(144) \
3553	VG_CONTRACT_FRAME_BY(512) \
3554	: /out/ "=r" (_res) \
3555	: /in/ "r" (&_argvec[2]) \
3556	: /trash/ "cc", "memory", __CALLER_SAVED_REGS \
3557	); \
3558	lval = (__typeof__(lval)) _res; \
3559	} while (0)
3560
3561	#endif /* PLAT_ppc64_aix5 */
3562
3563
3564	/* ------------------------------------------------------------------ */
3565	/* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS. */
3566	/* */
3567	/* ------------------------------------------------------------------ */
3568
3569	/* Some request codes. There are many more of these, but most are not
3570	exposed to end-user view. These are the public ones, all of the
3571	form 0x1000 + small_number.
3572
3573	Core ones are in the range 0x00000000--0x0000ffff. The non-public
3574	ones start at 0x2000.
3575	*/
3576
3577	/* These macros are used by tools -- they must be public, but don't
3578	embed them into other programs. */
3579	#define VG_USERREQ_TOOL_BASE(a,b) \
3580	((unsigned int)(((a)&0xff) << 24 \| ((b)&0xff) << 16))
3581	#define VG_IS_TOOL_USERREQ(a, b, v) \
3582	(VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
3583
3584	/* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
3585	This enum comprises an ABI exported by Valgrind to programs
3586	which use client requests. DO NOT CHANGE THE ORDER OF THESE
3587	ENTRIES, NOR DELETE ANY -- add new ones at the end. */
3588	typedef
3589	enum { VG_USERREQ__RUNNING_ON_VALGRIND = 0x1001,
3590	VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
3591
3592	/* These allow any function to be called from the simulated
3593	CPU but run on the real CPU. Nb: the first arg passed to
3594	the function is always the ThreadId of the running
3595	thread! So CLIENT_CALL0 actually requires a 1 arg
3596	function, etc. */
3597	VG_USERREQ__CLIENT_CALL0 = 0x1101,
3598	VG_USERREQ__CLIENT_CALL1 = 0x1102,
3599	VG_USERREQ__CLIENT_CALL2 = 0x1103,
3600	VG_USERREQ__CLIENT_CALL3 = 0x1104,
3601
3602	/* Can be useful in regression testing suites -- eg. can
3603	send Valgrind's output to /dev/null and still count
3604	errors. */
3605	VG_USERREQ__COUNT_ERRORS = 0x1201,
3606
3607	/* These are useful and can be interpreted by any tool that
3608	tracks malloc() et al, by using vg_replace_malloc.c. */
3609	VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
3610	VG_USERREQ__FREELIKE_BLOCK = 0x1302,
3611	/* Memory pool support. */
3612	VG_USERREQ__CREATE_MEMPOOL = 0x1303,
3613	VG_USERREQ__DESTROY_MEMPOOL = 0x1304,
3614	VG_USERREQ__MEMPOOL_ALLOC = 0x1305,
3615	VG_USERREQ__MEMPOOL_FREE = 0x1306,
3616	VG_USERREQ__MEMPOOL_TRIM = 0x1307,
3617	VG_USERREQ__MOVE_MEMPOOL = 0x1308,
3618	VG_USERREQ__MEMPOOL_CHANGE = 0x1309,
3619	VG_USERREQ__MEMPOOL_EXISTS = 0x130a,
3620
3621	/* Allow printfs to valgrind log. */
3622	VG_USERREQ__PRINTF = 0x1401,
3623	VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
3624
3625	/* Stack support. */
3626	VG_USERREQ__STACK_REGISTER = 0x1501,
3627	VG_USERREQ__STACK_DEREGISTER = 0x1502,
3628	VG_USERREQ__STACK_CHANGE = 0x1503
3629	} Vg_ClientRequest;
3630
3631	#if !defined(__GNUC__)
3632	# define __extension__ /* */
3633	#endif
3634
3635	/* Returns the number of Valgrinds this code is running under. That
3636	is, 0 if running natively, 1 if running under Valgrind, 2 if
3637	running under Valgrind which is running under another Valgrind,
3638	etc. */
3639	#define RUNNING_ON_VALGRIND __extension__ \
3640	({unsigned int _qzz_res; \
3641	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* if not */, \
3642	VG_USERREQ__RUNNING_ON_VALGRIND, \
3643	0, 0, 0, 0, 0); \
3644	_qzz_res; \
3645	})
3646
3647
3648	/* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
3649	_qzz_len - 1]. Useful if you are debugging a JITter or some such,
3650	since it provides a way to make sure valgrind will retranslate the
3651	invalidated area. Returns no value. */
3652	#define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len) \
3653	{unsigned int _qzz_res; \
3654	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3655	VG_USERREQ__DISCARD_TRANSLATIONS, \
3656	_qzz_addr, _qzz_len, 0, 0, 0); \
3657	}
3658
3659
3660	/* These requests are for getting Valgrind itself to print something.
3661	Possibly with a backtrace. This is a really ugly hack. */
3662
3663	#if defined(NVALGRIND)
3664
3665	# define VALGRIND_PRINTF(...)
3666	# define VALGRIND_PRINTF_BACKTRACE(...)
3667
3668	#else /* NVALGRIND */
3669
3670	/* Modern GCC will optimize the static routine out if unused,
3671	and unused attribute will shut down warnings about it. */
3672	static int VALGRIND_PRINTF(const char *format, ...)
3673	__attribute__((format(__printf__, 1, 2), __unused__));
3674	static int
3675	VALGRIND_PRINTF(const char *format, ...)
3676	{
3677	unsigned long _qzz_res;
3678	va_list vargs;
3679	va_start(vargs, format);
3680	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF,
3681	(unsigned long)format, (unsigned long)vargs,
3682	0, 0, 0);
3683	va_end(vargs);
3684	return (int)_qzz_res;
3685	}
3686
3687	static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
3688	__attribute__((format(__printf__, 1, 2), __unused__));
3689	static int
3690	VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
3691	{
3692	unsigned long _qzz_res;
3693	va_list vargs;
3694	va_start(vargs, format);
3695	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF_BACKTRACE,
3696	(unsigned long)format, (unsigned long)vargs,
3697	0, 0, 0);
3698	va_end(vargs);
3699	return (int)_qzz_res;
3700	}
3701
3702	#endif /* NVALGRIND */
3703
3704
3705	/* These requests allow control to move from the simulated CPU to the
3706	real CPU, calling an arbitary function.
3707
3708	Note that the current ThreadId is inserted as the first argument.
3709	So this call:
3710
3711	VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
3712
3713	requires f to have this signature:
3714
3715	Word f(Word tid, Word arg1, Word arg2)
3716
3717	where "Word" is a word-sized type.
3718
3719	Note that these client requests are not entirely reliable. For example,
3720	if you call a function with them that subsequently calls printf(),
3721	there's a high chance Valgrind will crash. Generally, your prospects of
3722	these working are made higher if the called function does not refer to
3723	any global variables, and does not refer to any libc or other functions
3724	(printf et al). Any kind of entanglement with libc or dynamic linking is
3725	likely to have a bad outcome, for tricky reasons which we've grappled
3726	with a lot in the past.
3727	*/
3728	#define VALGRIND_NON_SIMD_CALL0(_qyy_fn) \
3729	__extension__ \
3730	({unsigned long _qyy_res; \
3731	VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \
3732	VG_USERREQ__CLIENT_CALL0, \
3733	_qyy_fn, \
3734	0, 0, 0, 0); \
3735	_qyy_res; \
3736	})
3737
3738	#define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1) \
3739	__extension__ \
3740	({unsigned long _qyy_res; \
3741	VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \
3742	VG_USERREQ__CLIENT_CALL1, \
3743	_qyy_fn, \
3744	_qyy_arg1, 0, 0, 0); \
3745	_qyy_res; \
3746	})
3747
3748	#define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2) \
3749	__extension__ \
3750	({unsigned long _qyy_res; \
3751	VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \
3752	VG_USERREQ__CLIENT_CALL2, \
3753	_qyy_fn, \
3754	_qyy_arg1, _qyy_arg2, 0, 0); \
3755	_qyy_res; \
3756	})
3757
3758	#define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
3759	__extension__ \
3760	({unsigned long _qyy_res; \
3761	VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \
3762	VG_USERREQ__CLIENT_CALL3, \
3763	_qyy_fn, \
3764	_qyy_arg1, _qyy_arg2, \
3765	_qyy_arg3, 0); \
3766	_qyy_res; \
3767	})
3768
3769
3770	/* Counts the number of errors that have been recorded by a tool. Nb:
3771	the tool must record the errors with VG_(maybe_record_error)() or
3772	VG_(unique_error)() for them to be counted. */
3773	#define VALGRIND_COUNT_ERRORS \
3774	__extension__ \
3775	({unsigned int _qyy_res; \
3776	VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \
3777	VG_USERREQ__COUNT_ERRORS, \
3778	0, 0, 0, 0, 0); \
3779	_qyy_res; \
3780	})
3781
3782	/* Mark a block of memory as having been allocated by a malloc()-like
3783	function. `addr' is the start of the usable block (ie. after any
3784	redzone) `rzB' is redzone size if the allocator can apply redzones;
3785	use '0' if not. Adding redzones makes it more likely Valgrind will spot
3786	block overruns. `is_zeroed' indicates if the memory is zeroed, as it is
3787	for calloc(). Put it immediately after the point where a block is
3788	allocated.
3789
3790	If you're using Memcheck: If you're allocating memory via superblocks,
3791	and then handing out small chunks of each superblock, if you don't have
3792	redzones on your small blocks, it's worth marking the superblock with
3793	VALGRIND_MAKE_MEM_NOACCESS when it's created, so that block overruns are
3794	detected. But if you can put redzones on, it's probably better to not do
3795	this, so that messages for small overruns are described in terms of the
3796	small block rather than the superblock (but if you have a big overrun
3797	that skips over a redzone, you could miss an error this way). See
3798	memcheck/tests/custom_alloc.c for an example.
3799
3800	WARNING: if your allocator uses malloc() or 'new' to allocate
3801	superblocks, rather than mmap() or brk(), this will not work properly --
3802	you'll likely get assertion failures during leak detection. This is
3803	because Valgrind doesn't like seeing overlapping heap blocks. Sorry.
3804
3805	Nb: block must be freed via a free()-like function specified
3806	with VALGRIND_FREELIKE_BLOCK or mismatch errors will occur. */
3807	#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \
3808	{unsigned int _qzz_res; \
3809	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3810	VG_USERREQ__MALLOCLIKE_BLOCK, \
3811	addr, sizeB, rzB, is_zeroed, 0); \
3812	}
3813
3814	/* Mark a block of memory as having been freed by a free()-like function.
3815	`rzB' is redzone size; it must match that given to
3816	VALGRIND_MALLOCLIKE_BLOCK. Memory not freed will be detected by the leak
3817	checker. Put it immediately after the point where the block is freed. */
3818	#define VALGRIND_FREELIKE_BLOCK(addr, rzB) \
3819	{unsigned int _qzz_res; \
3820	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3821	VG_USERREQ__FREELIKE_BLOCK, \
3822	addr, rzB, 0, 0, 0); \
3823	}
3824
3825	/* Create a memory pool. */
3826	#define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed) \
3827	{unsigned int _qzz_res; \
3828	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3829	VG_USERREQ__CREATE_MEMPOOL, \
3830	pool, rzB, is_zeroed, 0, 0); \
3831	}
3832
3833	/* Destroy a memory pool. */
3834	#define VALGRIND_DESTROY_MEMPOOL(pool) \
3835	{unsigned int _qzz_res; \
3836	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3837	VG_USERREQ__DESTROY_MEMPOOL, \
3838	pool, 0, 0, 0, 0); \
3839	}
3840
3841	/* Associate a piece of memory with a memory pool. */
3842	#define VALGRIND_MEMPOOL_ALLOC(pool, addr, size) \
3843	{unsigned int _qzz_res; \
3844	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3845	VG_USERREQ__MEMPOOL_ALLOC, \
3846	pool, addr, size, 0, 0); \
3847	}
3848
3849	/* Disassociate a piece of memory from a memory pool. */
3850	#define VALGRIND_MEMPOOL_FREE(pool, addr) \
3851	{unsigned int _qzz_res; \
3852	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3853	VG_USERREQ__MEMPOOL_FREE, \
3854	pool, addr, 0, 0, 0); \
3855	}
3856
3857	/* Disassociate any pieces outside a particular range. */
3858	#define VALGRIND_MEMPOOL_TRIM(pool, addr, size) \
3859	{unsigned int _qzz_res; \
3860	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3861	VG_USERREQ__MEMPOOL_TRIM, \
3862	pool, addr, size, 0, 0); \
3863	}
3864
3865	/* Resize and/or move a piece associated with a memory pool. */
3866	#define VALGRIND_MOVE_MEMPOOL(poolA, poolB) \
3867	{unsigned int _qzz_res; \
3868	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3869	VG_USERREQ__MOVE_MEMPOOL, \
3870	poolA, poolB, 0, 0, 0); \
3871	}
3872
3873	/* Resize and/or move a piece associated with a memory pool. */
3874	#define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size) \
3875	{unsigned int _qzz_res; \
3876	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3877	VG_USERREQ__MEMPOOL_CHANGE, \
3878	pool, addrA, addrB, size, 0); \
3879	}
3880
3881	/* Return 1 if a mempool exists, else 0. */
3882	#define VALGRIND_MEMPOOL_EXISTS(pool) \
3883	({unsigned int _qzz_res; \
3884	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3885	VG_USERREQ__MEMPOOL_EXISTS, \
3886	pool, 0, 0, 0, 0); \
3887	_qzz_res; \
3888	})
3889
3890	/* Mark a piece of memory as being a stack. Returns a stack id. */
3891	#define VALGRIND_STACK_REGISTER(start, end) \
3892	({unsigned int _qzz_res; \
3893	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3894	VG_USERREQ__STACK_REGISTER, \
3895	start, end, 0, 0, 0); \
3896	_qzz_res; \
3897	})
3898
3899	/* Unmark the piece of memory associated with a stack id as being a
3900	stack. */
3901	#define VALGRIND_STACK_DEREGISTER(id) \
3902	{unsigned int _qzz_res; \
3903	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3904	VG_USERREQ__STACK_DEREGISTER, \
3905	id, 0, 0, 0, 0); \
3906	}
3907
3908	/* Change the start and end address of the stack id. */
3909	#define VALGRIND_STACK_CHANGE(id, start, end) \
3910	{unsigned int _qzz_res; \
3911	VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \
3912	VG_USERREQ__STACK_CHANGE, \
3913	id, start, end, 0, 0); \
3914	}
3915
3916
3917	#undef PLAT_x86_linux
3918	#undef PLAT_amd64_linux
3919	#undef PLAT_ppc32_linux
3920	#undef PLAT_ppc64_linux
3921	#undef PLAT_ppc32_aix5
3922	#undef PLAT_ppc64_aix5
3923
3924	#endif /* __VALGRIND_H */

Note: See TracBrowser for help on using the repository browser.

Download in other formats: