summaryrefslogtreecommitdiff
path: root/arch/arm64/kernel/vdso/gettimeofday.S
blob: 8bf658d974f947da714eb5e7a6812552173fe140 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
/*
 * Userspace implementations of gettimeofday() and friends.
 *
 * Copyright (C) 2012 ARM Limited
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Author: Will Deacon <will.deacon@arm.com>
 */

#include <linux/linkage.h>
#include <asm/asm-offsets.h>
#include <asm/unistd.h>

#define NSEC_PER_SEC_LO16	0xca00
#define NSEC_PER_SEC_HI16	0x3b9a

vdso_data	.req	x6
use_syscall	.req	w7
seqcnt		.req	w8

	.macro	seqcnt_acquire
9999:	ldr	seqcnt, [vdso_data, #VDSO_TB_SEQ_COUNT]
	tbnz	seqcnt, #0, 9999b
	dmb	ishld
	ldr	use_syscall, [vdso_data, #VDSO_USE_SYSCALL]
	.endm

	.macro	seqcnt_read, cnt
	dmb	ishld
	ldr	\cnt, [vdso_data, #VDSO_TB_SEQ_COUNT]
	.endm

	.macro	seqcnt_check, cnt, fail
	cmp	\cnt, seqcnt
	b.ne	\fail
	.endm

	.text

/* int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz); */
ENTRY(__kernel_gettimeofday)
	.cfi_startproc
	mov	x2, x30
	.cfi_register x30, x2

	/* Acquire the sequence counter and get the timespec. */
	adr	vdso_data, _vdso_data
1:	seqcnt_acquire
	cbnz	use_syscall, 4f

	/* If tv is NULL, skip to the timezone code. */
	cbz	x0, 2f
	bl	__do_get_tspec
	seqcnt_check w9, 1b

	/* Convert ns to us. */
	mov	x13, #1000
	lsl	x13, x13, x12
	udiv	x11, x11, x13
	stp	x10, x11, [x0, #TVAL_TV_SEC]
2:
	/* If tz is NULL, return 0. */
	cbz	x1, 3f
	ldp	w4, w5, [vdso_data, #VDSO_TZ_MINWEST]
	seqcnt_read w9
	seqcnt_check w9, 1b
	stp	w4, w5, [x1, #TZ_MINWEST]
3:
	mov	x0, xzr
	ret	x2
4:
	/* Syscall fallback. */
	mov	x8, #__NR_gettimeofday
	svc	#0
	ret	x2
	.cfi_endproc
ENDPROC(__kernel_gettimeofday)

/* int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp); */
ENTRY(__kernel_clock_gettime)
	.cfi_startproc
	cmp	w0, #CLOCK_REALTIME
	ccmp	w0, #CLOCK_MONOTONIC, #0x4, ne
	b.ne	2f

	mov	x2, x30
	.cfi_register x30, x2

	/* Get kernel timespec. */
	adr	vdso_data, _vdso_data
1:	seqcnt_acquire
	cbnz	use_syscall, 7f

	bl	__do_get_tspec
	seqcnt_check w9, 1b

	cmp	w0, #CLOCK_MONOTONIC
	b.ne	6f

	/* Get wtm timespec. */
	ldp	x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC]

	/* Check the sequence counter. */
	seqcnt_read w9
	seqcnt_check w9, 1b
	b	4f
2:
	cmp	w0, #CLOCK_REALTIME_COARSE
	ccmp	w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne
	b.ne	8f

	/* Get coarse timespec. */
	adr	vdso_data, _vdso_data
3:	seqcnt_acquire
	ldp	x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC]

	/* Get wtm timespec. */
	ldp	x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC]

	/* Check the sequence counter. */
	seqcnt_read w9
	seqcnt_check w9, 3b

	cmp	w0, #CLOCK_MONOTONIC_COARSE
	b.ne	6f
4:
	/* Add on wtm timespec. */
	add	x10, x10, x13
	lsl	x14, x14, x12
	add	x11, x11, x14

	/* Normalise the new timespec. */
	mov	x15, #NSEC_PER_SEC_LO16
	movk	x15, #NSEC_PER_SEC_HI16, lsl #16
	lsl	x15, x15, x12
	cmp	x11, x15
	b.lt	5f
	sub	x11, x11, x15
	add	x10, x10, #1
5:
	cmp	x11, #0
	b.ge	6f
	add	x11, x11, x15
	sub	x10, x10, #1

6:	/* Store to the user timespec. */
	lsr	x11, x11, x12
	stp	x10, x11, [x1, #TSPEC_TV_SEC]
	mov	x0, xzr
	ret	x2
7:
	mov	x30, x2
8:	/* Syscall fallback. */
	mov	x8, #__NR_clock_gettime
	svc	#0
	ret
	.cfi_endproc
ENDPROC(__kernel_clock_gettime)

/* int __kernel_clock_getres(clockid_t clock_id, struct timespec *res); */
ENTRY(__kernel_clock_getres)
	.cfi_startproc
	cbz	w1, 3f

	cmp	w0, #CLOCK_REALTIME
	ccmp	w0, #CLOCK_MONOTONIC, #0x4, ne
	b.ne	1f

	ldr	x2, 5f
	b	2f
1:
	cmp	w0, #CLOCK_REALTIME_COARSE
	ccmp	w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne
	b.ne	4f
	ldr	x2, 6f
2:
	stp	xzr, x2, [x1]

3:	/* res == NULL. */
	mov	w0, wzr
	ret

4:	/* Syscall fallback. */
	mov	x8, #__NR_clock_getres
	svc	#0
	ret
5:
	.quad	CLOCK_REALTIME_RES
6:
	.quad	CLOCK_COARSE_RES
	.cfi_endproc
ENDPROC(__kernel_clock_getres)

/*
 * Read the current time from the architected counter.
 * Expects vdso_data to be initialised.
 * Clobbers the temporary registers (x9 - x15).
 * Returns:
 *  - w9		= vDSO sequence counter
 *  - (x10, x11)	= (ts->tv_sec, shifted ts->tv_nsec)
 *  - w12		= cs_shift
 */
ENTRY(__do_get_tspec)
	.cfi_startproc

	/* Read from the vDSO data page. */
	ldr	x10, [vdso_data, #VDSO_CS_CYCLE_LAST]
	ldp	x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC]
	ldp	w11, w12, [vdso_data, #VDSO_CS_MULT]
	seqcnt_read w9

	/* Read the virtual counter. */
	isb
	mrs	x15, cntvct_el0

	/* Calculate cycle delta and convert to ns. */
	sub	x10, x15, x10
	/* We can only guarantee 56 bits of precision. */
	movn	x15, #0xff00, lsl #48
	and	x10, x15, x10
	mul	x10, x10, x11

	/* Use the kernel time to calculate the new timespec. */
	mov	x11, #NSEC_PER_SEC_LO16
	movk	x11, #NSEC_PER_SEC_HI16, lsl #16
	lsl	x11, x11, x12
	add	x15, x10, x14
	udiv	x14, x15, x11
	add	x10, x13, x14
	mul	x13, x14, x11
	sub	x11, x15, x13

	ret
	.cfi_endproc
ENDPROC(__do_get_tspec)