|
|
1 /*
2 *
3 * Optimized version of the copy_user() routine.
4 * It is used to copy date across the kernel/user boundary.
5 *
6 * The source and destination are always on opposite side of
7 * the boundary. When reading from user space we must catch
8 * faults on loads. When writing to user space we must catch
9 * errors on stores. Note that because of the nature of the copy
10 * we don't need to worry about overlapping regions.
11 *
12 *
13 * Inputs:
14 * in0 address of source buffer
15 * in1 address of destination buffer
16 * in2 number of bytes to copy
17 *
18 * Outputs:
19 * ret0 0 in case of success. The number of bytes NOT copied in
20 * case of error.
21 *
22 * Copyright (C) 2000-2001 Hewlett-Packard Co
23 * Stephane Eranian <eranian@hpl.hp.com>
24 *
25 * Fixme:
26 * - handle the case where we have more than 16 bytes and the alignment
27 * are different.
28 * - more benchmarking
29 * - fix extraneous stop bit introduced by the EX() macro.
30 */
31
32 #include <asm/asmmacro.h>
33
34 //
35 // Tuneable parameters
36 //
37 #define COPY_BREAK 16 // we do byte copy below (must be >=16)
38 #define PIPE_DEPTH 21 // pipe depth
39
40 #define EPI p[PIPE_DEPTH-1]
41
42 //
43 // arguments
44 //
45 #define dst in0
46 #define src in1
47 #define len in2
48
49 //
50 // local registers
51 //
52 #define t1 r2 // rshift in bytes
53 #define t2 r3 // lshift in bytes
54 #define rshift r14 // right shift in bits
55 #define lshift r15 // left shift in bits
56 #define word1 r16
57 #define word2 r17
58 #define cnt r18
59 #define len2 r19
60 #define saved_lc r20
61 #define saved_pr r21
62 #define tmp r22
63 #define val r23
64 #define src1 r24
65 #define dst1 r25
66 #define src2 r26
67 #define dst2 r27
68 #define len1 r28
69 #define enddst r29
70 #define endsrc r30
71 #define saved_pfs r31
72
73 GLOBAL_ENTRY(__copy_user)
74 .prologue
75 .save ar.pfs, saved_pfs
76 alloc saved_pfs=ar.pfs,3,((2*PIPE_DEPTH+7)&~7),0,((2*PIPE_DEPTH+7)&~7)
77
78 .rotr val1[PIPE_DEPTH],val2[PIPE_DEPTH]
79 .rotp p[PIPE_DEPTH]
80
81 adds len2=-1,len // br.ctop is repeat/until
82 mov ret0=r0
83
84 ;; // RAW of cfm when len=0
85 cmp.eq p8,p0=r0,len // check for zero length
86 .save ar.lc, saved_lc
87 mov saved_lc=ar.lc // preserve ar.lc (slow)
88 (p8) br.ret.spnt.many rp // empty mempcy()
89 ;;
90 add enddst=dst,len // first byte after end of source
91 add endsrc=src,len // first byte after end of destination
92 .save pr, saved_pr
93 mov saved_pr=pr // preserve predicates
94
95 .body
96
97 mov dst1=dst // copy because of rotation
98 mov ar.ec=PIPE_DEPTH
99 mov pr.rot=1<<16 // p16=true all others are false
100
101 mov src1=src // copy because of rotation
102 mov ar.lc=len2 // initialize lc for small count
103 cmp.lt p10,p7=COPY_BREAK,len // if len > COPY_BREAK then long copy
104
105 xor tmp=src,dst // same alignment test prepare
106 (p10) br.cond.dptk .long_copy_user
107 ;; // RAW pr.rot/p16 ?
108 //
109 // Now we do the byte by byte loop with software pipeline
110 //
111 // p7 is necessarily false by now
112 1:
113 EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1)
114 EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1)
115 br.ctop.dptk.few 1b
116 ;;
117 mov ar.lc=saved_lc
118 mov pr=saved_pr,0xffffffffffff0000
119 mov ar.pfs=saved_pfs // restore ar.ec
120 br.ret.sptk.many rp // end of short memcpy
121
122 //
123 // Not 8-byte aligned
124 //
125 .diff_align_copy_user:
126 // At this point we know we have more than 16 bytes to copy
127 // and also that src and dest do _not_ have the same alignment.
128 and src2=0x7,src1 // src offset
129 and dst2=0x7,dst1 // dst offset
130 ;;
131 // The basic idea is that we copy byte-by-byte at the head so
132 // that we can reach 8-byte alignment for both src1 and dst1.
133 // Then copy the body using software pipelined 8-byte copy,
134 // shifting the two back-to-back words right and left, then copy
135 // the tail by copying byte-by-byte.
136 //
137 // Fault handling. If the byte-by-byte at the head fails on the
138 // load, then restart and finish the pipleline by copying zeros
139 // to the dst1. Then copy zeros for the rest of dst1.
140 // If 8-byte software pipeline fails on the load, do the same as
141 // failure_in3 does. If the byte-by-byte at the tail fails, it is
142 // handled simply by failure_in_pipe1.
143 //
144 // The case p14 represents the source has more bytes in the
145 // the first word (by the shifted part), whereas the p15 needs to
146 // copy some bytes from the 2nd word of the source that has the
147 // tail of the 1st of the destination.
148 //
149
150 //
151 // Optimization. If dst1 is 8-byte aligned (quite common), we don't need
152 // to copy the head to dst1, to start 8-byte copy software pipeline.
153 // We know src1 is not 8-byte aligned in this case.
154 //
155 cmp.eq p14,p15=r0,dst2
156 (p15) br.cond.spnt 1f
157 ;;
158 sub t1=8,src2
159 mov t2=src2
160 ;;
161 shl rshift=t2,3
162 sub len1=len,t1 // set len1
163 ;;
164 sub lshift=64,rshift
165 ;;
166 br.cond.spnt .word_copy_user
167 ;;
168 1:
169 cmp.leu p14,p15=src2,dst2
170 sub t1=dst2,src2
171 ;;
172 .pred.rel "mutex", p14, p15
173 (p14) sub word1=8,src2 // (8 - src offset)
174 (p15) sub t1=r0,t1 // absolute value
175 (p15) sub word1=8,dst2 // (8 - dst offset)
176 ;;
177 // For the case p14, we don't need to copy the shifted part to
178 // the 1st word of destination.
179 sub t2=8,t1
180 (p14) sub word1=word1,t1
181 ;;
182 sub len1=len,word1 // resulting len
183 (p15) shl rshift=t1,3 // in bits
184 (p14) shl rshift=t2,3
185 ;;
186 (p14) sub len1=len1,t1
187 adds cnt=-1,word1
188 ;;
189 sub lshift=64,rshift
190 mov ar.ec=PIPE_DEPTH
191 mov pr.rot=1<<16 // p16=true all others are false
192 mov ar.lc=cnt
193 ;;
194 2:
195 EX(.failure_in_pipe2,(p16) ld1 val1[0]=[src1],1)
196 EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1)
197 br.ctop.dptk.few 2b
198 ;;
199 clrrrb
200 ;;
201 .word_copy_user:
202 cmp.gtu p9,p0=16,len1
203 (p9) br.cond.spnt 4f // if (16 > len1) skip 8-byte copy
204 ;;
205 shr.u cnt=len1,3 // number of 64-bit words
206 ;;
207 adds cnt=-1,cnt
208 ;;
209 .pred.rel "mutex", p14, p15
210 (p14) sub src1=src1,t2
211 (p15) sub src1=src1,t1
212 //
213 // Now both src1 and dst1 point to an 8-byte aligned address. And
214 // we have more than 8 bytes to copy.
215 //
216 mov ar.lc=cnt
217 mov ar.ec=PIPE_DEPTH
218 mov pr.rot=1<<16 // p16=true all others are false
219 ;;
220 3:
221 //
222 // The pipleline consists of 3 stages:
223 // 1 (p16): Load a word from src1
224 // 2 (EPI_1): Shift right pair, saving to tmp
225 // 3 (EPI): Store tmp to dst1
226 //
227 // To make it simple, use at least 2 (p16) loops to set up val1[n]
228 // because we need 2 back-to-back val1[] to get tmp.
229 // Note that this implies EPI_2 must be p18 or greater.
230 //
231
232 #define EPI_1 p[PIPE_DEPTH-2]
233 #define SWITCH(pred, shift) cmp.eq pred,p0=shift,rshift
234 #define CASE(pred, shift) \
235 (pred) br.cond.spnt .copy_user_bit##shift
236 #define BODY(rshift) \
237 .copy_user_bit##rshift: \
238 1: \
239 EX(.failure_out,(EPI) st8 [dst1]=tmp,8); \
240 (EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \
241 EX(3f,(p16) ld8 val1[1]=[src1],8); \
242 (p16) mov val1[0]=r0; \
243 br.ctop.dptk 1b; \
244 ;; \
245 br.cond.sptk.many .diff_align_do_tail; \
246 2: \
247 (EPI) st8 [dst1]=tmp,8; \
248 (EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \
249 3: \
250 (p16) mov val1[1]=r0; \
251 (p16) mov val1[0]=r0; \
252 br.ctop.dptk 2b; \
253 ;; \
254 br.cond.sptk.many .failure_in2
255
256 //
257 // Since the instruction 'shrp' requires a fixed 128-bit value
258 // specifying the bits to shift, we need to provide 7 cases
259 // below.
260 //
261 SWITCH(p6, 8)
262 SWITCH(p7, 16)
263 SWITCH(p8, 24)
264 SWITCH(p9, 32)
265 SWITCH(p10, 40)
266 SWITCH(p11, 48)
267 SWITCH(p12, 56)
268 ;;
269 CASE(p6, 8)
270 CASE(p7, 16)
271 CASE(p8, 24)
272 CASE(p9, 32)
273 CASE(p10, 40)
274 CASE(p11, 48)
275 CASE(p12, 56)
276 ;;
277 BODY(8)
278 BODY(16)
279 BODY(24)
280 BODY(32)
281 BODY(40)
282 BODY(48)
283 BODY(56)
284 ;;
285 .diff_align_do_tail:
286 .pred.rel "mutex", p14, p15
287 (p14) sub src1=src1,t1
288 (p14) adds dst1=-8,dst1
289 (p15) sub dst1=dst1,t1
290 ;;
291 4:
292 // Tail correction.
293 //
294 // The problem with this piplelined loop is that the last word is not
295 // loaded and thus parf of the last word written is not correct.
296 // To fix that, we simply copy the tail byte by byte.
297
298 sub len1=endsrc,src1,1
299 clrrrb
300 ;;
301 mov ar.ec=PIPE_DEPTH
302 mov pr.rot=1<<16 // p16=true all others are false
303 mov ar.lc=len1
304 ;;
305 5:
306 EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1)
307 EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1)
308 br.ctop.dptk.few 5b
309 ;;
310 mov ar.lc=saved_lc
311 mov pr=saved_pr,0xffffffffffff0000
312 mov ar.pfs=saved_pfs
313 br.ret.sptk.many rp
314
315 //
316 // Beginning of long mempcy (i.e. > 16 bytes)
317 //
318 .long_copy_user:
319 tbit.nz p6,p7=src1,0 // odd alignement
320 and tmp=7,tmp
321 ;;
322 cmp.eq p10,p8=r0,tmp
323 mov len1=len // copy because of rotation
324 (p8) br.cond.dpnt .diff_align_copy_user
325 ;;
326 // At this point we know we have more than 16 bytes to copy
327 // and also that both src and dest have the same alignment
328 // which may not be the one we want. So for now we must move
329 // forward slowly until we reach 16byte alignment: no need to
330 // worry about reaching the end of buffer.
331 //
332 EX(.failure_in1,(p6) ld1 val1[0]=[src1],1) // 1-byte aligned
333 (p6) adds len1=-1,len1;;
334 tbit.nz p7,p0=src1,1
335 ;;
336 EX(.failure_in1,(p7) ld2 val1[1]=[src1],2) // 2-byte aligned
337 (p7) adds len1=-2,len1;;
338 tbit.nz p8,p0=src1,2
339 ;;
340 //
341 // Stop bit not required after ld4 because if we fail on ld4
342 // we have never executed the ld1, therefore st1 is not executed.
343 //
344 EX(.failure_in1,(p8) ld4 val2[0]=[src1],4) // 4-byte aligned
345 ;;
346 EX(.failure_out,(p6) st1 [dst1]=val1[0],1)
347 tbit.nz p9,p0=src1,3
348 ;;
349 //
350 // Stop bit not required after ld8 because if we fail on ld8
351 // we have never executed the ld2, therefore st2 is not executed.
352 //
353 EX(.failure_in1,(p9) ld8 val2[1]=[src1],8) // 8-byte aligned
354 EX(.failure_out,(p7) st2 [dst1]=val1[1],2)
355 (p8) adds len1=-4,len1
356 ;;
357 EX(.failure_out, (p8) st4 [dst1]=val2[0],4)
358 (p9) adds len1=-8,len1;;
359 shr.u cnt=len1,4 // number of 128-bit (2x64bit) words
360 ;;
361 EX(.failure_out, (p9) st8 [dst1]=val2[1],8)
362 tbit.nz p6,p0=len1,3
363 cmp.eq p7,p0=r0,cnt
364 adds tmp=-1,cnt // br.ctop is repeat/until
365 (p7) br.cond.dpnt .dotail // we have less than 16 bytes left
366 ;;
367 adds src2=8,src1
368 adds dst2=8,dst1
369 mov ar.lc=tmp
370 ;;
371 //
372 // 16bytes/iteration
373 //
374 2:
375 EX(.failure_in3,(p16) ld8 val1[0]=[src1],16)
376 (p16) ld8 val2[0]=[src2],16
377
378 EX(.failure_out, (EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16)
379 (EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16
380 br.ctop.dptk 2b
381 ;; // RAW on src1 when fall through from loop
382 //
383 // Tail correction based on len only
384 //
385 // No matter where we come from (loop or test) the src1 pointer
386 // is 16 byte aligned AND we have less than 16 bytes to copy.
387 //
388 .dotail:
389 EX(.failure_in1,(p6) ld8 val1[0]=[src1],8) // at least 8 bytes
390 tbit.nz p7,p0=len1,2
391 ;;
392 EX(.failure_in1,(p7) ld4 val1[1]=[src1],4) // at least 4 bytes
393 tbit.nz p8,p0=len1,1
394 ;;
395 EX(.failure_in1,(p8) ld2 val2[0]=[src1],2) // at least 2 bytes
396 tbit.nz p9,p0=len1,0
397 ;;
398 EX(.failure_out, (p6) st8 [dst1]=val1[0],8)
399 ;;
400 EX(.failure_in1,(p9) ld1 val2[1]=[src1]) // only 1 byte left
401 mov ar.lc=saved_lc
402 ;;
403 EX(.failure_out,(p7) st4 [dst1]=val1[1],4)
404 mov pr=saved_pr,0xffffffffffff0000
405 ;;
406 EX(.failure_out, (p8) st2 [dst1]=val2[0],2)
407 mov ar.pfs=saved_pfs
408 ;;
409 EX(.failure_out, (p9) st1 [dst1]=val2[1])
410 br.ret.sptk.many rp
411
412
413 //
414 // Here we handle the case where the byte by byte copy fails
415 // on the load.
416 // Several factors make the zeroing of the rest of the buffer kind of
417 // tricky:
418 // - the pipeline: loads/stores are not in sync (pipeline)
419 //
420 // In the same loop iteration, the dst1 pointer does not directly
421 // reflect where the faulty load was.
422 //
423 // - pipeline effect
424 // When you get a fault on load, you may have valid data from
425 // previous loads not yet store in transit. Such data must be
426 // store normally before moving onto zeroing the rest.
427 //
428 // - single/multi dispersal independence.
429 //
430 // solution:
431 // - we don't disrupt the pipeline, i.e. data in transit in
432 // the software pipeline will be eventually move to memory.
433 // We simply replace the load with a simple mov and keep the
434 // pipeline going. We can't really do this inline because
435 // p16 is always reset to 1 when lc > 0.
436 //
437 .failure_in_pipe1:
438 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
439 1:
440 (p16) mov val1[0]=r0
441 (EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1
442 br.ctop.dptk 1b
443 ;;
444 mov pr=saved_pr,0xffffffffffff0000
445 mov ar.lc=saved_lc
446 mov ar.pfs=saved_pfs
447 br.ret.sptk.many rp
448
449 //
450 // This is the case where the byte by byte copy fails on the load
451 // when we copy the head. We need to finish the pipeline and copy
452 // zeros for the rest of the destination. Since this happens
453 // at the top we still need to fill the body and tail.
454 .failure_in_pipe2:
455 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
456 2:
457 (p16) mov val1[0]=r0
458 (EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1
459 br.ctop.dptk 2b
460 ;;
461 sub len=enddst,dst1,1 // precompute len
462 br.cond.dptk.many .failure_in1bis
463 ;;
464
465 //
466 // Here we handle the head & tail part when we check for alignment.
467 // The following code handles only the load failures. The
468 // main diffculty comes from the fact that loads/stores are
469 // scheduled. So when you fail on a load, the stores corresponding
470 // to previous successful loads must be executed.
471 //
472 // However some simplifications are possible given the way
473 // things work.
474 //
475 // 1) HEAD
476 // Theory of operation:
477 //
478 // Page A | Page B
479 // ---------|-----
480 // 1|8 x
481 // 1 2|8 x
482 // 4|8 x
483 // 1 4|8 x
484 // 2 4|8 x
485 // 1 2 4|8 x
486 // |1
487 // |2 x
488 // |4 x
489 //
490 // page_size >= 4k (2^12). (x means 4, 2, 1)
491 // Here we suppose Page A exists and Page B does not.
492 //
493 // As we move towards eight byte alignment we may encounter faults.
494 // The numbers on each page show the size of the load (current alignment).
495 //
496 // Key point:
497 // - if you fail on 1, 2, 4 then you have never executed any smaller
498 // size loads, e.g. failing ld4 means no ld1 nor ld2 executed
499 // before.
500 //
501 // This allows us to simplify the cleanup code, because basically you
502 // only have to worry about "pending" stores in the case of a failing
503 // ld8(). Given the way the code is written today, this means only
504 // worry about st2, st4. There we can use the information encapsulated
505 // into the predicates.
506 //
507 // Other key point:
508 // - if you fail on the ld8 in the head, it means you went straight
509 // to it, i.e. 8byte alignment within an unexisting page.
510 // Again this comes from the fact that if you crossed just for the ld8 then
511 // you are 8byte aligned but also 16byte align, therefore you would
512 // either go for the 16byte copy loop OR the ld8 in the tail part.
513 // The combination ld1, ld2, ld4, ld8 where you fail on ld8 is impossible
514 // because it would mean you had 15bytes to copy in which case you
515 // would have defaulted to the byte by byte copy.
516 //
517 //
518 // 2) TAIL
519 // Here we now we have less than 16 bytes AND we are either 8 or 16 byte
520 // aligned.
521 //
522 // Key point:
523 // This means that we either:
524 // - are right on a page boundary
525 // OR
526 // - are at more than 16 bytes from a page boundary with
527 // at most 15 bytes to copy: no chance of crossing.
528 //
529 // This allows us to assume that if we fail on a load we haven't possibly
530 // executed any of the previous (tail) ones, so we don't need to do
531 // any stores. For instance, if we fail on ld2, this means we had
532 // 2 or 3 bytes left to copy and we did not execute the ld8 nor ld4.
533 //
534 // This means that we are in a situation similar the a fault in the
535 // head part. That's nice!
536 //
537 .failure_in1:
538 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
539 sub len=endsrc,src1,1
540 //
541 // we know that ret0 can never be zero at this point
542 // because we failed why trying to do a load, i.e. there is still
543 // some work to do.
544 // The failure_in1bis and length problem is taken care of at the
545 // calling side.
546 //
547 ;;
548 .failure_in1bis: // from (.failure_in3)
549 mov ar.lc=len // Continue with a stupid byte store.
550 ;;
551 5:
552 st1 [dst1]=r0,1
553 br.cloop.dptk 5b
554 ;;
555 mov pr=saved_pr,0xffffffffffff0000
556 mov ar.lc=saved_lc
557 mov ar.pfs=saved_pfs
558 br.ret.sptk.many rp
559
560 //
561 // Here we simply restart the loop but instead
562 // of doing loads we fill the pipeline with zeroes
563 // We can't simply store r0 because we may have valid
564 // data in transit in the pipeline.
565 // ar.lc and ar.ec are setup correctly at this point
566 //
567 // we MUST use src1/endsrc here and not dst1/enddst because
568 // of the pipeline effect.
569 //
570 .failure_in3:
571 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
572 ;;
573 2:
574 (p16) mov val1[0]=r0
575 (p16) mov val2[0]=r0
576 (EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16
577 (EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16
578 br.ctop.dptk 2b
579 ;;
580 cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ?
581 sub len=enddst,dst1,1 // precompute len
582 (p6) br.cond.dptk .failure_in1bis
583 ;;
584 mov pr=saved_pr,0xffffffffffff0000
585 mov ar.lc=saved_lc
586 mov ar.pfs=saved_pfs
587 br.ret.sptk.many rp
588
589 .failure_in2:
590 sub ret0=endsrc,src1
591 cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ?
592 sub len=enddst,dst1,1 // precompute len
593 (p6) br.cond.dptk .failure_in1bis
594 ;;
595 mov pr=saved_pr,0xffffffffffff0000
596 mov ar.lc=saved_lc
597 mov ar.pfs=saved_pfs
598 br.ret.sptk.many rp
599
600 //
601 // handling of failures on stores: that's the easy part
602 //
603 .failure_out:
604 sub ret0=enddst,dst1
605 mov pr=saved_pr,0xffffffffffff0000
606 mov ar.lc=saved_lc
607
608 mov ar.pfs=saved_pfs
609 br.ret.sptk.many rp
610 END(__copy_user)
