//		    ANSI C Runtime Library
//
//	Copyright 1983-2019 Green Hills Software, Inc.
//
//    This program is the property of Green Hills Software, Inc,
//    its contents are proprietary information and no part of it
//    is to be disclosed to anyone except employees of Green Hills
//    Software, Inc., or as agreed in writing signed by the President
//    of Green Hills Software, Inc.

	.file	"ind_mcpy.a64"

	.data

#if !defined(__GHS_USE_BACKWARD_COPY)
	// If you customize or replace memcpy, then delete the following
	// symbol definition to allow memmove to work. The symbol
	// __ghs_forward_memcpy allows the memmove library function to
	// perform better by calling memcpy in some cases where the arrays
	// overlap.
	.globl	__ghs_forward_memcpy
__ghs_forward_memcpy:
	.byte 0
#  if !defined(__VXWORKS)
	.type     __ghs_forward_memcpy, @object
	.size     __ghs_forward_memcpy, .-__ghs_forward_memcpy
#  endif
#else
	// Provide an alternate version of the function under a different name.
	// If you are customizing memcpy, do not use this name, as the name is
	// used internally by the GHS libraries.
#  define memcpy __ghs_memmove_backward
#endif

	.text

#if defined(__ghs_asm)
// Define an argcheck __callee symbol for the function memcpy.
// void * memcpy(void *, void *, size_t)
__callee.memcpy.p.ppl::
	.size __callee.memcpy.p.ppl,0
#endif

	.align 16
	.globl	memcpy
memcpy:
	// Parameters:
	//     x0 -- Dest pointer
	//     x1 -- Src pointer
	//     x2 -- Size of array in bytes
	// Return value:
	//     x0 -- The dest pointer passed in
	//           Note: since x0 is already set to the correct value, this
	//           function never modifies this register.
	//
	// Other registers used:
	//     x3       -- Next location to be written to
	//     x4..x11  -- Various short-lived temporary values
	//     x12..x13 -- First 16-byte chunk
	//     x14..x15 -- Last 16-byte chunk
	//     x16      -- Original end of dest array

	// This is a leaf function, so for efficiency don't create a stack
	// frame.

	// Basic approach:
	//   - If array is large, copy most of it via an unrolled loop that
	//     copies 4 pairs of registers (totalling 64 bytes) worth of memory
	//     per iteration.
	//   - If the array is small, and also to handle the odd 16-byte chunks
	//     for large arrays, copy one pair of registers at a time until
	//     we've copied enough.
	//   - If the array is really tiny, copy the entire array in two
	//     (possibly overlapping) chunks, taking care in case the src and
	//     dest arrays overlap. See the below comment "handling of extra
	//     bytes at array boundaries" for why we care about that.

	// Handling of extra bytes at array boundaries:
	//
	// Several of these implementations use a common strategy for
	// aligning arrays and dealing with arrays whose lengths aren't a
	// multiple of the chunk size. We explain that strategy here.
	//
	// The basic idea is that at each end, we copy one chunk that starts
	// or ends exactly at the boundary of the array, even though that
	// chunk may overlap one of the interior chunks. For example,
	// suppose we're copying a 21-byte array using a simple word loop,
	// which copies one 8-byte word at a time. Then we would copy the
	// word at bytes [0..7], the word at bytes [8..15], and the word at
	// bytes [13..20], as shown below:
	//
	//      _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
	//     |_|_|_|_|_|_|_|_|_|_|_|_|_|_|_|_|_|_|_|_|_|
	//
	//      \_____________/ \_____________/
	//                                \_____________/
	//
	// In principle, for memcpy it doesn't matter the order in which we
	// copy these words. It is invalid to call memcpy on overlapping
	// arrays, so in the above example, we may safely assume that
	// writing bytes [0..15] of the dest array won't modify the source
	// array, which means that when we read bytes [13..15] a second time
	// (as part of the third word), they will contain the same values as
	// the first time we read them.
	//
	// However, a nice property for memcpy to have is that one of the
	// following two things is true:
	//
	//     1. If dest begins before src, then memcpy will work correctly
	//        even if the arrays overlap; or
	//     2. If dest begins after  src, then memcpy will work correctly
	//        even if the arrays overlap.
	//
	// That is, it is nice for memcpy to behave as if it uses either a
	// simple forward copy or a simple backward copy. This allows (for
	// example) memmove to safely call memcpy in some overlapping cases,
	// as long as the arrays overlap in the correct direction.
	//
	// This implementation of memcpy acts like a forward copy, which
	// means that it will work correctly as long as dest begins before
	// src (option 1 from above). We ensure this by reading the boundary
	// chunks first (before we've copied any of the other chunks) and
	// writing them last (after we've copied all of the other chunks).
	// This ensures that the values we write for these boundary chunks
	// are the same values that were originally at the ends of the
	// source array, so these chunks are guaranteed to be correct even
	// if the arrays overlap. The remaining chunks _are_ copied using a
	// simple forward copy, so there's no problem with them.

	// General note on terminology: where the comments refer to the "first"
	// or "last" chunk, that is relative to the direction of iteration. So
	// in the default build, "first" means the lowest-addressed chunk and
	// "last" means the highest-addressed chunk. But under
	// __GHS_USE_BACKWARD_COPY, it's the other way around: "first" means
	// the highest-addressed chunk, while "last" means the lowest-addressed
	// chunk.

	cmp	x2, #16
	b.ls	array_is_tiny

#if !defined(__GHS_USE_BACKWARD_COPY)
	mov	x3, x0

	// x2 > 16, so there's at least one chunk. To handle the odd bytes,
	// we'll copy an overlapping chunk at the end. We need to read that
	// chunk before we start copying everything else, to ensure that we get
	// the right chunk even if the arrays overlap.
	add	x4, x1, x2
	ldp	x14, x15, [x4, #-16]  /* Last chunk */
	add	x16, x0, x2           /* End of dest array */
#else
	// As in the forward-copy case, save off the "last" chunk to handle odd
	// bytes at the end of our iteration. Since we're iterating backwards,
	// the "last" chunk is the one with the lowest address.
	add	x16, x0, x2     /* End of dest array */
	ldp	x14, x15, [x1]  /* Lowest-address chunk */

	// These registers are used for iterating over the two arrays.
	// Initialize them to the ends of the arrays to iterate backward.
	add	x1, x1, x2      /* End of src */
	mov	x3, x16         /* End of dest */
#endif

	// In order to use the unrolled loop below, there need to be at least
	// 64 bytes to copy after adjusting for the possibly-misaligned chunks
	// at the end (this is so that we can put the loop check at the end).
	// The length will be decreased by up to 16 bytes due to the first
	// chunk (in particular, by exactly 16 if the dest array was already
	// aligned), and by 1 byte regardless due to the last chunk. Thus we
	// need at least 81 bytes to ensure that there are enough for a full
	// iteration.
	cmp	x2, #81
	b.lo	array_is_small

	// Since the array is not too short, we can afford the overhead to
	// align the dest pointer, so that at least all the writes will be to
	// aligned addresses. To handle any odd bytes at the start, we save off
	// the first chunk (which may overlap the second chunk) and write it
	// later.
	//
	// If the first chunk of the dest array is already aligned, move a full
	// chunk in the direction of iteration so we don't copy the first chunk
	// twice.
#if !defined(__GHS_USE_BACKWARD_COPY)
	ldp	x12, x13, [x1]
	add	x3, x3, #16
#else
	ldp	x12, x13, [x1, #-16]
	// x3 = (x3 - 16, rounded up to a multiple of 16)
	// Equivalently: x3 - 16 + (16 - 1), rounded down to a multiple of 16.
	sub	x3, x3, #1
#endif
	and	x3, x3, #0xfffffffffffffff0

	// Adjust the src pointer and array size accordingly.
#if !defined(__GHS_USE_BACKWARD_COPY)
	//     src += (new dest - old dest)
	//     len -= (new dest - old dest)
	// That is:
	//     x1  += (x3       - x0      )
	//     x2  -= (x3       - x0      )
	add	x2, x2, x0
	sub	x1, x1, x0
	sub	x2, x2, x3
	add	x1, x1, x3
#else
	//     srcEnd -= (old destEnd - new destEnd)
	//     remaining size = (adjusted dest end ptr) - (dest start ptr).
	// That is:
	//     x1 -= (x16 - x3)
	//     x2  = x3 - x0
	sub	x1, x1, x16
	sub	x2, x3, x0
	add	x1, x1, x3
#endif

	// We now subtract 1 from the length so that if it's an exact multiple
	// of 16, we don't copy the last chunk twice.
	//
	// To see why this works, consider the two cases:
	//
	//     Case 1: the length is a multiple of 16. Then this will cause the
	//         main loop to skip the last chunk. However, we are already
	//         copying the last chunk separately, so we will still copy
	//         every byte.
	//
	//     Case 2: the length is not a multiple of 16. Since the main loop
	//         deals only in 16-byte chunks, this will not change the
	//         number of chunks copied by that loop, so is definitely safe.
	//
	// Note: we can't subtract any more than 1 here because the length
	// could be exactly one more than a multiple of 16, in which case
	// subtracting more than 1 would cause us to miss the byte at the start
	// of the last whole chunk.

	// Rather than doing a subtract followed by a compare against 64 at the
	// end of each iteration of the unrolled loop, we save an instruction
	// by also decreasing x2 by 64 here, so that we can do a single 'subs'
	// on all subsequent iterations. Unlike the above subtract, this one
	// only lasts for the duration of the unrolled loop.

	// As an optimization, the above two are combined into a single
	// subtract.
	sub	x2, x2, #65

unrolled_regpair_copy:
#if !defined(__GHS_USE_BACKWARD_COPY)
	// Note: we could eliminate the 'add x1, x1, #64' and 'add x3, x3, #64'
	// instructions by using the pre- or post-index addressing mode
	// ('ldp x10, x11, [x1,#64]!' or 'ldp x10, x11, [x1], #64]',
	// respectively). However, the former requires we offset x1 and x3
	// before and after this loop, and the latter means updating x1 and x3
	// more times (with every load/store, rather than once per iteration).
	// Empirically, either of those ways is actually slightly slower
	// overall than just using an 'add'.
	ldp	x4,  x5,  [x1]
	ldp	x6,  x7,  [x1,#16]
	ldp	x8,  x9,  [x1,#32]
	ldp	x10, x11, [x1,#48]
	add	x1, x1, #64
	stp	x4,  x5,  [x3]
	stp	x6,  x7,  [x3,#16]
	stp	x8,  x9,  [x3,#32]
	stp	x10, x11, [x3,#48]
	add	x3, x3, #64
#else
	ldp	x4,  x5,  [x1,#-16]
	ldp	x6,  x7,  [x1,#-32]
	ldp	x8,  x9,  [x1,#-48]
	ldp	x10, x11, [x1,#-64]!
	stp	x4,  x5,  [x3,#-16]
	stp	x6,  x7,  [x3,#-32]
	stp	x8,  x9,  [x3,#-48]
	stp	x10, x11, [x3,#-64]!
#endif
	subs	x2, x2, #64
	b.hs	unrolled_regpair_copy

	// Undo the offset we made to x2 for the duration of the loop.
	add	x2, x2, #64

	// We have now copied at least one iteration worth of memory (64 bytes)
	// after the first chunk (the one stored in (x12,x13)). In particular,
	// this means that we have now read the second chunk, which is the only
	// chunk that may overlap the first chunk -- see "Handling of extra
	// bytes at array boundaries" above. This means that it is now safe to
	// write the first chunk to the dest array.
	//
	// We need to do that here, rather than below under fixup_done where we
	// handle the last chunk, because we can reach fixup_done without
	// having read the first chunk into (x12,x13).
#if !defined(__GHS_USE_BACKWARD_COPY)
	stp	x12, x13, [x0]
#else
	stp	x12, x13, [x16,#-16]
#endif

	// There are now fewer than 64 bytes left, which means there are at
	// most 3 full chunks to copy (the odd bytes will be handled afterward,
	// under fixup_done).
	b	fixup_copies

array_is_small:
	// As with the unrolled loop above, we can subtract 1 from the length
	// because we're going to write the last chunk later. We also want to
	// compare the resulting length to 16 to know if there's enough bytes
	// left that we should copy any chunks. If so, we *also* want to
	// decrease the length by 16, because the fixup copies below are
	// written to assume that x2 stores the remaining number of bytes to be
	// copied as of when the execution hits fixup_copies. We can combine
	// all three of these operations into a single 'subs x2, x2, #17'.
	subs	x2, x2, #17
	b.lo	fixup_done
	// We are going to fall through into fixup_copies, so make sure to move
	// the array pointers one chunk in the direction of iteration because
	// the fixup copies use hardcoded offsets.
#if !defined(__GHS_USE_BACKWARD_COPY)
	ldp	x4,  x5,  [x1], #16
	stp	x4,  x5,  [x3], #16
#else
	ldp	x4,  x5,  [x1,#-16]!
	stp	x4,  x5,  [x3,#-16]!
#endif

fixup_copies:
	// Handle any chunks that were leftover from the unrolled loop. Also
	// used to handle the last (up to) 3 chunks when there were at most 4
	// whole chunks to begin with (array_is_small).
	cmp	x2, #16
	b.lo	fixup_done
#if !defined(__GHS_USE_BACKWARD_COPY)
	ldp	x6,  x7,  [x1]
	stp	x6,  x7,  [x3]
#else
	ldp	x6,  x7,  [x1,#-16]
	stp	x6,  x7,  [x3,#-16]
#endif

	cmp	x2, #32
	b.lo	fixup_done
#if !defined(__GHS_USE_BACKWARD_COPY)
	ldp	x8,  x9,  [x1,#16]
	stp	x8,  x9,  [x3,#16]
#else
	ldp	x8,  x9,  [x1,#-32]
	stp	x8,  x9,  [x3,#-32]
#endif

	cmp	x2, #48
	b.lo	fixup_done
#if !defined(__GHS_USE_BACKWARD_COPY)
	ldp	x10, x11, [x1,#32]
	stp	x10, x11, [x3,#32]
#else
	ldp	x10, x11, [x1,#-48]
	stp	x10, x11, [x3,#-48]
#endif

fixup_done:
	// Write the last chunk, which may overlap with the previous chunk.
#if !defined(__GHS_USE_BACKWARD_COPY)
	stp	x14, x15, [x16,#-16]
#else
	stp	x14, x15, [x0]
#endif
	ret

array_is_tiny:
	// Copy the whole array in two (possibly overlapping) chunks of the
	// same power-of-two size, one starting at the start of the array
	// and the other ending at the end of the array. The chunks' sizes
	// are chosen based on the size of the array. Read both chunks
	// before writing either, in case the src and dest arrays overlap;
	// see the above comment "handling of extra bytes at array
	// boundaries" for why we care about this.

	add	x4, x0, x2  /* End of dest array. */
	add	x5, x1, x2  /* End of src array. */

// at_most_16_bytes:
	cmp	x2, #8
	b.ls	at_most_8_bytes

	// Copy (8..16] bytes using two 8-byte copies.
	ldr	x6, [x1]
	ldr	x7, [x5,#-8]
	str	x6, [x0]
	str	x7, [x4,#-8]
	ret

at_most_8_bytes:
	cmp	x2, #4
	b.ls	at_most_4_bytes

	// Copy (4..8] bytes using two 4-byte copies.
	ldr	w6, [x1]
	ldr	w7, [x5,#-4]
	str	w6, [x0]
	str	w7, [x4,#-4]
	ret

at_most_4_bytes:
	cmp	x2, #2
	b.ls	at_most_2_bytes

	// Copy (2..4] bytes using two 2-byte copies.
	ldrh	w6, [x1]
	ldrh	w7, [x5,#-2]
	strh	w6, [x0]
	strh	w7, [x4,#-2]
	ret

at_most_2_bytes:
	cbz	x2, all_done

	// Copy 1 or 2 bytes using two 1-byte copies.
	ldrb	w6, [x1]
	ldrb	w7, [x5,#-1]
	strb	w6, [x0]
	strb	w7, [x4,#-1]
	// Fall through

all_done:
	ret

#if !defined(__VXWORKS)
	.type     memcpy, @function
	.size     memcpy, .-memcpy
#if !defined(__linux)
	.maxstack memcpy, 0
	.scall    memcpy, __leaf__
#endif
#endif

#if !defined(__GHS_USE_BACKWARD_COPY)
	.global	__ghs_fast_memcpy
	.set    __ghs_fast_memcpy, memcpy
	
#if defined(__ghs_asm)
// Define an argcheck __callee symbol for the function memcpy.
// void * memcpy(void *, void *, size_t)
__callee.__ghs_fast_memcpy.p.ppl::
	.size __callee.__ghs_fast_memcpy.p.ppl,0
#endif
#endif