/*
 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
 *           (C) 2020 Vladimir Sadovnikov <sadko4u@gmail.com>
 *
 * This file is part of lsp-dsp-lib
 * Created on: 31 мар. 2020 г.
 *
 * lsp-dsp-lib is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * any later version.
 *
 * lsp-dsp-lib is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
 */

#ifndef PRIVATE_DSP_ARCH_X86_SSE_COPY_H_
#define PRIVATE_DSP_ARCH_X86_SSE_COPY_H_

#ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
    #error "This header should not be included directly"
#endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */

namespace lsp
{
    namespace sse
    {
        void copy(float *dst, const float *src, size_t count)
        {
            IF_ARCH_X86(size_t off);

            ARCH_X86_ASM
            (
                __ASM_EMIT("cmp         %[dst], %[src]")
                __ASM_EMIT("je          10f")

                /* Destination is aligned */
                __ASM_EMIT("xor         %[off], %[off]")
                __ASM_EMIT("sub         $0x20, %[count]")
                __ASM_EMIT("jb          2f")

                /* Block of 32 items */
                __ASM_EMIT("1:")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      0x20(%[src], %[off]), %%xmm2")
                __ASM_EMIT("movups      0x30(%[src], %[off]), %%xmm3")
                __ASM_EMIT("movups      0x40(%[src], %[off]), %%xmm4")
                __ASM_EMIT("movups      0x50(%[src], %[off]), %%xmm5")
                __ASM_EMIT("movups      0x60(%[src], %[off]), %%xmm6")
                __ASM_EMIT("movups      0x70(%[src], %[off]), %%xmm7")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm2, 0x20(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm3, 0x30(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm4, 0x40(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm5, 0x50(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm6, 0x60(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm7, 0x70(%[dst], %[off])")
                __ASM_EMIT("add         $0x80, %[off]")
                __ASM_EMIT("sub         $0x20, %[count]")
                __ASM_EMIT("jae         1b")
                /* Block of 16 items */
                __ASM_EMIT("2:")
                __ASM_EMIT("add         $0x10, %[count]")
                __ASM_EMIT("jl          4f")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      0x20(%[src], %[off]), %%xmm2")
                __ASM_EMIT("movups      0x30(%[src], %[off]), %%xmm3")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm2, 0x20(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm3, 0x30(%[dst], %[off])")
                __ASM_EMIT("add         $0x40, %[off]")
                __ASM_EMIT("sub         $0x10, %[count]")
                /* Block of 8 items */
                __ASM_EMIT("4:")
                __ASM_EMIT("add         $0x08, %[count]")
                __ASM_EMIT("jl          6f")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("add         $0x20, %[off]")
                __ASM_EMIT("sub         $0x08, %[count]")
                /* Block of 4 items */
                __ASM_EMIT("6:")
                __ASM_EMIT("add         $0x04, %[count]")
                __ASM_EMIT("jl          8f")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("add         $0x10, %[off]")
                __ASM_EMIT("sub         $0x04, %[count]")
                /* 1x items */
                __ASM_EMIT("8:")
                __ASM_EMIT("add         $0x03, %[count]")
                __ASM_EMIT("jl          10f")
                __ASM_EMIT("9:")
                __ASM_EMIT("movss       0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("add         $0x04, %[off]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jge         9b")

                __ASM_EMIT("10:")

                : [src] "+r" (src), [dst] "+r"(dst),
                  [count] "+r" (count), [off] "=&r" (off)
                :
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        void copy_movntps(float *dst, const float *src, size_t count)
        {
            IF_ARCH_X86(size_t off);

            ARCH_X86_ASM
            (
                __ASM_EMIT("cmp         %[dst], %[src]")
                __ASM_EMIT("je          100f")
                __ASM_EMIT("test        %[count], %[count]")
                __ASM_EMIT("jz          100f")

                /* Align destination */
                __ASM_EMIT("test        $0x0f, %[dst]")
                __ASM_EMIT("jz          2f")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")
                __ASM_EMIT("movss       %%xmm0, (%[dst])")
                __ASM_EMIT("add         $4, %[src]")
                __ASM_EMIT("add         $4, %[dst]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jz          100f")
                __ASM_EMIT("test        $0x0f, %[dst]")
                __ASM_EMIT("jnz         1b")

                /* Block of 32 items */
                __ASM_EMIT("2:")
                __ASM_EMIT("xor         %[off], %[off]")
                __ASM_EMIT("sub         $0x20, %[count]")
                __ASM_EMIT("jb          4f")

                __ASM_EMIT("3:")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      0x20(%[src], %[off]), %%xmm2")
                __ASM_EMIT("movups      0x30(%[src], %[off]), %%xmm3")
                __ASM_EMIT("movups      0x40(%[src], %[off]), %%xmm4")
                __ASM_EMIT("movups      0x50(%[src], %[off]), %%xmm5")
                __ASM_EMIT("movups      0x60(%[src], %[off]), %%xmm6")
                __ASM_EMIT("movups      0x70(%[src], %[off]), %%xmm7")
                __ASM_EMIT("movntps     %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm2, 0x20(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm3, 0x30(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm4, 0x40(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm5, 0x50(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm6, 0x60(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm7, 0x70(%[dst], %[off])")
                __ASM_EMIT("add         $0x80, %[off]")
                __ASM_EMIT("sub         $0x20, %[count]")
                __ASM_EMIT("jae         3b")

                /* Block of 16 items */
                __ASM_EMIT("4:")
                __ASM_EMIT("add         $0x10, %[count]")
                __ASM_EMIT("jl          6f")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      0x20(%[src], %[off]), %%xmm2")
                __ASM_EMIT("movups      0x30(%[src], %[off]), %%xmm3")
                __ASM_EMIT("movntps     %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm2, 0x20(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm3, 0x30(%[dst], %[off])")
                __ASM_EMIT("add         $0x40, %[off]")
                __ASM_EMIT("sub         $0x10, %[count]")
                /* Block of 8 items */
                __ASM_EMIT("6:")
                __ASM_EMIT("add         $0x08, %[count]")
                __ASM_EMIT("jl          8f")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movntps     %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movntps     %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("add         $0x20, %[off]")
                __ASM_EMIT("sub         $0x08, %[count]")
                /* Block of 4 items */
                __ASM_EMIT("8:")
                __ASM_EMIT("add         $0x04, %[count]")
                __ASM_EMIT("jl          10f")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movntps     %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("add         $0x10, %[off]")
                __ASM_EMIT("sub         $0x04, %[count]")
                /* 1x items */
                __ASM_EMIT("10:")
                __ASM_EMIT("add         $0x03, %[count]")
                __ASM_EMIT("jl          12f")
                __ASM_EMIT("9:")
                __ASM_EMIT("movss       0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("add         $0x04, %[off]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jge         9b")

                __ASM_EMIT("12:")
                __ASM_EMIT("sfence")

                __ASM_EMIT("100:")

                : [src] "+r" (src), [dst] "+r"(dst),
                  [count] "+r" (count), [off] "=&r" (off)
                :
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        void copy_backward(float *dst, const float *src, size_t count)
        {
            IF_ARCH_X86(size_t off);

            ARCH_X86_ASM
            (
                __ASM_EMIT("cmp         %[dst], %[src]")
                __ASM_EMIT("je          10f")

                __ASM_EMIT("lea         (,%[count], 4), %[off]")
                __ASM_EMIT("sub         $0x20, %[count]")
                __ASM_EMIT("jb          2f")

                /* Block of 32 items */
                __ASM_EMIT("1:")
                __ASM_EMIT("sub         $0x80, %[off]")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      0x20(%[src], %[off]), %%xmm2")
                __ASM_EMIT("movups      0x30(%[src], %[off]), %%xmm3")
                __ASM_EMIT("movups      0x40(%[src], %[off]), %%xmm4")
                __ASM_EMIT("movups      0x50(%[src], %[off]), %%xmm5")
                __ASM_EMIT("movups      0x60(%[src], %[off]), %%xmm6")
                __ASM_EMIT("movups      0x70(%[src], %[off]), %%xmm7")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm2, 0x20(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm3, 0x30(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm4, 0x40(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm5, 0x50(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm6, 0x60(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm7, 0x70(%[dst], %[off])")
                __ASM_EMIT("sub         $0x20, %[count]")
                __ASM_EMIT("jae         1b")
                /* Block of 16 items */
                __ASM_EMIT("2:")
                __ASM_EMIT("add         $0x10, %[count]")
                __ASM_EMIT("jl          4f")
                __ASM_EMIT("sub         $0x40, %[off]")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      0x20(%[src], %[off]), %%xmm2")
                __ASM_EMIT("movups      0x30(%[src], %[off]), %%xmm3")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm2, 0x20(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm3, 0x30(%[dst], %[off])")
                __ASM_EMIT("sub         $0x10, %[count]")
                /* Block of 8 items */
                __ASM_EMIT("4:")
                __ASM_EMIT("add         $0x08, %[count]")
                __ASM_EMIT("jl          6f")
                __ASM_EMIT("sub         $0x20, %[off]")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      0x10(%[src], %[off]), %%xmm1")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("movups      %%xmm1, 0x10(%[dst], %[off])")
                __ASM_EMIT("sub         $0x08, %[count]")
                /* Block of 4 items */
                __ASM_EMIT("6:")
                __ASM_EMIT("add         $0x04, %[count]")
                __ASM_EMIT("jl          8f")
                __ASM_EMIT("sub         $0x10, %[off]")
                __ASM_EMIT("movups      0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movups      %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("sub         $0x04, %[count]")
                /* 1x items */
                __ASM_EMIT("8:")
                __ASM_EMIT("add         $0x03, %[count]")
                __ASM_EMIT("jl          10f")
                __ASM_EMIT("9:")
                __ASM_EMIT("sub         $0x04, %[off]")
                __ASM_EMIT("movss       0x00(%[src], %[off]), %%xmm0")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[dst], %[off])")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jge         9b")

                __ASM_EMIT("10:")

                : [src] "+r" (src), [dst] "+r"(dst),
                  [count] "+r" (count), [off] "=&r" (off)
                :
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        void move(float *dst, const float *src, size_t count)
        {
            if (dst < src)
                sse::copy(dst, src, count);
            else if (dst > src)
                sse::copy_backward(dst, src, count);
        }

        void fill(float *dst, float value, size_t count)
        {
            #define FILL_CORE   \
                /* Blocks of 32 items */ \
                __ASM_EMIT("sub         $0x20, %[count]")   \
                __ASM_EMIT("jb          4f")   \
                __ASM_EMIT("movaps      %%xmm0, %%xmm4") \
                __ASM_EMIT("movaps      %%xmm1, %%xmm5") \
                __ASM_EMIT("movaps      %%xmm2, %%xmm6") \
                __ASM_EMIT("movaps      %%xmm3, %%xmm7") \
                __ASM_EMIT("3:") \
                __ASM_EMIT("movaps      %%xmm0, 0x00(%[dst])") \
                __ASM_EMIT("movaps      %%xmm1, 0x10(%[dst])") \
                __ASM_EMIT("movaps      %%xmm2, 0x20(%[dst])") \
                __ASM_EMIT("movaps      %%xmm3, 0x30(%[dst])") \
                __ASM_EMIT("movaps      %%xmm4, 0x40(%[dst])") \
                __ASM_EMIT("movaps      %%xmm5, 0x50(%[dst])") \
                __ASM_EMIT("movaps      %%xmm6, 0x60(%[dst])") \
                __ASM_EMIT("movaps      %%xmm7, 0x70(%[dst])") \
                __ASM_EMIT("add         $0x80, %[dst]") \
                __ASM_EMIT("sub         $0x20, %[count]") \
                __ASM_EMIT("jae         3b")   \
                /* Blocks of 16 items */ \
                __ASM_EMIT("4:") \
                __ASM_EMIT("add         $0x20, %[count]") \
                __ASM_EMIT("test        $0x10, %[count]") \
                __ASM_EMIT("jz          5f") \
                __ASM_EMIT("movaps      %%xmm0, 0x00(%[dst])") \
                __ASM_EMIT("movaps      %%xmm1, 0x10(%[dst])") \
                __ASM_EMIT("movaps      %%xmm2, 0x20(%[dst])") \
                __ASM_EMIT("movaps      %%xmm3, 0x30(%[dst])") \
                __ASM_EMIT("add         $0x40, %[dst]") \
                /* Blocks of 8 items */ \
                __ASM_EMIT("5:") \
                __ASM_EMIT("test        $0x08, %[count]") \
                __ASM_EMIT("jz          6f") \
                __ASM_EMIT("movaps      %%xmm0, 0x00(%[dst])") \
                __ASM_EMIT("movaps      %%xmm1, 0x10(%[dst])") \
                __ASM_EMIT("add         $0x20, %[dst]") \
                /* Blocks of 4 items */ \
                __ASM_EMIT("6:") \
                __ASM_EMIT("test        $0x04, %[count]") \
                __ASM_EMIT("jz          7f") \
                __ASM_EMIT("movaps      %%xmm0, 0x00(%[dst])") \
                __ASM_EMIT("add         $0x10, %[dst]") \
                /* Tail */ \
                __ASM_EMIT("7:") \
                __ASM_EMIT("and         $0x03, %[count]") \
                __ASM_EMIT("jz          2000f") \
                __ASM_EMIT("8:") \
                __ASM_EMIT("movss       %%xmm0, 0x00(%[dst])") \
                __ASM_EMIT("add         $0x04, %[dst]") \
                __ASM_EMIT("dec         %[count]") \
                __ASM_EMIT("jnz         8b")

            ARCH_X86_ASM
            (
                __ASM_EMIT("test        %[count], %[count]")
                __ASM_EMIT("jz          2000f")
                __ASM_EMIT("shufps      $0x00, %%xmm0, %%xmm0")

                /* Align destination */
                __ASM_EMIT("1:")
                __ASM_EMIT("test        $0x0f, %[dst]")
                __ASM_EMIT("jz          2f")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[dst])")
                __ASM_EMIT("add         $0x4, %[dst]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         1b")
                __ASM_EMIT("jmp         2000f")

                __ASM_EMIT("2:")
                __ASM_EMIT("movaps      %%xmm0, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %%xmm2")
                __ASM_EMIT("movaps      %%xmm1, %%xmm3")

                /* Destination is aligned */
                FILL_CORE

                __ASM_EMIT("2000:")

                : [dst] "+r"(dst), [count] "+r" (count), "+Yz"(value) :
                : "cc", "memory",
                  "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );

            #undef FILL_CORE
        }

        void fill_zero(float *dst, size_t count)
        {
            fill(dst, 0.0f, count);
        }

        void fill_one(float *dst, size_t count)
        {
            fill(dst, 1.0f, count);
        }

        void fill_minus_one(float *dst, size_t count)
        {
            fill(dst, -1.0f, count);
        }

        #define REVERSE1_CORE(MV_TAIL) \
            __ASM_EMIT("sub         $0x10, %[count]") \
            __ASM_EMIT("jb          4f") \
            /* x16 blocks */ \
            __ASM_EMIT("3:") \
            __ASM_EMIT("sub         $0x40, %[tail]") \
            __ASM_EMIT("movaps      0x00(%[head]), %%xmm3") \
            __ASM_EMIT("movaps      0x10(%[head]), %%xmm2") \
            __ASM_EMIT("movaps      0x20(%[head]), %%xmm1") \
            __ASM_EMIT("movaps      0x30(%[head]), %%xmm0") \
            __ASM_EMIT(MV_TAIL "    0x00(%[tail]), %%xmm7") \
            __ASM_EMIT(MV_TAIL "    0x10(%[tail]), %%xmm6") \
            __ASM_EMIT(MV_TAIL "    0x20(%[tail]), %%xmm5") \
            __ASM_EMIT(MV_TAIL "    0x30(%[tail]), %%xmm4") \
            __ASM_EMIT("shufps      $0x1b, %%xmm0, %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm1, %%xmm1") \
            __ASM_EMIT("shufps      $0x1b, %%xmm2, %%xmm2") \
            __ASM_EMIT("shufps      $0x1b, %%xmm3, %%xmm3") \
            __ASM_EMIT("shufps      $0x1b, %%xmm4, %%xmm4") \
            __ASM_EMIT("shufps      $0x1b, %%xmm5, %%xmm5") \
            __ASM_EMIT("shufps      $0x1b, %%xmm6, %%xmm6") \
            __ASM_EMIT("shufps      $0x1b, %%xmm7, %%xmm7") \
            __ASM_EMIT("movaps      %%xmm4, 0x00(%[head])") \
            __ASM_EMIT("movaps      %%xmm5, 0x10(%[head])") \
            __ASM_EMIT("movaps      %%xmm6, 0x20(%[head])") \
            __ASM_EMIT("movaps      %%xmm7, 0x30(%[head])") \
            __ASM_EMIT(MV_TAIL "    %%xmm0, 0x00(%[tail])") \
            __ASM_EMIT(MV_TAIL "    %%xmm1, 0x10(%[tail])") \
            __ASM_EMIT(MV_TAIL "    %%xmm2, 0x20(%[tail])") \
            __ASM_EMIT(MV_TAIL "    %%xmm3, 0x30(%[tail])") \
            __ASM_EMIT("add         $0x40, %[head]") \
            __ASM_EMIT("sub         $0x10, %[count]") \
            __ASM_EMIT("jae         3b") \
            /* x8 block */ \
            __ASM_EMIT("4:") \
            __ASM_EMIT("add         $0x10, %[count]") \
            __ASM_EMIT("test        $0x08, %[count]") \
            __ASM_EMIT("jz          5f") \
            __ASM_EMIT("sub         $0x20, %[tail]") \
            __ASM_EMIT("movaps      0x00(%[head]), %%xmm1") \
            __ASM_EMIT("movaps      0x10(%[head]), %%xmm0") \
            __ASM_EMIT(MV_TAIL "    0x00(%[tail]), %%xmm3") \
            __ASM_EMIT(MV_TAIL "    0x10(%[tail]), %%xmm2") \
            __ASM_EMIT("shufps      $0x1b, %%xmm0, %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm1, %%xmm1") \
            __ASM_EMIT("shufps      $0x1b, %%xmm2, %%xmm2") \
            __ASM_EMIT("shufps      $0x1b, %%xmm3, %%xmm3") \
            __ASM_EMIT("movaps      %%xmm2, 0x00(%[head])") \
            __ASM_EMIT("movaps      %%xmm3, 0x10(%[head])") \
            __ASM_EMIT(MV_TAIL "    %%xmm0, 0x00(%[tail])") \
            __ASM_EMIT(MV_TAIL "    %%xmm1, 0x10(%[tail])") \
            __ASM_EMIT("add         $0x20, %[head]") \
            /* x4 block */ \
            __ASM_EMIT("5:") \
            __ASM_EMIT("test        $0x04, %[count]") \
            __ASM_EMIT("jz          3000f") \
            __ASM_EMIT("sub         $0x10, %[tail]") \
            __ASM_EMIT("movaps      0x00(%[head]), %%xmm0") \
            __ASM_EMIT(MV_TAIL "    0x00(%[tail]), %%xmm1") \
            __ASM_EMIT("shufps      $0x1b, %%xmm0, %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm1, %%xmm1") \
            __ASM_EMIT("movaps      %%xmm1, 0x00(%[head])") \
            __ASM_EMIT(MV_TAIL "    %%xmm0, 0x00(%[tail])") \
            __ASM_EMIT("add         $0x10, %[head]")

        void reverse1(float *dst, size_t count)
        {
            const float *src;

            ARCH_X86_ASM
            (
                __ASM_EMIT("lea         (%[head], %[count], 4), %[tail]")
                __ASM_EMIT("shr         $1, %[count]")
                __ASM_EMIT("jz          2000f")

                // Align head
                __ASM_EMIT("1:")
                __ASM_EMIT("test        $0x0f, %[head]")
                __ASM_EMIT("jz          2f")
                __ASM_EMIT("sub         $0x04, %[tail]")
                __ASM_EMIT("movss       0x00(%[head]), %%xmm0")
                __ASM_EMIT("movss       0x00(%[tail]), %%xmm1")
                __ASM_EMIT("movss       %%xmm1, 0x00(%[head])")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[tail])")
                __ASM_EMIT("add         $0x04, %[head]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         1b")
                __ASM_EMIT("jmp         2000f")

                // Block operation
                __ASM_EMIT("2:")
                __ASM_EMIT("test        $0x0f, %[tail]")
                __ASM_EMIT("jnz         1001f")
                    REVERSE1_CORE("movaps")
                    __ASM_EMIT("jmp         3000f")
                __ASM_EMIT("1001:")
                    REVERSE1_CORE("movups")

                /* x1 blocks */
                __ASM_EMIT("3000:")
                __ASM_EMIT("and         $0x03, %[count]")
                __ASM_EMIT("jz          2000f")
                __ASM_EMIT("7:")
                __ASM_EMIT("sub         $0x04, %[tail]")
                __ASM_EMIT("movss       0x00(%[head]), %%xmm0")
                __ASM_EMIT("movss       0x00(%[tail]), %%xmm1")
                __ASM_EMIT("movss       %%xmm1, 0x00(%[head])")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[tail])")
                __ASM_EMIT("add         $0x04, %[head]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         7b")

                __ASM_EMIT("2000:")

                : [head] "+r" (dst), [tail] "=&r" (src), [count] "+r" (count)
                :
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        #undef REVERSE1_CORE

        #define REVERSE2_CORE(MV_TAIL) \
            __ASM_EMIT("sub         $0x20, %[count]") \
            __ASM_EMIT("jb          4f") \
            /* x32 blocks */ \
            __ASM_EMIT("3:") \
            __ASM_EMIT(MV_TAIL "   -0x80(%[tail]), %%xmm7") \
            __ASM_EMIT(MV_TAIL "   -0x70(%[tail]), %%xmm6") \
            __ASM_EMIT(MV_TAIL "   -0x60(%[tail]), %%xmm5") \
            __ASM_EMIT(MV_TAIL "   -0x50(%[tail]), %%xmm4") \
            __ASM_EMIT(MV_TAIL "   -0x40(%[tail]), %%xmm3") \
            __ASM_EMIT(MV_TAIL "   -0x30(%[tail]), %%xmm2") \
            __ASM_EMIT(MV_TAIL "   -0x20(%[tail]), %%xmm1") \
            __ASM_EMIT(MV_TAIL "   -0x10(%[tail]), %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm4, %%xmm4") \
            __ASM_EMIT("shufps      $0x1b, %%xmm5, %%xmm5") \
            __ASM_EMIT("shufps      $0x1b, %%xmm6, %%xmm6") \
            __ASM_EMIT("shufps      $0x1b, %%xmm7, %%xmm7") \
            __ASM_EMIT("shufps      $0x1b, %%xmm0, %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm1, %%xmm1") \
            __ASM_EMIT("shufps      $0x1b, %%xmm2, %%xmm2") \
            __ASM_EMIT("shufps      $0x1b, %%xmm3, %%xmm3") \
            __ASM_EMIT("movaps      %%xmm0, 0x00(%[head])") \
            __ASM_EMIT("movaps      %%xmm1, 0x10(%[head])") \
            __ASM_EMIT("movaps      %%xmm2, 0x20(%[head])") \
            __ASM_EMIT("movaps      %%xmm3, 0x30(%[head])") \
            __ASM_EMIT("movaps      %%xmm4, 0x40(%[head])") \
            __ASM_EMIT("movaps      %%xmm5, 0x50(%[head])") \
            __ASM_EMIT("movaps      %%xmm6, 0x60(%[head])") \
            __ASM_EMIT("movaps      %%xmm7, 0x70(%[head])") \
            __ASM_EMIT("sub         $0x80, %[tail]") \
            __ASM_EMIT("add         $0x80, %[head]") \
            __ASM_EMIT("sub         $0x20, %[count]") \
            __ASM_EMIT("jae         3b") \
            /* x16 block */ \
            __ASM_EMIT("4:") \
            __ASM_EMIT("add         $0x20, %[count]") \
            __ASM_EMIT("test        $0x10, %[count]") \
            __ASM_EMIT("jz          5f") \
            __ASM_EMIT(MV_TAIL "   -0x40(%[tail]), %%xmm3") \
            __ASM_EMIT(MV_TAIL "   -0x30(%[tail]), %%xmm2") \
            __ASM_EMIT(MV_TAIL "   -0x20(%[tail]), %%xmm1") \
            __ASM_EMIT(MV_TAIL "   -0x10(%[tail]), %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm2, %%xmm2") \
            __ASM_EMIT("shufps      $0x1b, %%xmm3, %%xmm3") \
            __ASM_EMIT("shufps      $0x1b, %%xmm0, %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm1, %%xmm1") \
            __ASM_EMIT("movaps      %%xmm0, 0x00(%[head])") \
            __ASM_EMIT("movaps      %%xmm1, 0x10(%[head])") \
            __ASM_EMIT("movaps      %%xmm2, 0x20(%[head])") \
            __ASM_EMIT("movaps      %%xmm3, 0x30(%[head])") \
            __ASM_EMIT("sub         $0x40, %[tail]") \
            __ASM_EMIT("add         $0x40, %[head]") \
            /* x8 block */ \
            __ASM_EMIT("5:") \
            __ASM_EMIT("test        $0x08, %[count]") \
            __ASM_EMIT("jz          6f") \
            __ASM_EMIT(MV_TAIL "   -0x20(%[tail]), %%xmm1") \
            __ASM_EMIT(MV_TAIL "   -0x10(%[tail]), %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm1, %%xmm1") \
            __ASM_EMIT("shufps      $0x1b, %%xmm0, %%xmm0") \
            __ASM_EMIT("movaps      %%xmm0, 0x00(%[head])") \
            __ASM_EMIT("movaps      %%xmm1, 0x10(%[head])") \
            __ASM_EMIT("sub         $0x20, %[tail]") \
            __ASM_EMIT("add         $0x20, %[head]") \
            /* x4 block */ \
            __ASM_EMIT("6:") \
            __ASM_EMIT("test        $0x04, %[count]") \
            __ASM_EMIT("jz          3000f") \
            __ASM_EMIT(MV_TAIL "   -0x10(%[tail]), %%xmm0") \
            __ASM_EMIT("shufps      $0x1b, %%xmm0, %%xmm0") \
            __ASM_EMIT("movaps      %%xmm0, 0x00(%[head])") \
            __ASM_EMIT("sub         $0x10, %[tail]") \
            __ASM_EMIT("add         $0x10, %[head]")

        void reverse2(float *dst, const float *src, size_t count)
        {
            if (dst == src)
            {
                reverse1(dst, count);
                return;
            }

            ARCH_X86_ASM
            (
                __ASM_EMIT("test        %[count], %[count]")
                __ASM_EMIT("jz          2000f")

                __ASM_EMIT("lea         (%[tail], %[count], 4), %[tail]")

                // Align head
                __ASM_EMIT("1:")
                __ASM_EMIT("test        $0x0f, %[head]")
                __ASM_EMIT("jz          2f")
                __ASM_EMIT("sub         $0x04, %[tail]")
                __ASM_EMIT("movss       0x00(%[tail]), %%xmm0")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[head])")
                __ASM_EMIT("add         $0x04, %[head]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         1b")
                __ASM_EMIT("jmp         2000f")

                // Block operation
                __ASM_EMIT("2:")
                __ASM_EMIT("test        $0x0f, %[tail]")
                __ASM_EMIT("jnz         1001f")
                    REVERSE2_CORE("movaps")
                    __ASM_EMIT("jmp         3000f")
                __ASM_EMIT("1001:")
                    REVERSE2_CORE("movups")

                /* x1 blocks */
                __ASM_EMIT("3000:")
                __ASM_EMIT("and         $0x03, %[count]")
                __ASM_EMIT("jz          2000f")
                __ASM_EMIT("8:")
                __ASM_EMIT("sub         $0x04, %[tail]")
                __ASM_EMIT("movss       0x00(%[tail]), %%xmm0")
                __ASM_EMIT("movss       %%xmm0, 0x00(%[head])")
                __ASM_EMIT("add         $0x04, %[head]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         8b")

                __ASM_EMIT("2000:")

                : [head] "+r" (dst), [tail] "+r" (src), [count] "+r" (count)
                :
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        #undef REVERSE2_CORE
    } /* namespace sse */
} /* namespace lsp */

#endif /* PRIVATE_DSP_ARCH_X86_SSE_COPY_H_ */
