Add x86_64 asm versions of blendpixel and filldata

Author: ollydev

Source/asm/blendpixel_x86_64.inc

@@ -0,0 +1,40 @@
+// x86-64 SSE alpha blend of one pixel: dst = (dst*(255-a) + src*a) / 255 per channel, with
+// the /255 as the exact (t*32897) shr 23 (pmulhuw + psrlw). Where dst.A = 0 the source is
+// copied, matching the scalar path. `const Color` is passed by reference. Bit-identical to
+// the scalar blend for opaque or fully-transparent destinations.
+assembler; nostackframe;
+asm
+{$IFDEF UNIX}
+  mov    rcx, rdi                 // Pixel
+  mov    rdx, rsi                 // @Color
+{$ENDIF}
+  movzx  eax, byte ptr [rcx+3]    // dst.A
+  test   eax, eax
+  jz     @copy
+  movd   xmm0, [rcx]              // dst pixel
+  movd   xmm1, [rdx]              // src color
+  pxor   xmm2, xmm2
+  punpcklbw xmm0, xmm2            // dst -> 4 words
+  punpcklbw xmm1, xmm2            // src -> 4 words
+  pshuflw xmm3, xmm1, $FF         // a = src.A, broadcast to 4 words
+  mov    eax, 255
+  pinsrw xmm1, eax, 3             // src.A lane -> 255, so the A addend is 255*a
+  pcmpeqw xmm4, xmm4
+  psrlw  xmm4, 8                  // 255 in each word
+  psubw  xmm4, xmm3               // ia = 255 - a
+  pmullw xmm0, xmm4               // dst * ia
+  pmullw xmm1, xmm3               // addend: src*a  (A lane = 255*a)
+  paddw  xmm0, xmm1
+  mov    eax, 32897
+  movd   xmm5, eax
+  pshuflw xmm5, xmm5, 0           // div-255 magic, broadcast
+  pmulhuw xmm0, xmm5              // (t * 32897) >> 16
+  psrlw  xmm0, 7                  // >> 7  =>  t / 255
+  packuswb xmm0, xmm0
+  movd   [rcx], xmm0
+  jmp    @done
+@copy:
+  mov    eax, [rdx]
+  mov    [rcx], eax
+@done:
+end;

Source/asm/distancehsl_x64_86.inc Source/asm/distancehsl_x86_64.incSource/asm/distancehsl_x64_86.inc renamed to Source/asm/distancehsl_x86_64.inc

@@ -0,0 +1,77 @@
+// x86-64 fill for a row of pixels (pure asm). Small/medium fills use cached movdqu stores
+// (fast, and keeps the pixels in cache for the draw that usually follows); fills of 4 MB or
+// more use aligned non-temporal (movntdq) stores - ~3x faster for the large image-clear case.
+// `const Value` is passed by reference. All registers touched are volatile, hence nostackframe.
+assembler; nostackframe;
+asm
+{$IFDEF UNIX}
+  mov    rcx, rdi                 // Data
+  mov    r8,  rdx                 // @Value
+  mov    rdx, rsi                 // Count
+{$ENDIF}
+  mov    r9d, [r8]                // Value (const record, passed by reference)
+  test   rdx, rdx
+  jle    @done                    // Count <= 0
+  movd   xmm0, r9d
+  pshufd xmm0, xmm0, 0            // broadcast the pixel to all 4 lanes
+  mov    rax, rcx                 // rax = running destination
+  mov    rcx, rdx                 // rcx = pixel count
+  cmp    rcx, 1048576             // >= 4 MB (1M pixels) -> non-temporal
+  jae    @large
+
+  // small / medium: cached stores, 8 px (32 bytes) per iteration - two store ports keep up
+  mov    edx, ecx
+  and    rcx, -8                   // whole blocks of 8 pixels
+  jz     @stail
+@sloop:
+  movdqu [rax], xmm0
+  movdqu [rax+16], xmm0
+  add    rax, 32
+  sub    rcx, 8
+  jnz    @sloop
+@stail:
+  and    edx, 7                    // 0..7 leftover pixels
+  jz     @done
+  cmp    edx, 4
+  jb     @stloop                   // fewer than 4 -> scalar tail
+  movdqu [rax], xmm0               // one 4-pixel block
+  add    rax, 16
+  sub    edx, 4
+  jz     @done
+@stloop:
+  mov    [rax], r9d
+  add    rax, 4
+  dec    edx
+  jnz    @stloop
+  jmp    @done
+
+  // large: align to 16 bytes, then non-temporal
+@large:
+@align:
+  test   rax, 15
+  jz     @lbulk
+  mov    [rax], r9d
+  add    rax, 4
+  dec    rcx
+  jmp    @align
+@lbulk:
+  mov    edx, ecx
+  and    rcx, -4
+  jz     @ltail
+@lloop:
+  movntdq [rax], xmm0
+  add    rax, 16
+  sub    rcx, 4
+  jnz    @lloop
+@ltail:
+  and    edx, 3
+  jz     @lfence
+@ltloop:
+  mov    [rax], r9d
+  add    rax, 4
+  dec    edx
+  jnz    @ltloop
+@lfence:
+  sfence
+@done:
+end;

Source/asm/distancehsv_x64_86.inc Source/asm/distancehsv_x86_64.incSource/asm/distancehsv_x64_86.inc renamed to Source/asm/distancehsv_x86_64.inc

@@ -72,8 +72,8 @@ function SynLZDecompressedSize(Src: PByte): Int32;
 end;
 
 {$IF DEFINED(SYNLZ_ASM)}
-  {$I ../asm/synlzcompress_x64.inc}
-  {$I ../asm/synlzdecompress_x64.inc}
+  {$I ../asm/synlzcompress_x86_64.inc}
+  {$I ../asm/synlzdecompress_x86_64.inc}
 {$ENDIF}
 
 function SynLZCompressPas(Src: PByte; Size: Int32; Dst: PByte): Int32;

Source/asm/distancergb_x64_86.inc Source/asm/distancergb_x86_64.incSource/asm/distancergb_x64_86.inc renamed to Source/asm/distancergb_x86_64.inc

@@ -31,17 +31,17 @@ function DistanceDeltaE_UnRolled(const C1: PColorLAB; const C2: PColorBGRA; cons
 implementation
 
 function DistanceRGB_UnRolled(const C1: PColorRGB; const C2: PColorBGRA; const mul: TChannelMultipliers): Single;
-{$IF DEFINED(COLORDIST_ASM) and DEFINED(CPUX86_64)}
-  {$I asm/distancergb_x64_86.inc}
+{$IF DEFINED(COLORDIST_ASM)}
+  {$I asm/distancergb_x86_64.inc}
 {$ELSE}
 begin
   Result := Sqrt(Sqr((C1^.R - C2^.R) * mul[0]) + Sqr((C1^.G - C2^.G) * mul[1]) + Sqr((C1^.B - C2^.B) * mul[2]));
 end;
 {$ENDIF}
 
 function DistanceHSL_UnRolled(const C1: PColorHSL; const C2: PColorBGRA; const mul: TChannelMultipliers): Single;
-{$IF DEFINED(COLORDIST_ASM) and DEFINED(CPUX86_64)}
-  {$I asm/distancehsl_x64_86.inc}
+{$IF DEFINED(COLORDIST_ASM)}
+  {$I asm/distancehsl_x86_64.inc}
 {$ELSE}
 var
   R,G,B,deltaC,deltaH,cMax,cMin, H,S,L: Single;
@@ -88,8 +88,8 @@ function DistanceHSL_UnRolled(const C1: PColorHSL; const C2: PColorBGRA; const m
 {$ENDIF}
 
 function DistanceHSV_UnRolled(const C1: PColorHSV; const C2: PColorBGRA; const mul: TChannelMultipliers): Single;
-{$IF DEFINED(COLORDIST_ASM) and DEFINED(CPUX86_64)}
-  {$I asm/distancehsv_x64_86.inc}
+{$IF DEFINED(COLORDIST_ASM)}
+  {$I asm/distancehsv_x86_64.inc}
 {$ELSE}
 var
   R, G, B: Single;