HAXY

▄▀▄ ▄▀▀▀▀▄ ▀ ▄▄▀▀▀▀▀▀▄ ▀ ▀ ▀▀▀▀▀▀▀▄▀▄ ▄▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀ ▄▀▄ ▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▄ ▀▀▀ ▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀ ▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▄▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▄▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀ ▄▀▀▀▀▀▀▀▄▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▄▄▄▄▄▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▄▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀ ▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▄▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀ ▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▄▀▀▀ ▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▄▀▀ ▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▄▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀ ▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▄▀▀ ▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▄ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▄▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀ ▄▄▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▄

╭ changed ────────────────╮ │lib/std/crypto/argon2.zig│ ╰─────────────────────────╯ ╭─────────────────────────────────────────╮ │ 8 const blake2 = crypto.hash.blake2; │ │ 9 const crypto = std.crypto; │ │10 const Io = std.Io; │ │11 -const math = std.math; │ │11 const mem = std.mem; │ │12 const phc_format = pwhash.phc_format;│ │13 const pwhash = crypto.pwhash; │ ╰─────────────────────────────────────────╯ ╭─────────────────────────────────────────────────────────────────────────────────────────╮ │175 H.hash(&in_buf, &out_buf, .{ .expected_out_bits = out_slice.len * 8 }); │ │176 @memcpy(out_slice, out_buf[0..out_slice.len]); │ │177 } │ │179 - │ │180 -fn initBlocks( │ │181 - blocks: *Blocks, │ │182 - h0: *H0, │ │183 - memory: u32, │ │184 - threads: u24, │ │185 -) void { │ │186 - var block0: [1024]u8 = undefined; │ │187 - var lane: u24 = 0; │ │188 - while (lane < threads) : (lane += 1) { │ │189 - const j = lane * (memory / threads); │ │190 - mem.writeInt(u32, h0[Blake2b512.digest_length + 4 ..][0..4], lane, .little);│ │191 - │ │192 - mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 0, .little); │ │193 - blake2bLong(&block0, h0); │ │194 - for (&blocks.items[j + 0], 0..) |*v, i| { │ │195 - v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); │ │196 - } │ │178 │ │198 - mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 1, .little); │ │199 - blake2bLong(&block0, h0); │ │200 - for (&blocks.items[j + 1], 0..) |*v, i| { │ │201 - v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); │ │202 - } │ │203 - } │ │204 -} │ │205 - │ │179 fn processBlocks( │ │180 blocks: *Blocks, │ │181 time: u32, │ ╰─────────────────────────────────────────────────────────────────────────────────────────╯ ╭────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │295 random = blocks.items[prev][0]; │ │296 } │ │297 const new_offset = indexAlpha(random, lanes, segments, threads, n, slice, lane, index); │ │325 - processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]); │ │298 + if (n == 0) { │ │299 + processBlock(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]); │ │300 + } else { │ │301 + processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);│ │302 + } │ │303 } │ │304 } │ │305 │ ╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭────────────────────────────────────────────────────────────────╮ │331 } │ │332 var i: usize = 0; │ │333 while (i < block_length) : (i += 16) { │ │357 - blamkaGeneric(t[i..][0..16]); │ │334 + blamkaGeneric(.{ │ │335 + &t[i + 0], &t[i + 1], &t[i + 2], &t[i + 3], │ │336 + &t[i + 4], &t[i + 5], &t[i + 6], &t[i + 7], │ │337 + &t[i + 8], &t[i + 9], &t[i + 10], &t[i + 11],│ │338 + &t[i + 12], &t[i + 13], &t[i + 14], &t[i + 15],│ │339 + }); │ │340 } │ │341 i = 0; │ │360 - var buffer: [16]u64 = undefined; │ │342 while (i < block_length / 8) : (i += 2) { │ │362 - var j: usize = 0; │ │363 - while (j < block_length / 8) : (j += 2) { │ │364 - buffer[j] = t[j * 8 + i]; │ │365 - buffer[j + 1] = t[j * 8 + i + 1]; │ │366 - } │ │367 - blamkaGeneric(&buffer); │ │368 - j = 0; │ │369 - while (j < block_length / 8) : (j += 2) { │ │370 - t[j * 8 + i] = buffer[j]; │ │371 - t[j * 8 + i + 1] = buffer[j + 1]; │ │372 - } │ │343 + blamkaGeneric(.{ │ │344 + &t[0 * 16 + i], &t[0 * 16 + i + 1], │ │345 + &t[1 * 16 + i], &t[1 * 16 + i + 1], │ │346 + &t[2 * 16 + i], &t[2 * 16 + i + 1], │ │347 + &t[3 * 16 + i], &t[3 * 16 + i + 1], │ │348 + &t[4 * 16 + i], &t[4 * 16 + i + 1], │ │349 + &t[5 * 16 + i], &t[5 * 16 + i + 1], │ │350 + &t[6 * 16 + i], &t[6 * 16 + i + 1], │ │351 + &t[7 * 16 + i], &t[7 * 16 + i + 1], │ │352 + }); │ │353 } │ │354 if (xor) { │ │355 for (t, 0..) |v, j| { │ ╰────────────────────────────────────────────────────────────────╯ ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │361 } │ │362 } │ │363 } │ │364 + │ │365 +const BlamkaVector = @Vector(4, u64); │ │366 │ │385 -const QuarterRound = struct { a: usize, b: usize, c: usize, d: usize }; │ │367 +fn fBlaMka(x: BlamkaVector, y: BlamkaVector) BlamkaVector { │ │368 + const x_lo: @Vector(4, u32) = @truncate(x); │ │369 + const y_lo: @Vector(4, u32) = @truncate(y); │ │370 + const xy = @as(BlamkaVector, x_lo) * @as(BlamkaVector, y_lo); │ │371 + return x +% y +% @as(BlamkaVector, @splat(2)) *% xy; │ │372 +} │ │373 │ │387 -fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound { │ │388 - return .{ .a = a, .b = b, .c = c, .d = d }; │ │374 +fn rotrVector(x: BlamkaVector, comptime n: comptime_int) BlamkaVector { │ │375 + return (x >> @splat(n)) | (x << @splat(64 - n)); │ │376 } │ │377 │ │391 -fn fBlaMka(x: u64, y: u64) u64 { │ │392 - const xy = @as(u64, @as(u32, @truncate(x))) * @as(u64, @as(u32, @truncate(y))); │ │393 - return x +% y +% 2 *% xy; │ │378 +// The `inline` is load-bearing: without it ReleaseSmall keeps this out of line │ │379 +// and shuffles the four vectors through memory across every call, costing │ │380 +// roughly 9% on Apple Silicon. │ │381 +inline fn blamkaRound(a: *BlamkaVector, b: *BlamkaVector, c: *BlamkaVector, d: *BlamkaVector) void {│ │382 + a.* = fBlaMka(a.*, b.*); │ │383 + d.* = rotrVector(d.* ^ a.*, 32); │ │384 + c.* = fBlaMka(c.*, d.*); │ │385 + b.* = rotrVector(b.* ^ c.*, 24); │ │386 + a.* = fBlaMka(a.*, b.*); │ │387 + d.* = rotrVector(d.* ^ a.*, 16); │ │388 + c.* = fBlaMka(c.*, d.*); │ │389 + b.* = rotrVector(b.* ^ c.*, 63); │ │390 } │ │391 │ │396 -fn blamkaGeneric(x: *[16]u64) void { │ │397 - const rounds = comptime [_]QuarterRound{ │ │398 - Rp(0, 4, 8, 12), │ │399 - Rp(1, 5, 9, 13), │ │400 - Rp(2, 6, 10, 14), │ │401 - Rp(3, 7, 11, 15), │ │402 - Rp(0, 5, 10, 15), │ │403 - Rp(1, 6, 11, 12), │ │404 - Rp(2, 7, 8, 13), │ │405 - Rp(3, 4, 9, 14), │ │406 - }; │ │407 - inline for (rounds) |r| { │ │408 - x[r.a] = fBlaMka(x[r.a], x[r.b]); │ │409 - x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 32); │ │410 - x[r.c] = fBlaMka(x[r.c], x[r.d]); │ │411 - x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 24); │ │412 - x[r.a] = fBlaMka(x[r.a], x[r.b]); │ │413 - x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 16); │ │414 - x[r.c] = fBlaMka(x[r.c], x[r.d]); │ │415 - x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 63); │ │416 - } │ │392 +fn blamkaGeneric(xs: [16]*u64) void { │ │393 + var a: BlamkaVector = .{ xs[0].*, xs[1].*, xs[2].*, xs[3].* }; │ │394 + var b: BlamkaVector = .{ xs[4].*, xs[5].*, xs[6].*, xs[7].* }; │ │395 + var c: BlamkaVector = .{ xs[8].*, xs[9].*, xs[10].*, xs[11].* }; │ │396 + var d: BlamkaVector = .{ xs[12].*, xs[13].*, xs[14].*, xs[15].* }; │ │397 + │ │398 + blamkaRound(&a, &b, &c, &d); │ │399 + │ │400 + var b_diag = @shuffle(u64, b, undefined, [_]i32{ 1, 2, 3, 0 }); │ │401 + var c_diag = @shuffle(u64, c, undefined, [_]i32{ 2, 3, 0, 1 }); │ │402 + var d_diag = @shuffle(u64, d, undefined, [_]i32{ 3, 0, 1, 2 }); │ │403 + blamkaRound(&a, &b_diag, &c_diag, &d_diag); │ │404 + b = @shuffle(u64, b_diag, undefined, [_]i32{ 3, 0, 1, 2 }); │ │405 + c = @shuffle(u64, c_diag, undefined, [_]i32{ 2, 3, 0, 1 }); │ │406 + d = @shuffle(u64, d_diag, undefined, [_]i32{ 1, 2, 3, 0 }); │ │407 + │ │408 + inline for (0..4) |k| xs[k].* = a[k]; │ │409 + inline for (0..4) |k| xs[4 + k].* = b[k]; │ │410 + inline for (0..4) |k| xs[8 + k].* = c[k]; │ │411 + inline for (0..4) |k| xs[12 + k].* = d[k]; │ │412 } │ │413 │ │414 fn finalize( │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─────────────────────────────────────────────────────────────────────────────────────────╮ │506 var blocks = try Blocks.initCapacity(allocator, memory); │ │507 defer blocks.deinit(); │ │508 │ │514 - blocks.appendNTimesAssumeCapacity(@splat(0), memory); │ │509 + const items = blocks.addManyAsSliceAssumeCapacity(memory); │ │510 │ │516 - initBlocks(&blocks, &h0, memory, params.p); │ │511 + var block0: [1024]u8 = undefined; │ │512 + var lane: u24 = 0; │ │513 + while (lane < params.p) : (lane += 1) { │ │514 + const j = lane * (memory / params.p); │ │515 + mem.writeInt(u32, h0[Blake2b512.digest_length + 4 ..][0..4], lane, .little);│ │516 + │ │517 + mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 0, .little); │ │518 + blake2bLong(&block0, &h0); │ │519 + for (&items[j + 0], 0..) |*v, i| { │ │520 + v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); │ │521 + } │ │522 + │ │523 + mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 1, .little); │ │524 + blake2bLong(&block0, &h0); │ │525 + for (&items[j + 1], 0..) |*v, i| { │ │526 + v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); │ │527 + } │ │528 + } │ │529 + │ │530 try processBlocks(&blocks, params.t, memory, params.p, mode, io); │ │531 finalize(&blocks, memory, params.p, derived_key); │ │532 } │ ╰─────────────────────────────────────────────────────────────────────────────────────────╯