╭─────────────╮ │scroll to top│ ╰─────────────╯ ╭ process_headers.zig ──────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯
🬂🬂🬒 🬂🬰🬂 🬑🬂🬀 ╭master╮ master xeubie: 🬋 🬋 🬃🬇🬋 │files │ commits refs discuss issues patches new login 🬯🬭🬭 🬭🬰🬭 🬟🬭🬑 ╰──────╯ (15882) user
╭ find file ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╮ ╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │ │ │http│ ssh │ │ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╯ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯
.. crc/ loongarch/ spirv/ bsp.zig check_mingw.zig compiler_gdb_pretty_printers.py docgen.zig doctest.zig dump-cov.zig fetch_them_macos_headers.zig gen_loongarch_encoding.zig gen_macos_headers_c.zig gen_outline_atomics.zig gen_parser_oracle.zig gen_spirv_spec.zig gen_stubs.zig generate_JSONTestSuite.zig generate_linux_syscalls.zig incr-check.zig macos-headers.c migrate_langref.zig ╭─────────────────────────────────╮ │process_headers.zig │ ╰─────────────────────────────────╯ unskip_spirv_tests.zig update-linux-headers.zig update_clang_options.zig update_cpu_features.zig update_crc_catalog.zig update_freebsd_libc.zig update_glibc.zig update_mingw.zig update_netbsd_libc.zig update_openbsd_libc.zig
70 .{ .arch = .mips64, .abi = .musl }, 71 .{ .arch = .mips64, .abi = .muslabin32 }, 72 .{ .arch = .powerpc, .abi = .musl }, 73 .{ .arch = .powerpc64, .abi = .musl }, 74 .{ .arch = .riscv32, .abi = .musl }, 75 .{ .arch = .riscv64, .abi = .musl }, 76 .{ .arch = .s390x, .abi = .musl }, 77 .{ .arch = .x86, .abi = .musl }, 78 .{ .arch = .x86_64, .abi = .musl }, 79 .{ .arch = .x86_64, .abi = .muslx32 }, 80 }; 81 82 const freebsd_targets = [_]LibCTarget{ 83 .{ .arch = .arm, .abi = .eabihf }, 84 .{ .arch = .aarch64, .abi = .none }, 85 .{ .arch = .powerpc64, .abi = .none }, 86 .{ .arch = .riscv64, .abi = .none }, 87 .{ .arch = .x86, .abi = .none }, 88 .{ .arch = .x86_64, .abi = .none }, 89 }; 90 91 const netbsd_targets = [_]LibCTarget{ 92 .{ .arch = .arm, .abi = .eabi, .dest = "arm-netbsd-eabi" }, 93 .{ .arch = .arm, .abi = .eabihf, .dest = "arm-netbsd-eabi" }, 94 .{ .arch = .aarch64, .abi = .none }, 95 .{ .arch = .m68k, .abi = .none }, 96 .{ .arch = .mips, .abi = .eabi, .dest = "mips-netbsd-eabi" }, 97 .{ .arch = .mips, .abi = .eabihf, .dest = "mips-netbsd-eabi" }, 98 .{ .arch = .powerpc, .abi = .eabihf, .dest = "powerpc-netbsd-eabi" }, 99 .{ .arch = .riscv32, .abi = .none }, 100 .{ .arch = .riscv64, .abi = .none }, 101 .{ .arch = .sparc, .abi = .none }, 102 .{ .arch = .sparc64, .abi = .none }, 103 .{ .arch = .x86, .abi = .none }, 104 .{ .arch = .x86_64, .abi = .none }, 105 }; 106 107 const openbsd_targets = [_]LibCTarget{ 108 .{ .arch = .arm, .abi = .eabi }, 109 .{ .arch = .aarch64, .abi = .none }, 110 .{ .arch = .mips64, .abi = .none }, 111 .{ .arch = .mips64el, .abi = .none }, 112 .{ .arch = .powerpc, .abi = .eabihf }, 113 .{ .arch = .powerpc64, .abi = .none }, 114 .{ .arch = .riscv64, .abi = .none }, 115 .{ .arch = .sparc64, .abi = .none }, 116 .{ .arch = .x86, .abi = .none }, 117 .{ .arch = .x86_64, .abi = .none }, 118 }; 119 120 const Contents = struct { 121 bytes: []const u8, 122 hit_count: usize, 123 hash: []const u8, 124 is_generic: bool, 125 126 fn hitCountLessThan(context: void, lhs: *const Contents, rhs: *const Contents) bool { 127 _ = context; 128 return lhs.hit_count < rhs.hit_count; 129 } 130 }; 131 132 const HashToContents = std.StringHashMap(Contents); 133 const TargetToHash = std.array_hash_map.String([]const u8); 134 const PathTable = std.StringHashMap(*TargetToHash); 135 136 const LibCVendor = enum { 137 musl, 138 glibc, 139 freebsd, 140 netbsd, 141 openbsd, 142 }; 143 144 pub fn main(init: std.process.Init) !void { 145 const arena = init.arena.allocator(); 146 const io = init.io; 147 const args = try init.minimal.args.toSlice(arena); 148 const cwd_path = try std.process.currentPathAlloc(io, arena); 149 const environ_map = init.environ_map; 150 151 var search_paths = std.array_list.Managed([]const u8).init(arena); 152 var opt_out_dir: ?[]const u8 = null; 153 var opt_abi: ?[]const u8 = null; 154 155 var arg_i: usize = 1; 156 while (arg_i < args.len) : (arg_i += 1) { 157 if (std.mem.eql(u8, args[arg_i], "--help")) 158 usageAndExit(args[0]); 159 if (arg_i + 1 >= args.len) { 160 std.debug.print("expected argument after '{s}'\n", .{args[arg_i]}); 161 usageAndExit(args[0]); 162 } 163 164 if (std.mem.eql(u8, args[arg_i], "--search-path")) { 165 try search_paths.append(args[arg_i + 1]); 166 } else if (std.mem.eql(u8, args[arg_i], "--out")) { 167 assert(opt_out_dir == null); 168 opt_out_dir = args[arg_i + 1]; 169 } else if (std.mem.eql(u8, args[arg_i], "--abi")) { 170 assert(opt_abi == null); 171 opt_abi = args[arg_i + 1]; 172 } else { 173 std.debug.print("unrecognized argument: {s}\n", .{args[arg_i]}); 174 usageAndExit(args[0]); 175 } 176 177 arg_i += 1; 178 } 179 180 const out_dir = opt_out_dir orelse usageAndExit(args[0]); 181 const abi_name = opt_abi orelse usageAndExit(args[0]); 182 const vendor = std.meta.stringToEnum(LibCVendor, abi_name) orelse { 183 std.debug.print("unrecognized C ABI: {s}\n", .{abi_name}); 184 usageAndExit(args[0]); 185 }; 186 187 const generic_name = try std.fmt.allocPrint(arena, "generic-{s}", .{abi_name}); 188 const libc_targets = switch (vendor) { 189 .glibc => &glibc_targets, 190 .musl => &musl_targets, 191 .freebsd => &freebsd_targets, 192 .netbsd => &netbsd_targets, 193 .openbsd => &openbsd_targets, 194 }; 195 196 var path_table = PathTable.init(arena); 197 var hash_to_contents = HashToContents.init(arena); 198 var max_bytes_saved: usize = 0; 199 var total_bytes: usize = 0; 200 201 var hasher = Blake3.init(.{}); 202 203 for (libc_targets) |libc_target| { 204 const libc_dir = switch (vendor) { 205 .glibc => try std.zig.target.glibcRuntimeTriple(arena, libc_target.arch, .linux, libc_target.abi), 206 .musl => std.zig.target.muslArchName(libc_target.arch, libc_target.abi), 207 .freebsd => switch (libc_target.arch) { 208 .arm => "armv7", 209 .x86 => "i386", 210 .x86_64 => "amd64", 211 212 .aarch64, 213 .powerpc, 214 .powerpc64, 215 .riscv64, 216 => |a| @tagName(a), 217 218 else => unreachable, 219 }, 220 .netbsd => switch (libc_target.arch) { 221 .arm => if (libc_target.abi == .eabihf) "evbarmv7hf" else "evbarmv7", 222 .aarch64 => "evbarm64", 223 .m68k => "mac68k", 224 .mips => if (libc_target.abi == .eabihf) "evbmips" else "evbmipssf", 225 .powerpc => if (libc_target.abi == .eabihf) "evbppc" else "evbppcsf", 226 .x86 => "i386", 227 .x86_64 => "amd64", 228 229 .riscv32, 230 .riscv64, 231 .sparc, 232 .sparc64, 233 => |a| @tagName(a), 234 235 else => unreachable, 236 }, 237 .openbsd => switch (libc_target.arch) { 238 .arm => "armv7", 239 .aarch64 => "arm64", 240 .mips64 => "octeon", 241 .mips64el => "loongson", 242 .powerpc => "macppc", 243 .x86 => "i386", 244 .x86_64 => "amd64", 245 246 .powerpc64, 247 .riscv64, 248 .sparc64, 249 => |a| @tagName(a), 250 251 else => unreachable, 252 }, 253 }; 254 255 const dest_target = if (libc_target.dest) |dest| dest else try std.fmt.allocPrint(arena, "{s}-{s}-{s}", .{ 256 @tagName(libc_target.arch), 257 switch (vendor) { 258 .musl, .glibc => "linux", 259 .freebsd => "freebsd", 260 .netbsd => "netbsd", 261 .openbsd => "openbsd", 262 }, 263 @tagName(libc_target.abi), 264 }); 265 266 search: for (search_paths.items) |search_path| { 267 const sub_path = switch (vendor) { 268 .glibc, 269 .freebsd, 270 .netbsd, 271 .openbsd, 272 => &[_][]const u8{ search_path, libc_dir, "usr", "include" }, 273 .musl => &[_][]const u8{ search_path, libc_dir, "usr", "local", "musl", "include" }, 274 }; 275 const target_include_dir = try Dir.path.join(arena, sub_path); 276 var dir_stack = std.array_list.Managed([]const u8).init(arena); 277 try dir_stack.append(target_include_dir); 278 279 while (dir_stack.pop()) |full_dir_name| { 280 var dir = Dir.cwd().openDir(io, full_dir_name, .{ .iterate = true }) catch |err| switch (err) { 281 error.FileNotFound => continue :search, 282 error.AccessDenied => continue :search, 283 else => return err, 284 }; 285 defer dir.close(io); 286 287 var dir_it = dir.iterate(); 288 289 while (try dir_it.next(io)) |entry| { 290 const full_path = try Dir.path.join(arena, &[_][]const u8{ full_dir_name, entry.name }); 291 switch (entry.kind) { 292 .directory => try dir_stack.append(full_path), 293 .file, .sym_link => { 294 const rel_path = try Dir.path.relative(arena, cwd_path, environ_map, target_include_dir, full_path); 295 const max_size = 2 * 1024 * 1024 * 1024; 296 const raw_bytes = try Dir.cwd().readFileAlloc(io, full_path, arena, .limited(max_size)); 297 const trimmed = std.mem.trim(u8, raw_bytes, " \r\n\t"); 298 total_bytes += raw_bytes.len; 299 const hash = try arena.alloc(u8, 32); 300 hasher = Blake3.init(.{}); 301 hasher.update(rel_path); 302 hasher.update(trimmed); 303 hasher.final(hash); 304 const gop = try hash_to_contents.getOrPut(hash); 305 if (gop.found_existing) { 306 max_bytes_saved += raw_bytes.len; 307 gop.value_ptr.hit_count += 1; 308 std.debug.print("duplicate: {s} {s} ({B})\n", .{ 309 libc_dir, rel_path, raw_bytes.len, 310 }); 311 } else { 312 gop.value_ptr.* = Contents{ 313 .bytes = trimmed, 314 .hit_count = 1, 315 .hash = hash, 316 .is_generic = false, 317 }; 318 } 319 const path_gop = try path_table.getOrPut(rel_path); 320 const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: { 321 const ptr = try arena.create(TargetToHash); 322 ptr.* = .empty; 323 path_gop.value_ptr.* = ptr; 324 break :blk ptr; 325 }; 326 // When `dest` is set, there are a few rare cases where we expect to overwrite a header. For 327 // example, `bits/long-double.h` differs very slightly between `powerpc64le-linux-gnu` and 328 // other `powerpc*-linux-gnu` targets, and we unify those targets as `powerpc-linux-gnu`. In 329 // such cases, we manually patch the affected header after processing, so it's fine that 330 // only one header wins here. 331 if (libc_target.dest != null) { 332 const hash_gop = try target_to_hash.getOrPut(arena, dest_target); 333 if (hash_gop.found_existing) std.debug.print("overwrote: {s} {s} {s}\n", .{ 334 libc_dir, 335 rel_path, 336 dest_target, 337 }) else hash_gop.value_ptr.* = hash; 338 } else { 339 try target_to_hash.putNoClobber(arena, dest_target, hash); 340 } 341 }, 342 else => std.debug.print("warning: weird file: {s}\n", .{full_path}), 343 } 344 } 345 } 346 break; 347 } else { 348 std.debug.print("warning: libc target not found: {s}\n", .{libc_dir}); 349 } 350 } 351 std.debug.print("summary: {B} could be reduced to {B}\n", .{ 352 total_bytes, 353 total_bytes - max_bytes_saved, 354 }); 355 try Dir.cwd().createDirPath(io, out_dir); 356 357 var missed_opportunity_bytes: usize = 0; 358 // iterate path_table. for each path, put all the hashes into a list. sort by hit_count. 359 // the hash with the highest hit_count gets to be the "generic" one. everybody else 360 // gets their header in a separate arch directory. 361 var path_it = path_table.iterator(); 362 while (path_it.next()) |path_kv| { 363 var contents_list = std.array_list.Managed(*Contents).init(arena); 364 { 365 var hash_it = path_kv.value_ptr.*.iterator(); 366 while (hash_it.next()) |hash_kv| { 367 const contents = hash_to_contents.getPtr(hash_kv.value_ptr.*).?; 368 try contents_list.append(contents); 369 } 370 } 371 std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan); 372 const best_contents = contents_list.pop().?; 373 if (best_contents.hit_count > 1) { 374 // worth it to make it generic 375 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, generic_name, path_kv.key_ptr.* }); 376 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?); 377 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = best_contents.bytes }); 378 best_contents.is_generic = true; 379 while (contents_list.pop()) |contender| { 380 if (contender.hit_count > 1) { 381 const this_missed_bytes = contender.hit_count * contender.bytes.len; 382 missed_opportunity_bytes += this_missed_bytes; 383 std.debug.print("Missed opportunity ({B}): {s}\n", .{ 384 this_missed_bytes, 385 path_kv.key_ptr.*, 386 }); 387 } else break; 388 } 389 } 390 var hash_it = path_kv.value_ptr.*.iterator(); 391 while (hash_it.next()) |hash_kv| { 392 const contents = hash_to_contents.get(hash_kv.value_ptr.*).?; 393 if (contents.is_generic) continue; 394 395 const dest_target = hash_kv.key_ptr.*; 396 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, dest_target, path_kv.key_ptr.* }); 397 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?); 398 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = contents.bytes }); 399 } 400 } 401 } 402 403 fn usageAndExit(arg0: []const u8) noreturn { 404 std.debug.print("Usage: {s} [--search-path <dir>] --out <dir> --abi <name>\n", .{arg0}); 405 std.debug.print("--search-path can be used any number of times.\n", .{}); 406 std.debug.print(" subdirectories of search paths look like, e.g. x86_64-linux-gnu\n", .{}); 407 std.debug.print("--out is a dir that will be created, and populated with the results\n", .{}); 408 std.debug.print("--abi is either glibc, musl, freebsd, netbsd, or openbsd\n", .{}); 409 std.process.exit(1); 410 } 411