╭─────────────╮ │scroll to top│ ╰─────────────╯ ╭ process_headers.zig ──────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯
🬂🬂🬒 🬂🬰🬂 🬑🬂🬀 ╭master╮ master xeubie: 🬋 🬋 🬃🬇🬋 │files │ commits refs discuss issues patches new login 🬯🬭🬭 🬭🬰🬭 🬟🬭🬑 ╰──────╯ (15882) user
╭ find file ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╮ ╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │ │ │http│ ssh │ │ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╯ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯
.. crc/ loongarch/ spirv/ bsp.zig check_mingw.zig compiler_gdb_pretty_printers.py docgen.zig doctest.zig dump-cov.zig fetch_them_macos_headers.zig gen_loongarch_encoding.zig gen_macos_headers_c.zig gen_outline_atomics.zig gen_parser_oracle.zig gen_spirv_spec.zig gen_stubs.zig generate_JSONTestSuite.zig generate_linux_syscalls.zig incr-check.zig macos-headers.c migrate_langref.zig ╭─────────────────────────────────╮ │process_headers.zig │ ╰─────────────────────────────────╯ unskip_spirv_tests.zig update-linux-headers.zig update_clang_options.zig update_cpu_features.zig update_crc_catalog.zig update_freebsd_libc.zig update_glibc.zig update_mingw.zig update_netbsd_libc.zig update_openbsd_libc.zig
281 error.FileNotFound => continue :search, 282 error.AccessDenied => continue :search, 283 else => return err, 284 }; 285 defer dir.close(io); 286 287 var dir_it = dir.iterate(); 288 289 while (try dir_it.next(io)) |entry| { 290 const full_path = try Dir.path.join(arena, &[_][]const u8{ full_dir_name, entry.name }); 291 switch (entry.kind) { 292 .directory => try dir_stack.append(full_path), 293 .file, .sym_link => { 294 const rel_path = try Dir.path.relative(arena, cwd_path, environ_map, target_include_dir, full_path); 295 const max_size = 2 * 1024 * 1024 * 1024; 296 const raw_bytes = try Dir.cwd().readFileAlloc(io, full_path, arena, .limited(max_size)); 297 const trimmed = std.mem.trim(u8, raw_bytes, " \r\n\t"); 298 total_bytes += raw_bytes.len; 299 const hash = try arena.alloc(u8, 32); 300 hasher = Blake3.init(.{}); 301 hasher.update(rel_path); 302 hasher.update(trimmed); 303 hasher.final(hash); 304 const gop = try hash_to_contents.getOrPut(hash); 305 if (gop.found_existing) { 306 max_bytes_saved += raw_bytes.len; 307 gop.value_ptr.hit_count += 1; 308 std.debug.print("duplicate: {s} {s} ({B})\n", .{ 309 libc_dir, rel_path, raw_bytes.len, 310 }); 311 } else { 312 gop.value_ptr.* = Contents{ 313 .bytes = trimmed, 314 .hit_count = 1, 315 .hash = hash, 316 .is_generic = false, 317 }; 318 } 319 const path_gop = try path_table.getOrPut(rel_path); 320 const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: { 321 const ptr = try arena.create(TargetToHash); 322 ptr.* = .empty; 323 path_gop.value_ptr.* = ptr; 324 break :blk ptr; 325 }; 326 // When `dest` is set, there are a few rare cases where we expect to overwrite a header. For 327 // example, `bits/long-double.h` differs very slightly between `powerpc64le-linux-gnu` and 328 // other `powerpc*-linux-gnu` targets, and we unify those targets as `powerpc-linux-gnu`. In 329 // such cases, we manually patch the affected header after processing, so it's fine that 330 // only one header wins here. 331 if (libc_target.dest != null) { 332 const hash_gop = try target_to_hash.getOrPut(arena, dest_target); 333 if (hash_gop.found_existing) std.debug.print("overwrote: {s} {s} {s}\n", .{ 334 libc_dir, 335 rel_path, 336 dest_target, 337 }) else hash_gop.value_ptr.* = hash; 338 } else { 339 try target_to_hash.putNoClobber(arena, dest_target, hash); 340 } 341 }, 342 else => std.debug.print("warning: weird file: {s}\n", .{full_path}), 343 } 344 } 345 } 346 break; 347 } else { 348 std.debug.print("warning: libc target not found: {s}\n", .{libc_dir}); 349 } 350 } 351 std.debug.print("summary: {B} could be reduced to {B}\n", .{ 352 total_bytes, 353 total_bytes - max_bytes_saved, 354 }); 355 try Dir.cwd().createDirPath(io, out_dir); 356 357 var missed_opportunity_bytes: usize = 0; 358 // iterate path_table. for each path, put all the hashes into a list. sort by hit_count. 359 // the hash with the highest hit_count gets to be the "generic" one. everybody else 360 // gets their header in a separate arch directory. 361 var path_it = path_table.iterator(); 362 while (path_it.next()) |path_kv| { 363 var contents_list = std.array_list.Managed(*Contents).init(arena); 364 { 365 var hash_it = path_kv.value_ptr.*.iterator(); 366 while (hash_it.next()) |hash_kv| { 367 const contents = hash_to_contents.getPtr(hash_kv.value_ptr.*).?; 368 try contents_list.append(contents); 369 } 370 } 371 std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan); 372 const best_contents = contents_list.pop().?; 373 if (best_contents.hit_count > 1) { 374 // worth it to make it generic 375 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, generic_name, path_kv.key_ptr.* }); 376 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?); 377 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = best_contents.bytes }); 378 best_contents.is_generic = true; 379 while (contents_list.pop()) |contender| { 380 if (contender.hit_count > 1) { 381 const this_missed_bytes = contender.hit_count * contender.bytes.len; 382 missed_opportunity_bytes += this_missed_bytes; 383 std.debug.print("Missed opportunity ({B}): {s}\n", .{ 384 this_missed_bytes, 385 path_kv.key_ptr.*, 386 }); 387 } else break; 388 } 389 } 390 var hash_it = path_kv.value_ptr.*.iterator(); 391 while (hash_it.next()) |hash_kv| { 392 const contents = hash_to_contents.get(hash_kv.value_ptr.*).?; 393 if (contents.is_generic) continue; 394 395 const dest_target = hash_kv.key_ptr.*; 396 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, dest_target, path_kv.key_ptr.* }); 397 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?); 398 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = contents.bytes }); 399 } 400 } 401 } 402 403 fn usageAndExit(arg0: []const u8) noreturn { 404 std.debug.print("Usage: {s} [--search-path <dir>] --out <dir> --abi <name>\n", .{arg0}); 405 std.debug.print("--search-path can be used any number of times.\n", .{}); 406 std.debug.print(" subdirectories of search paths look like, e.g. x86_64-linux-gnu\n", .{}); 407 std.debug.print("--out is a dir that will be created, and populated with the results\n", .{}); 408 std.debug.print("--abi is either glibc, musl, freebsd, netbsd, or openbsd\n", .{}); 409 std.process.exit(1); 410 } 411