╭─────────────╮ │scroll to top│ ╰─────────────╯ ╭ update-linux-headers.zig ─────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯
🬂🬂🬒 🬂🬰🬂 🬑🬂🬀 ╭master╮ master xeubie: 🬋 🬋 🬃🬇🬋 │files │ commits refs discuss issues patches new login 🬯🬭🬭 🬭🬰🬭 🬟🬭🬑 ╰──────╯ (15882) user
╭ find file ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╮ ╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │ │ │http│ ssh │ │ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╯ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯
.. crc/ loongarch/ spirv/ bsp.zig check_mingw.zig compiler_gdb_pretty_printers.py docgen.zig doctest.zig dump-cov.zig fetch_them_macos_headers.zig gen_loongarch_encoding.zig gen_macos_headers_c.zig gen_outline_atomics.zig gen_parser_oracle.zig gen_spirv_spec.zig gen_stubs.zig generate_JSONTestSuite.zig generate_linux_syscalls.zig incr-check.zig macos-headers.c migrate_langref.zig process_headers.zig unskip_spirv_tests.zig ╭─────────────────────────────────╮ │update-linux-headers.zig │ ╰─────────────────────────────────╯ update_clang_options.zig update_cpu_features.zig update_crc_catalog.zig update_freebsd_libc.zig update_glibc.zig update_mingw.zig update_netbsd_libc.zig update_openbsd_libc.zig
87 LibCTarget{ 88 .name = "riscv", 89 .arch = .riscv, 90 }, 91 LibCTarget{ 92 .name = "s390", 93 .arch = .{ .specific = .s390x }, 94 }, 95 LibCTarget{ 96 .name = "sparc", 97 .arch = .{ .specific = .sparc }, 98 }, 99 LibCTarget{ 100 .name = "x86", 101 .arch = .x86, 102 }, 103 LibCTarget{ 104 .name = "xtensa", 105 .arch = .{ .specific = .xtensa }, 106 }, 107 }; 108 109 const DestTarget = struct { 110 arch: MultiArch, 111 112 const HashContext = struct { 113 pub fn hash(self: @This(), a: DestTarget) u32 { 114 _ = self; 115 var hasher = std.hash.Wyhash.init(0); 116 std.hash.autoHash(&hasher, a.arch); 117 return @as(u32, @truncate(hasher.final())); 118 } 119 120 pub fn eql(self: @This(), a: DestTarget, b: DestTarget, b_index: usize) bool { 121 _ = self; 122 _ = b_index; 123 return a.arch.eql(b.arch); 124 } 125 }; 126 }; 127 128 const Contents = struct { 129 bytes: []const u8, 130 hit_count: usize, 131 hash: []const u8, 132 is_generic: bool, 133 134 fn hitCountLessThan(context: void, lhs: *const Contents, rhs: *const Contents) bool { 135 _ = context; 136 return lhs.hit_count < rhs.hit_count; 137 } 138 }; 139 140 const HashToContents = std.StringHashMap(Contents); 141 const TargetToHash = std.array_hash_map.Custom(DestTarget, []const u8, DestTarget.HashContext, true); 142 const PathTable = std.StringHashMap(*TargetToHash); 143 144 pub fn main(init: std.process.Init) !void { 145 const arena = init.arena.allocator(); 146 const io = init.io; 147 const args = try init.minimal.args.toSlice(arena); 148 const environ_map = init.environ_map; 149 const cwd = try std.process.currentPathAlloc(io, arena); 150 151 var search_paths = std.array_list.Managed([]const u8).init(arena); 152 var opt_out_dir: ?[]const u8 = null; 153 154 var arg_i: usize = 1; 155 while (arg_i < args.len) : (arg_i += 1) { 156 if (std.mem.eql(u8, args[arg_i], "--help")) 157 usageAndExit(args[0]); 158 if (arg_i + 1 >= args.len) { 159 std.debug.print("expected argument after '{s}'\n", .{args[arg_i]}); 160 usageAndExit(args[0]); 161 } 162 163 if (std.mem.eql(u8, args[arg_i], "--search-path")) { 164 try search_paths.append(args[arg_i + 1]); 165 } else if (std.mem.eql(u8, args[arg_i], "--out")) { 166 assert(opt_out_dir == null); 167 opt_out_dir = args[arg_i + 1]; 168 } else { 169 std.debug.print("unrecognized argument: {s}\n", .{args[arg_i]}); 170 usageAndExit(args[0]); 171 } 172 173 arg_i += 1; 174 } 175 176 const out_dir = opt_out_dir orelse usageAndExit(args[0]); 177 const generic_name = "any-linux-any"; 178 179 var path_table = PathTable.init(arena); 180 var hash_to_contents = HashToContents.init(arena); 181 var max_bytes_saved: usize = 0; 182 var total_bytes: usize = 0; 183 184 var hasher = Blake3.init(.{}); 185 186 for (linux_targets) |linux_target| { 187 const dest_target = DestTarget{ 188 .arch = linux_target.arch, 189 }; 190 search: for (search_paths.items) |search_path| { 191 const target_include_dir = try Dir.path.join(arena, &.{ 192 search_path, linux_target.name, "include", 193 }); 194 var dir_stack = std.array_list.Managed([]const u8).init(arena); 195 try dir_stack.append(target_include_dir); 196 197 while (dir_stack.pop()) |full_dir_name| { 198 var dir = Dir.cwd().openDir(io, full_dir_name, .{ .iterate = true }) catch |err| switch (err) { 199 error.FileNotFound => continue :search, 200 error.AccessDenied => continue :search, 201 else => return err, 202 }; 203 defer dir.close(io); 204 205 var dir_it = dir.iterate(); 206 207 while (try dir_it.next(io)) |entry| { 208 const full_path = try Dir.path.join(arena, &[_][]const u8{ full_dir_name, entry.name }); 209 switch (entry.kind) { 210 .directory => try dir_stack.append(full_path), 211 .file => { 212 const rel_path = try Dir.path.relative(arena, cwd, environ_map, target_include_dir, full_path); 213 const max_size = 2 * 1024 * 1024 * 1024; 214 const raw_bytes = try Dir.cwd().readFileAlloc(io, full_path, arena, .limited(max_size)); 215 const trimmed = std.mem.trim(u8, raw_bytes, " \r\n\t"); 216 total_bytes += raw_bytes.len; 217 const hash = try arena.alloc(u8, 32); 218 hasher = Blake3.init(.{}); 219 hasher.update(rel_path); 220 hasher.update(trimmed); 221 hasher.final(hash); 222 const gop = try hash_to_contents.getOrPut(hash); 223 if (gop.found_existing) { 224 max_bytes_saved += raw_bytes.len; 225 gop.value_ptr.hit_count += 1; 226 std.debug.print("duplicate: {s} {s} ({B})\n", .{ 227 linux_target.name, 228 rel_path, 229 raw_bytes.len, 230 }); 231 } else { 232 gop.value_ptr.* = Contents{ 233 .bytes = trimmed, 234 .hit_count = 1, 235 .hash = hash, 236 .is_generic = false, 237 }; 238 } 239 const path_gop = try path_table.getOrPut(rel_path); 240 const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: { 241 const ptr = try arena.create(TargetToHash); 242 ptr.* = .empty; 243 path_gop.value_ptr.* = ptr; 244 break :blk ptr; 245 }; 246 try target_to_hash.putNoClobber(arena, dest_target, hash); 247 }, 248 else => std.debug.print("warning: weird file: {s}\n", .{full_path}), 249 } 250 } 251 } 252 break; 253 } else { 254 std.debug.print("warning: libc target not found: {s}\n", .{linux_target.name}); 255 } 256 } 257 std.debug.print("summary: {B} could be reduced to {B}\n", .{ 258 total_bytes, 259 total_bytes - max_bytes_saved, 260 }); 261 try Dir.cwd().createDirPath(io, out_dir); 262 263 var missed_opportunity_bytes: usize = 0; 264 // iterate path_table. for each path, put all the hashes into a list. sort by hit_count. 265 // the hash with the highest hit_count gets to be the "generic" one. everybody else 266 // gets their header in a separate arch directory. 267 var path_it = path_table.iterator(); 268 while (path_it.next()) |path_kv| { 269 var contents_list = std.array_list.Managed(*Contents).init(arena); 270 { 271 var hash_it = path_kv.value_ptr.*.iterator(); 272 while (hash_it.next()) |hash_kv| { 273 const contents = hash_to_contents.getPtr(hash_kv.value_ptr.*).?; 274 try contents_list.append(contents); 275 } 276 } 277 std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan); 278 const best_contents = contents_list.pop().?; 279 if (best_contents.hit_count > 1) { 280 // worth it to make it generic 281 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, generic_name, path_kv.key_ptr.* }); 282 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?); 283 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = best_contents.bytes }); 284 best_contents.is_generic = true; 285 while (contents_list.pop()) |contender| { 286 if (contender.hit_count > 1) { 287 const this_missed_bytes = contender.hit_count * contender.bytes.len; 288 missed_opportunity_bytes += this_missed_bytes; 289 std.debug.print("Missed opportunity ({B}): {s}\n", .{ 290 this_missed_bytes, 291 path_kv.key_ptr.*, 292 }); 293 } else break; 294 } 295 } 296 var hash_it = path_kv.value_ptr.*.iterator(); 297 while (hash_it.next()) |hash_kv| { 298 const contents = hash_to_contents.get(hash_kv.value_ptr.*).?; 299 if (contents.is_generic) continue; 300 301 const dest_target = hash_kv.key_ptr.*; 302 const arch_name = switch (dest_target.arch) { 303 .specific => |a| @tagName(a), 304 else => @tagName(dest_target.arch), 305 }; 306 const out_subpath = try std.fmt.allocPrint(arena, "{s}-linux-any", .{arch_name}); 307 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, out_subpath, path_kv.key_ptr.* }); 308 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?); 309 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = contents.bytes }); 310 } 311 } 312 313 const bad_files = [_][]const u8{ 314 "any-linux-any/linux/netfilter/xt_CONNMARK.h", 315 "any-linux-any/linux/netfilter/xt_DSCP.h", 316 "any-linux-any/linux/netfilter/xt_MARK.h", 317 "any-linux-any/linux/netfilter/xt_RATEEST.h", 318 "any-linux-any/linux/netfilter/xt_TCPMSS.h", 319 "any-linux-any/linux/netfilter_ipv4/ipt_ECN.h", 320 "any-linux-any/linux/netfilter_ipv4/ipt_TTL.h", 321 "any-linux-any/linux/netfilter_ipv6/ip6t_HL.h", 322 }; 323 for (bad_files) |bad_file| { 324 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, bad_file }); 325 try Dir.cwd().deleteFile(io, full_path); 326 } 327 } 328 329 fn usageAndExit(arg0: []const u8) noreturn { 330 std.debug.print("Usage: {s} [--search-path <dir>] --out <dir> --abi <name>\n", .{arg0}); 331 std.debug.print("--search-path can be used any number of times.\n", .{}); 332 std.debug.print(" subdirectories of search paths look like, e.g. x86_64-linux-gnu\n", .{}); 333 std.debug.print("--out is a dir that will be created, and populated with the results\n", .{}); 334 std.process.exit(1); 335 } 336