╭ Cases.zig ────────────────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯
🬂🬂🬒 🬂🬰🬂 🬑🬂🬀 ╭master╮ master xeubie: 🬋 🬋 🬃🬇🬋 │files │ commits refs discuss issues patches new login 🬯🬭🬭 🬭🬰🬭 🬟🬭🬑 ╰──────╯ (15882) user
╭ find file ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╮ ╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │ │ │http│ ssh │ │ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╯ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯
.. ╭─────────────────────────────────╮ │Cases.zig │ ╰─────────────────────────────────╯ Debugger.zig ErrorTrace.zig Libc.zig Link.zig LlvmIr.zig StackTrace.zig convert-stack-trace.zig
1 const Cases = @This(); 2 const builtin = @import("builtin"); 3 4 const std = @import("std"); 5 const Io = std.Io; 6 const assert = std.debug.assert; 7 const Allocator = std.mem.Allocator; 8 const getExternalExecutor = std.zig.system.getExternalExecutor; 9 const ArrayList = std.ArrayList; 10 11 gpa: Allocator, 12 arena: Allocator, 13 io: Io, 14 cases: std.array_list.Managed(Case), 15 16 pub const IncrementalCase = struct { 17 base_path: []const u8, 18 }; 19 20 pub const File = struct { 21 src: [:0]const u8, 22 path: []const u8, 23 }; 24 25 pub const DepModule = struct { 26 name: []const u8, 27 path: []const u8, 28 }; 29 30 pub const Backend = enum { 31 /// Test does not care which backend is used; compiler gets to pick the default. 32 auto, 33 selfhosted, 34 llvm, 35 }; 36 37 pub const CFrontend = enum { 38 clang, 39 aro, 40 }; 41 42 pub const Case = struct { 43 /// The name of the test case. This is shown if a test fails, and 44 /// otherwise ignored. 45 name: []const u8, 46 /// The platform the test targets. For non-native platforms, an emulator 47 /// such as QEMU is required for tests to complete. 48 target: std.Build.ResolvedTarget, 49 /// In order to be able to run e.g. Execution updates, this must be set 50 /// to Executable. 51 output_mode: std.builtin.OutputMode, 52 optimize_mode: std.builtin.Optimize = .debug, 53 54 files: std.array_list.Managed(File), 55 case: ?union(enum) { 56 /// Check that it compiles with no errors. 57 Compile: void, 58 /// Check the main binary output file against an expected set of bytes. 59 /// This is most useful with, for example, `-ofmt=c`. 60 CompareObjectFile: []const u8, 61 /// An error update attempts to compile bad code, and ensures that it 62 /// fails to compile, and for the expected reasons. 63 /// A slice containing the expected stderr template, which 64 /// gets some values substituted. 65 Error: []const []const u8, 66 /// An execution update compiles and runs the input, testing the 67 /// stdout against the expected results 68 /// This is a slice containing the expected message. 69 Execution: []const u8, 70 /// A header update compiles the input with the equivalent of 71 /// `-femit-h` and tests the produced header against the 72 /// expected result. 73 Header: []const u8, 74 }, 75 76 emit_asm: bool = false, 77 emit_bin: bool = true, 78 emit_h: bool = false, 79 is_test: bool = false, 80 expect_exact: bool = false, 81 backend: Backend = .auto, 82 link_libc: bool = false, 83 pic: ?bool = null, 84 pie: ?bool = null, 85 /// A list of imports to cache alongside the source file. 86 imports: []const []const u8 = &.{}, 87 /// Where to look for imports relative to the `cases_dir_path` given to 88 /// `lower_to_build_steps`. If null, file imports will assert. 89 import_path: ?[]const u8 = null, 90 91 deps: std.array_list.Managed(DepModule), 92 93 pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void { 94 case.files.append(.{ .path = name, .src = src }) catch @panic("OOM"); 95 } 96 97 pub fn addDepModule(case: *Case, name: []const u8, path: []const u8) void { 98 case.deps.append(.{ 99 .name = name, 100 .path = path, 101 }) catch @panic("out of memory"); 102 } 103 104 /// Adds a subcase in which the module is updated with `src`, compiled, 105 /// run, and the output is tested against `result`. 106 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void { 107 assert(self.case == null); 108 self.case = .{ .Execution = result }; 109 self.addSourceFile("tmp.zig", src); 110 } 111 112 /// Adds a subcase in which the module is updated with `src`, which 113 /// should contain invalid input, and ensures that compilation fails 114 /// for the expected reasons, given in sequential order in `errors` in 115 /// the form `:line:column: error: message`. 116 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void { 117 assert(errors.len != 0); 118 assert(self.case == null); 119 self.case = .{ .Error = errors }; 120 self.addSourceFile("tmp.zig", src); 121 } 122 123 /// Adds a subcase in which the module is updated with `src`, and 124 /// asserts that it compiles without issue 125 pub fn addCompile(self: *Case, src: [:0]const u8) void { 126 assert(self.case == null); 127 self.case = .Compile; 128 self.addSourceFile("tmp.zig", src); 129 } 130 }; 131 132 pub fn addExe( 133 ctx: *Cases, 134 name: []const u8, 135 target: std.Build.ResolvedTarget, 136 ) *Case { 137 ctx.cases.append(.{ 138 .name = name, 139 .target = target, 140 .files = .init(ctx.arena), 141 .case = null, 142 .output_mode = .Exe, 143 .deps = std.array_list.Managed(DepModule).init(ctx.arena), 144 }) catch @panic("out of memory"); 145 return &ctx.cases.items[ctx.cases.items.len - 1]; 146 } 147 148 /// Adds a test case for Zig input, producing an executable 149 pub fn exe(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case { 150 return ctx.addExe(name, target); 151 } 152 153 pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.Query, b: *std.Build) *Case { 154 var adjusted_query = target_query; 155 adjusted_query.ofmt = .c; 156 ctx.cases.append(.{ 157 .name = name, 158 .target = b.resolveTargetQuery(adjusted_query), 159 .files = .init(ctx.arena), 160 .case = null, 161 .output_mode = .Exe, 162 .deps = std.array_list.Managed(DepModule).init(ctx.arena), 163 .link_libc = true, 164 }) catch @panic("out of memory"); 165 return &ctx.cases.items[ctx.cases.items.len - 1]; 166 } 167 168 pub fn addObjLlvm(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case { 169 const can_emit_asm = switch (target.result.cpu.arch) { 170 .csky, 171 .xtensa, 172 => false, 173 else => true, 174 }; 175 const can_emit_bin = switch (target.result.cpu.arch) { 176 .arc, 177 .csky, 178 .nvptx, 179 .nvptx64, 180 .xcore, 181 .xtensa, 182 => false, 183 else => true, 184 }; 185 186 ctx.cases.append(.{ 187 .name = name, 188 .target = target, 189 .files = .init(ctx.arena), 190 .case = null, 191 .output_mode = .Obj, 192 .deps = std.array_list.Managed(DepModule).init(ctx.arena), 193 .backend = .llvm, 194 .emit_bin = can_emit_bin, 195 .emit_asm = can_emit_asm, 196 }) catch @panic("out of memory"); 197 return &ctx.cases.items[ctx.cases.items.len - 1]; 198 } 199 200 pub fn addObj( 201 ctx: *Cases, 202 name: []const u8, 203 target: std.Build.ResolvedTarget, 204 ) *Case { 205 ctx.cases.append(.{ 206 .name = name, 207 .target = target, 208 .files = .init(ctx.arena), 209 .case = null, 210 .output_mode = .Obj, 211 .deps = std.array_list.Managed(DepModule).init(ctx.arena), 212 }) catch @panic("out of memory"); 213 return &ctx.cases.items[ctx.cases.items.len - 1]; 214 } 215 216 pub fn addTest( 217 ctx: *Cases, 218 name: []const u8, 219 target: std.Build.ResolvedTarget, 220 ) *Case { 221 ctx.cases.append(.{ 222 .name = name, 223 .target = target, 224 .files = .init(ctx.arena), 225 .case = null, 226 .output_mode = .Exe, 227 .is_test = true, 228 .deps = std.array_list.Managed(DepModule).init(ctx.arena), 229 }) catch @panic("out of memory"); 230 return &ctx.cases.items[ctx.cases.items.len - 1]; 231 } 232 233 /// Adds a test case for Zig input, producing an object file. 234 pub fn obj(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case { 235 return ctx.addObj(name, target); 236 } 237 238 /// Adds a test case for ZIR input, producing an object file. 239 pub fn objZIR(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case { 240 return ctx.addObj(name, target, .ZIR); 241 } 242 243 /// Adds a test case for Zig or ZIR input, producing C code. 244 pub fn addC(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case { 245 var target_adjusted = target; 246 target_adjusted.ofmt = std.Target.ObjectFormat.c; 247 ctx.cases.append(.{ 248 .name = name, 249 .target = target_adjusted, 250 .files = .init(ctx.arena), 251 .case = null, 252 .output_mode = .Obj, 253 .deps = std.array_list.Managed(DepModule).init(ctx.arena), 254 }) catch @panic("out of memory"); 255 return &ctx.cases.items[ctx.cases.items.len - 1]; 256 } 257 258 pub fn addTransform( 259 ctx: *Cases, 260 name: []const u8, 261 target: std.Build.ResolvedTarget, 262 src: [:0]const u8, 263 result: [:0]const u8, 264 ) void { 265 ctx.addObj(name, target).addTransform(src, result); 266 } 267 268 /// Adds a test case that compiles the Zig given in `src` to ZIR and tests 269 /// the ZIR against `result` 270 pub fn transform( 271 ctx: *Cases, 272 name: []const u8, 273 target: std.Build.ResolvedTarget, 274 src: [:0]const u8, 275 result: [:0]const u8, 276 ) void { 277 ctx.addTransform(name, target, src, result); 278 } 279 280 pub fn addError( 281 ctx: *Cases, 282 name: []const u8, 283 target: std.Build.ResolvedTarget, 284 src: [:0]const u8, 285 expected_errors: []const []const u8, 286 ) void { 287 ctx.addObj(name, target).addError(src, expected_errors); 288 } 289 290 /// Adds a test case that ensures that the Zig given in `src` fails to 291 /// compile for the expected reasons, given in sequential order in 292 /// `expected_errors` in the form `:line:column: error: message`. 293 pub fn compileError( 294 ctx: *Cases, 295 name: []const u8, 296 target: std.Build.ResolvedTarget, 297 src: [:0]const u8, 298 expected_errors: []const []const u8, 299 ) void { 300 ctx.addError(name, target, src, expected_errors); 301 } 302 303 /// Adds a test case that asserts that the Zig given in `src` compiles 304 /// without any errors. 305 pub fn addCompile( 306 ctx: *Cases, 307 name: []const u8, 308 target: std.Build.ResolvedTarget, 309 src: [:0]const u8, 310 ) void { 311 ctx.addObj(name, target).addCompile(src); 312 } 313 314 /// Adds a test for each file in the provided directory. Recurses nested directories. 315 /// 316 /// Each file should include a test manifest as a contiguous block of comments at 317 /// the end of the file. The first line should be the test type, followed by a set of 318 /// key-value config values, followed by a blank line, then the expected output. 319 pub fn addFromDir(ctx: *Cases, dir: Io.Dir, path_from_root: []const u8, b: *std.Build) void { 320 var current_file: []const u8 = "none"; 321 ctx.addFromDirInner(dir, path_from_root, &current_file, b) catch |err| { 322 std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}", .{ 323 current_file, err, 324 }); 325 }; 326 } 327 328 fn addFromDirInner( 329 ctx: *Cases, 330 iterable_dir: Io.Dir, 331 path_from_root: []const u8, 332 /// This is kept up to date with the currently being processed file so 333 /// that if any errors occur the caller knows it happened during this file. 334 current_file: *[]const u8, 335 b: *std.Build, 336 ) !void { 337 const io = ctx.io; 338 var it = try iterable_dir.walk(ctx.arena); 339 var filenames: ArrayList([]const u8) = .empty; 340 341 while (try it.next(io)) |entry| { 342 // Ignore stuff such as .swp files 343 if (!knownFileExtension(entry.basename)) continue; 344 345 switch (entry.kind) { 346 .file => { 347 b.dependOnFileContents(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); 348 try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path)); 349 }, 350 .directory => { 351 b.dependOnDirectoryContents(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); 352 }, 353 else => continue, 354 } 355 } 356 357 for (filenames.items) |filename| { 358 current_file.* = filename; 359 360 const max_file_size = 10 * 1024 * 1024; 361 const src = try iterable_dir.readFileAllocOptions(io, filename, ctx.arena, .limited(max_file_size), .@"1", 0); 362 363 // Parse the manifest 364 var manifest = try TestManifest.parse(ctx.arena, src); 365 366 const backends = try manifest.getConfigForKeyAlloc(ctx.arena, "backend", Backend); 367 const target_strs = try manifest.getConfigForKeyAlloc(ctx.arena, "target", []const u8); 368 const cpu_features_str = manifest.config_map.get("cpu_features") orelse ""; 369 const targets = try ctx.arena.alloc(std.Target.Query, target_strs.len); 370 for (targets, target_strs) |*query, target_str| { 371 query.* = try std.Target.Query.parse(.{ 372 .arch_os_abi = target_str, 373 .cpu_features = if (cpu_features_str.len == 0) null else cpu_features_str, 374 }); 375 } 376 const is_test = try manifest.getConfigForKeyAssertSingle("is_test", bool); 377 const link_libc = try manifest.getConfigForKeyAssertSingle("link_libc", bool); 378 const output_mode = try manifest.getConfigForKeyAssertSingle("output_mode", std.builtin.OutputMode); 379 const pic = try manifest.getConfigForKeyAssertSingle("pic", ?bool); 380 const pie = try manifest.getConfigForKeyAssertSingle("pie", ?bool); 381 const emit_asm = try manifest.getConfigForKeyAssertSingle("emit_asm", bool); 382 const emit_bin = try manifest.getConfigForKeyAssertSingle("emit_bin", bool); 383 const imports = try manifest.getConfigForKeyAlloc(ctx.arena, "imports", []const u8); 384 385 var cases = std.array_list.Managed(usize).init(ctx.arena); 386 387 // Cross-product to get all possible test combinations 388 for (targets) |target_query| { 389 const resolved_target = b.resolveTargetQuery(target_query); 390 const target = &resolved_target.result; 391 for (backends) |backend| { 392 if (backend == .selfhosted) { 393 switch (target.cpu.arch) { 394 .aarch64, .wasm32, .x86_64, .spirv64, .spirv32 => {}, 395 // Other backends don't support new liveness format 396 else => continue, 397 } 398 } 399 400 if (backend == .selfhosted and target.cpu.arch == .aarch64) { 401 // https://codeberg.org/ziglang/zig/pulls/30232#issuecomment-9182045 402 continue; 403 } 404 405 if (backend == .selfhosted and target.os.tag == .macos and 406 target.cpu.arch == .x86_64 and builtin.cpu.arch == .aarch64) 407 { 408 // Rosetta has issues with ZLD 409 continue; 410 } 411 412 const next = ctx.cases.items.len; 413 try ctx.cases.append(.{ 414 .name = try caseNameFromPath(ctx.arena, filename), 415 .import_path = std.fs.path.dirname(filename), 416 .backend = backend, 417 .files = .init(ctx.arena), 418 .case = null, 419 .emit_asm = emit_asm, 420 .emit_bin = emit_bin, 421 .is_test = is_test, 422 .output_mode = output_mode, 423 .link_libc = link_libc, 424 .pic = pic, 425 .pie = pie, 426 .deps = std.array_list.Managed(DepModule).init(ctx.cases.allocator), 427 .imports = imports, 428 .target = resolved_target, 429 }); 430 try cases.append(next); 431 } 432 } 433 434 for (cases.items) |case_index| { 435 const case = &ctx.cases.items[case_index]; 436 switch (manifest.type) { 437 .compile => { 438 case.addCompile(src); 439 }, 440 .@"error" => { 441 const errors = try manifest.trailingLines(ctx.arena); 442 case.addError(src, errors); 443 }, 444 .run => { 445 const output = try manifest.trailingSplit(ctx.arena); 446 case.addCompareOutput(src, output); 447 }, 448 .cli => @panic("TODO cli tests"), 449 } 450 } 451 } 452 } 453 454 pub fn init(gpa: Allocator, arena: Allocator, io: Io) Cases { 455 return .{ 456 .gpa = gpa, 457 .io = io, 458 .cases = .init(gpa), 459 .arena = arena, 460 }; 461 } 462 463 pub const CaseTestOptions = struct { 464 test_filters: []const []const u8, 465 test_target_filters: []const []const u8, 466 skip_compile_errors: bool, 467 skip_non_native: bool, 468 skip_spirv: bool, 469 skip_wasm: bool, 470 skip_freebsd: bool, 471 skip_netbsd: bool, 472 skip_openbsd: bool, 473 skip_windows: bool, 474 skip_darwin: bool, 475 skip_linux: bool, 476 skip_llvm: bool, 477 skip_libc: bool, 478 }; 479 480 pub fn lowerToBuildSteps( 481 self: *Cases, 482 b: *std.Build, 483 parent_step: *std.Build.Step, 484 options: CaseTestOptions, 485 ) void { 486 const io = self.io; 487 const graph = b.graph; 488 const arena = graph.arena; 489 const host = b.resolveTargetQuery(.{}); 490 491 for (self.cases.items) |case| { 492 for (options.test_filters) |test_filter| { 493 if (std.mem.find(u8, case.name, test_filter)) |_| break; 494 } else if (options.test_filters.len > 0) continue; 495 496 if (case.case.? == .Error and options.skip_compile_errors) continue; 497 498 if (options.skip_non_native and !@import("../tests.zig").isNative(&case.target, &b.graph.host.result)) 499 continue; 500 501 if (options.skip_spirv and case.target.query.cpu_arch != null and case.target.query.cpu_arch.?.isSpirV()) continue; 502 if (options.skip_wasm and case.target.query.cpu_arch != null and case.target.query.cpu_arch.?.isWasm()) continue; 503 504 if (options.skip_freebsd and case.target.query.os_tag == .freebsd) continue; 505 if (options.skip_netbsd and case.target.query.os_tag == .netbsd) continue; 506 if (options.skip_openbsd and case.target.query.os_tag == .openbsd) continue; 507 if (options.skip_windows and case.target.query.os_tag == .windows) continue; 508 if (options.skip_darwin and case.target.query.os_tag != null and case.target.query.os_tag.?.isDarwin()) continue; 509 if (options.skip_linux and case.target.query.os_tag == .linux) continue; 510 511 const would_use_llvm = @import("../tests.zig").wouldUseLlvm( 512 switch (case.backend) { 513 .auto => null, 514 .selfhosted => false, 515 .llvm => true, 516 }, 517 case.target.query, 518 case.optimize_mode, 519 ); 520 if (options.skip_llvm and would_use_llvm) continue; 521 522 const triple_txt = case.target.query.zigTriple(arena) catch @panic("OOM"); 523 524 if (options.test_target_filters.len > 0) { 525 for (options.test_target_filters) |filter| { 526 if (std.mem.find(u8, triple_txt, filter) != null) break; 527 } else continue; 528 } 529 530 if (options.skip_libc and case.link_libc) 531 continue; 532 533 const writefiles = b.addWriteFiles(); 534 var file_sources = std.StringHashMap(std.Build.LazyPath).init(arena); 535 defer file_sources.deinit(); 536 const first_file = case.files.items[0]; 537 const root_source_file = writefiles.add(first_file.path, first_file.src); 538 file_sources.put(first_file.path, root_source_file) catch @panic("OOM"); 539 for (case.files.items[1..]) |file| { 540 file_sources.put(file.path, writefiles.add(file.path, file.src)) catch @panic("OOM"); 541 } 542 543 for (case.imports) |import_rel| { 544 _ = writefiles.addCopyFile(.{ .src_path = .{ 545 .owner = b, 546 .sub_path = b.pathJoin(&.{ 547 "test", 548 "cases", 549 case.import_path orelse @panic("import_path not set"), 550 import_rel, 551 }), 552 } }, import_rel); 553 } 554 555 const mod = b.createModule(.{ 556 .root_source_file = root_source_file, 557 .target = case.target, 558 .optimize = case.optimize_mode, 559 }); 560 561 if (case.link_libc) mod.link_libc = true; 562 if (case.pic) |pic| mod.pic = pic; 563 for (case.deps.items) |dep| { 564 mod.addAnonymousImport(dep.name, .{ 565 .root_source_file = file_sources.get(dep.path).?, 566 }); 567 } 568 569 const artifact = if (case.is_test) b.addTest(.{ 570 .name = case.name, 571 .root_module = mod, 572 }) else switch (case.output_mode) { 573 .Obj => b.addObject(.{ 574 .name = case.name, 575 .root_module = mod, 576 }), 577 .Lib => b.addLibrary(.{ 578 .linkage = .static, 579 .name = case.name, 580 .root_module = mod, 581 }), 582 .Exe => b.addExecutable(.{ 583 .name = case.name, 584 .root_module = mod, 585 }), 586 }; 587 588 if (case.pie) |pie| artifact.pie = pie; 589 590 switch (case.backend) { 591 .auto => {}, 592 .selfhosted => { 593 artifact.use_llvm = false; 594 artifact.use_lld = false; 595 }, 596 .llvm => { 597 artifact.use_llvm = true; 598 }, 599 } 600 601 switch (case.case.?) { 602 .Compile => { 603 // Force the assembly/binary to be emitted if requested. 604 if (case.emit_asm) { 605 _ = artifact.getEmittedAsm(); 606 } 607 if (case.emit_bin) { 608 _ = artifact.getEmittedBin(); 609 } 610 parent_step.dependOn(&artifact.step); 611 }, 612 .CompareObjectFile => |expected_output| { 613 const check = b.addCheckFile(artifact.getEmittedBin(), .{ 614 .expected_exact = expected_output, 615 }); 616 617 parent_step.dependOn(&check.step); 618 }, 619 .Error => |expected_msgs| { 620 assert(expected_msgs.len != 0); 621 artifact.expect_errors = .{ .exact = expected_msgs }; 622 parent_step.dependOn(&artifact.step); 623 }, 624 .Execution => |expected_stdout| no_exec: { 625 const run = if (case.target.result.ofmt == .c) run_step: { 626 if (getExternalExecutor(io, &case.target.result, .{ 627 .host_cpu_arch = host.result.cpu.arch, 628 .host_os_tag = host.result.os.tag, 629 .link_libc = true, 630 }) != .native) { 631 // We wouldn't be able to run the compiled C code. 632 break :no_exec; 633 } 634 const run_c = b.addSystemCommand(&.{ 635 b.graph.zig_exe, 636 "run", 637 "-cflags", 638 "-Ilib", 639 "-std=c99", 640 "-pedantic", 641 "-Werror", 642 "-Wno-dollar-in-identifier-extension", 643 "-Wno-incompatible-library-redeclaration", // https://github.com/ziglang/zig/issues/875 644 "-Wno-incompatible-pointer-types", 645 "-Wno-overlength-strings", 646 "--", 647 "-lc", 648 "-target", 649 triple_txt, 650 }); 651 run_c.addArtifactArg(artifact); 652 break :run_step run_c; 653 } else b.addRunArtifact(artifact); 654 run.skip_foreign_checks = true; 655 if (!case.is_test) { 656 run.expectStdOutEqual(expected_stdout); 657 } 658 parent_step.dependOn(&run.step); 659 }, 660 .Header => @panic("TODO"), 661 } 662 } 663 } 664 665 /// Default config values for known test manifest key-value pairings. 666 /// Currently handled defaults are: 667 /// * backend 668 /// * target 669 /// * output_mode 670 /// * is_test 671 const TestManifestConfigDefaults = struct { 672 /// Asserts if the key doesn't exist - yep, it's an oversight alright. 673 fn get(@"type": TestManifest.Type, key: []const u8) []const u8 { 674 if (std.mem.eql(u8, key, "backend")) { 675 return "auto"; 676 } else if (std.mem.eql(u8, key, "target")) { 677 if (@"type" == .@"error") { 678 return "native"; 679 } 680 return "native,wasm32-wasi"; 681 } else if (std.mem.eql(u8, key, "output_mode")) { 682 return switch (@"type") { 683 .@"error" => "Obj", 684 .run => "Exe", 685 .compile => "Obj", 686 .cli => @panic("TODO test harness for CLI tests"), 687 }; 688 } else if (std.mem.eql(u8, key, "emit_asm")) { 689 return "false"; 690 } else if (std.mem.eql(u8, key, "emit_bin")) { 691 return "true"; 692 } else if (std.mem.eql(u8, key, "is_test")) { 693 return "false"; 694 } else if (std.mem.eql(u8, key, "link_libc")) { 695 return "false"; 696 } else if (std.mem.eql(u8, key, "c_frontend")) { 697 return "clang"; 698 } else if (std.mem.eql(u8, key, "pic")) { 699 return "null"; 700 } else if (std.mem.eql(u8, key, "pie")) { 701 return "null"; 702 } else if (std.mem.eql(u8, key, "imports")) { 703 return ""; 704 } else if (std.mem.eql(u8, key, "cpu_features")) { 705 return ""; 706 } else unreachable; 707 } 708 }; 709 710 /// Manifest syntax example: 711 /// (see https://github.com/ziglang/zig/issues/11288) 712 /// 713 /// error 714 /// backend=selfhosted,llvm 715 /// output_mode=exe 716 /// 717 /// :3:19: error: foo 718 /// 719 /// run 720 /// target=x86_64-linux,aarch64-macos 721 /// 722 /// I am expected stdout! Hello! 723 /// 724 /// cli 725 /// 726 /// build test 727 const TestManifest = struct { 728 type: Type, 729 config_map: std.StringHashMap([]const u8), 730 trailing_bytes: []const u8 = "", 731 732 const valid_keys = std.StaticStringMap(void).initComptime(.{ 733 .{ "emit_asm", {} }, 734 .{ "emit_bin", {} }, 735 .{ "is_test", {} }, 736 .{ "output_mode", {} }, 737 .{ "target", {} }, 738 .{ "cpu_features", {} }, 739 .{ "c_frontend", {} }, 740 .{ "link_libc", {} }, 741 .{ "backend", {} }, 742 .{ "pic", {} }, 743 .{ "pie", {} }, 744 .{ "imports", {} }, 745 }); 746 747 const Type = enum { 748 @"error", 749 run, 750 cli, 751 compile, 752 }; 753 754 const TrailingIterator = struct { 755 inner: std.mem.TokenIterator(u8, .any), 756 757 fn next(self: *TrailingIterator) ?[]const u8 { 758 const next_inner = self.inner.next() orelse return null; 759 return if (next_inner.len == 2) "" else std.mem.trimEnd(u8, next_inner[3..], " \t"); 760 } 761 }; 762 763 fn ConfigValueIterator(comptime T: type) type { 764 return struct { 765 inner: std.mem.TokenIterator(u8, .scalar), 766 767 fn next(self: *@This()) !?T { 768 const next_raw = self.inner.next() orelse return null; 769 const parseFn = getDefaultParser(T); 770 return try parseFn(next_raw); 771 } 772 }; 773 } 774 775 fn parse(arena: Allocator, bytes: []const u8) !TestManifest { 776 // The manifest is the last contiguous block of comments in the file 777 // We scan for the beginning by searching backward for the first non-empty line that does not start with "//" 778 var start: ?usize = null; 779 var end: usize = bytes.len; 780 if (bytes.len > 0) { 781 var cursor: usize = bytes.len - 1; 782 while (true) { 783 // Move to beginning of line 784 while (cursor > 0 and bytes[cursor - 1] != '\n') cursor -= 1; 785 786 if (std.mem.startsWith(u8, bytes[cursor..], "//")) { 787 start = cursor; // Contiguous comment line, include in manifest 788 } else { 789 if (start != null) break; // Encountered non-comment line, end of manifest 790 791 // We ignore all-whitespace lines following the comment block, but anything else 792 // means that there is no manifest present. 793 if (std.mem.trim(u8, bytes[cursor..end], " \r\n\t").len == 0) { 794 end = cursor; 795 } else break; // If it's not whitespace, there is no manifest 796 } 797 798 // Move to previous line 799 if (cursor != 0) cursor -= 1 else break; 800 } 801 } 802 803 const actual_start = start orelse return error.MissingTestManifest; 804 const manifest_bytes = bytes[actual_start..end]; 805 806 var it = std.mem.tokenizeAny(u8, manifest_bytes, "\r\n"); 807 808 // First line is the test type 809 const tt: Type = blk: { 810 const line = it.next() orelse return error.MissingTestCaseType; 811 const raw = std.mem.trim(u8, line[2..], " \t"); 812 if (std.mem.eql(u8, raw, "error")) { 813 break :blk .@"error"; 814 } else if (std.mem.eql(u8, raw, "run")) { 815 break :blk .run; 816 } else if (std.mem.eql(u8, raw, "cli")) { 817 break :blk .cli; 818 } else if (std.mem.eql(u8, raw, "compile")) { 819 break :blk .compile; 820 } else { 821 std.log.warn("unknown test case type requested: {s}", .{raw}); 822 return error.UnknownTestCaseType; 823 } 824 }; 825 826 var manifest: TestManifest = .{ 827 .type = tt, 828 .config_map = std.StringHashMap([]const u8).init(arena), 829 }; 830 831 // Any subsequent line until a blank comment line is key=value(s) pair 832 while (it.next()) |line| { 833 const trimmed = std.mem.trim(u8, line[2..], " \t"); 834 if (trimmed.len == 0) break; 835 836 // Parse key=value(s) 837 var kv_it = std.mem.splitScalar(u8, trimmed, '='); 838 const key = kv_it.first(); 839 if (!valid_keys.has(key)) { 840 return error.InvalidKey; 841 } 842 try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig); 843 } 844 845 // Finally, trailing is expected output 846 manifest.trailing_bytes = manifest_bytes[it.index..]; 847 848 return manifest; 849 } 850 851 fn getConfigForKey( 852 self: TestManifest, 853 key: []const u8, 854 comptime T: type, 855 ) ConfigValueIterator(T) { 856 const bytes = self.config_map.get(key) orelse TestManifestConfigDefaults.get(self.type, key); 857 return ConfigValueIterator(T){ 858 .inner = std.mem.tokenizeScalar(u8, bytes, ','), 859 }; 860 } 861 862 fn getConfigForKeyAlloc( 863 self: TestManifest, 864 allocator: Allocator, 865 key: []const u8, 866 comptime T: type, 867 ) ![]const T { 868 var out = std.array_list.Managed(T).init(allocator); 869 defer out.deinit(); 870 var it = self.getConfigForKey(key, T); 871 while (try it.next()) |item| { 872 try out.append(item); 873 } 874 return try out.toOwnedSlice(); 875 } 876 877 fn getConfigForKeyAssertSingle(self: TestManifest, key: []const u8, comptime T: type) !T { 878 var it = self.getConfigForKey(key, T); 879 const res = (try it.next()) orelse unreachable; 880 assert((try it.next()) == null); 881 return res; 882 } 883 884 fn trailing(self: TestManifest) TrailingIterator { 885 return .{ 886 .inner = std.mem.tokenizeAny(u8, self.trailing_bytes, "\r\n"), 887 }; 888 } 889 890 fn trailingSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const u8 { 891 var out = std.array_list.Managed(u8).init(allocator); 892 defer out.deinit(); 893 var trailing_it = self.trailing(); 894 while (trailing_it.next()) |line| { 895 try out.appendSlice(line); 896 try out.append('\n'); 897 } 898 if (out.items.len > 0) { 899 try out.resize(out.items.len - 1); 900 } 901 return try out.toOwnedSlice(); 902 } 903 904 fn trailingLines(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 { 905 var out = std.array_list.Managed([]const u8).init(allocator); 906 defer out.deinit(); 907 var it = self.trailing(); 908 while (it.next()) |line| { 909 try out.append(line); 910 } 911 return try out.toOwnedSlice(); 912 } 913 914 fn trailingLinesSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 { 915 // Collect output lines split by empty lines 916 var out = std.array_list.Managed([]const u8).init(allocator); 917 defer out.deinit(); 918 var buf = std.array_list.Managed(u8).init(allocator); 919 defer buf.deinit(); 920 var it = self.trailing(); 921 while (it.next()) |line| { 922 if (line.len == 0) { 923 if (buf.items.len != 0) { 924 try out.append(try buf.toOwnedSlice()); 925 buf.items.len = 0; 926 } 927 continue; 928 } 929 try buf.appendSlice(line); 930 try buf.append('\n'); 931 } 932 try out.append(try buf.toOwnedSlice()); 933 return try out.toOwnedSlice(); 934 } 935 936 fn ParseFn(comptime T: type) type { 937 return fn ([]const u8) anyerror!T; 938 } 939 940 fn getDefaultParser(comptime T: type) ParseFn(T) { 941 if (T == std.Target.Query) return struct { 942 fn parse(str: []const u8) anyerror!T { 943 return std.Target.Query.parse(.{ .arch_os_abi = str }); 944 } 945 }.parse; 946 947 switch (@typeInfo(T)) { 948 .int => return struct { 949 fn parse(str: []const u8) anyerror!T { 950 return try std.fmt.parseInt(T, str, 0); 951 } 952 }.parse, 953 .bool => return struct { 954 fn parse(str: []const u8) anyerror!T { 955 if (std.mem.eql(u8, str, "true")) return true; 956 if (std.mem.eql(u8, str, "false")) return false; 957 std.debug.print("{s}\n", .{str}); 958 return error.InvalidBool; 959 } 960 }.parse, 961 .@"enum" => return struct { 962 fn parse(str: []const u8) anyerror!T { 963 return std.meta.stringToEnum(T, str) orelse { 964 std.log.err("unknown enum variant for {s}: {s}", .{ @typeName(T), str }); 965 return error.UnknownEnumVariant; 966 }; 967 } 968 }.parse, 969 .optional => |o| return struct { 970 fn parse(str: []const u8) anyerror!T { 971 if (std.mem.eql(u8, str, "null")) return null; 972 return try getDefaultParser(o.child)(str); 973 } 974 }.parse, 975 .@"struct" => @compileError("no default parser for " ++ @typeName(T)), 976 .pointer => { 977 if (T == []const u8) { 978 return struct { 979 fn parse(str: []const u8) anyerror!T { 980 return str; 981 } 982 }.parse; 983 } else { 984 @compileError("no default parser for " ++ @typeName(T)); 985 } 986 }, 987 else => @compileError("no default parser for " ++ @typeName(T)), 988 } 989 } 990 }; 991 992 fn knownFileExtension(filename: []const u8) bool { 993 // List taken from `Compilation.classifyFileExt` in the compiler. 994 for ([_][]const u8{ 995 ".c", ".C", ".cc", ".cpp", 996 ".cxx", ".stub", ".m", ".mm", 997 ".ll", ".bc", ".s", ".S", 998 ".h", ".zig", ".so", ".dll", 999 ".dylib", ".tbd", ".a", ".lib", 1000 ".o", ".obj", ".cu", ".def", 1001 ".rc", ".res", ".manifest", 1002 }) |ext| { 1003 if (std.mem.endsWith(u8, filename, ext)) return true; 1004 } 1005 // Final check for .so.X, .so.X.Y, .so.X.Y.Z. 1006 // From `Compilation.hasSharedLibraryExt`. 1007 var it = std.mem.splitScalar(u8, filename, '.'); 1008 _ = it.first(); 1009 var so_txt = it.next() orelse return false; 1010 while (!std.mem.eql(u8, so_txt, "so")) { 1011 so_txt = it.next() orelse return false; 1012 } 1013 const n1 = it.next() orelse return false; 1014 const n2 = it.next(); 1015 const n3 = it.next(); 1016 _ = std.fmt.parseInt(u32, n1, 10) catch return false; 1017 if (n2) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false; 1018 if (n3) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false; 1019 if (it.next() != null) return false; 1020 return false; 1021 } 1022 1023 /// `path` is a path relative to the root case directory. 1024 /// e.g. `compile_errors/undeclared_identifier.zig` 1025 /// The case name is computed by removing the extension and substituting path separators for dots. 1026 /// e.g. `compile_errors.undeclared_identifier` 1027 /// Including the directory components makes `-Dtest-filter` more useful, because you can filter 1028 /// based on subdirectory; e.g. `-Dtest-filter=compile_errors` to run the compile error tets. 1029 fn caseNameFromPath(arena: Allocator, path: []const u8) Allocator.Error![]const u8 { 1030 const ext_len = std.fs.path.extension(path).len; 1031 const path_sans_ext = path[0 .. path.len - ext_len]; 1032 const result = try arena.dupe(u8, path_sans_ext); 1033 std.mem.replaceScalar(u8, result, std.fs.path.sep, '.'); 1034 return result; 1035 } 1036