╭─────────────╮ │scroll to top│ ╰─────────────╯ ╭ event.zig ────────────────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯
╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╮ │http│ ssh │ │ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╯
2706 try upsertBytes(DB, hash_kind, map, key, &value); 2707 } else { 2708 @compileError("unsupported upsert field type: " ++ @typeName(Field)); 2709 } 2710 }, 2711 .int => |int_info| switch (int_info.signedness) { 2712 .unsigned => { 2713 if (try map.getCursor(key)) |value_cursor| { 2714 if (try value_cursor.readUint() == value) { 2715 return; 2716 } 2717 } 2718 2719 try map.put(key, .{ .uint = value }); 2720 }, 2721 .signed => { 2722 if (try map.getCursor(key)) |value_cursor| { 2723 if (try value_cursor.readInt() == value) { 2724 return; 2725 } 2726 } 2727 2728 try map.put(key, .{ .int = value }); 2729 }, 2730 }, 2731 .bool => { 2732 const bytes = if (value) "true" else "false"; 2733 if (try map.getCursor(key)) |value_cursor| { 2734 var buffer: [5]u8 = undefined; 2735 const existing = try value_cursor.readBytesObject(&buffer); 2736 if (existing.format_tag) |format_tag| { 2737 if (std.mem.eql(u8, &format_tag, "bl") and std.mem.eql(u8, existing.value, bytes)) { 2738 return; 2739 } 2740 } 2741 } 2742 2743 try map.put(key, .{ .bytes_object = .{ .value = bytes, .format_tag = "bl".* } }); 2744 }, 2745 .@"enum" => try upsertBytes(DB, hash_kind, map, key, @tagName(value)), 2746 .@"union" => switch (value) { 2747 inline else => |payload, tag| { 2748 if (@TypeOf(payload) == void) { 2749 try upsertBytes(DB, hash_kind, map, key, @tagName(tag)); 2750 } else { 2751 const tag_key = hash.hashInt(hash_kind, @tagName(tag)); 2752 if (try map.getCursor(key)) |cursor| { 2753 switch (cursor.slot().tag) { 2754 .hash_map => { 2755 const variants = try DB.HashMap(.read_only).init(cursor); 2756 if (try variants.getCursor(tag_key) == null) try map.put(key, .{ .slot = null }); 2757 }, 2758 else => try map.put(key, .{ .slot = null }), 2759 } 2760 } 2761 const variants = try DB.HashMap(.read_write).init(try map.putCursor(key)); 2762 if (@typeInfo(@TypeOf(payload)) == .optional) { 2763 // keep the variant even when its payload is null 2764 if (payload == null) return try variants.put(tag_key, .{ .slot = null }); 2765 } 2766 if (try variants.getCursor(tag_key)) |cursor| { 2767 if (cursor.slot().tag == .none) _ = try variants.remove(tag_key); 2768 } 2769 try upsertField(DB, hash_kind, variants, @tagName(tag), @TypeOf(payload), payload); 2770 } 2771 }, 2772 }, 2773 .optional => |optional_info| { 2774 // a missing key is null 2775 if (value) |child| { 2776 try upsertField(DB, hash_kind, map, field_name, optional_info.child, child); 2777 } else { 2778 _ = try map.remove(key); 2779 } 2780 }, 2781 .@"struct" => try upsert( 2782 Field, 2783 DB, 2784 hash_kind, 2785 try DB.HashMap(.read_write).init(try map.putCursor(key)), 2786 value, 2787 ), 2788 else => @compileError("unsupported upsert field type: " ++ @typeName(Field)), 2789 } 2790 } 2791 2792 fn upsertBytes( 2793 comptime DB: type, 2794 comptime hash_kind: hash.HashKind, 2795 map: DB.HashMap(.read_write), 2796 key: hash.HashInt(hash_kind), 2797 value: []const u8, 2798 ) !void { 2799 var existing_cursor_maybe = try map.getCursor(key); 2800 if (existing_cursor_maybe) |*existing_cursor| { 2801 if (try bytesEqual(DB, existing_cursor, value)) { 2802 return; 2803 } 2804 } 2805 2806 try map.put(key, .{ .bytes = value }); 2807 } 2808 2809 fn bytesEqual( 2810 comptime DB: type, 2811 cursor: *DB.Cursor(.read_only), 2812 value: []const u8, 2813 ) !bool { 2814 switch (cursor.slot().tag) { 2815 .bytes, .short_bytes => {}, 2816 else => return false, 2817 } 2818 var read_buffer: [1024]u8 = undefined; 2819 var reader = try cursor.reader(&read_buffer); 2820 if (reader.size != value.len) { 2821 return false; 2822 } 2823 2824 var chunk_buffer: [1024]u8 = undefined; 2825 var offset: usize = 0; 2826 while (offset < value.len) { 2827 const chunk_len = @min(chunk_buffer.len, value.len - offset); 2828 try reader.interface.readSliceAll(chunk_buffer[0..chunk_len]); 2829 if (!std.mem.eql(u8, chunk_buffer[0..chunk_len], value[offset .. offset + chunk_len])) { 2830 return false; 2831 } 2832 offset += chunk_len; 2833 } 2834 2835 return true; 2836 } 2837 2838 // 2839 // CommitIterator 2840 // 2841 // differs from xit's ObjectIterator in a few ways: 2842 // 2843 // 1. iterates parents first, rather than children first 2844 // 2. `next` only returns the object id, not the full Object 2845 // 3. stores its temporary state in a file on disk to avoid OOMs 2846 // 2847 2848 pub fn CommitIterator(comptime repo_kind: rp.RepoKind, comptime repo_opts: rp.RepoOpts(repo_kind)) type { 2849 return struct { 2850 const DB = EventDB(repo_opts.hash); 2851 // a plain file is used for the scratch db because each write outside 2852 // a transaction is synced anyway, so buffering would gain nothing 2853 const TempDB = xit.xitdb.Database(.file, hash.HashInt(repo_opts.hash)); 2854 const db_name = "haxy-repo-events.db"; 2855 2856 repo_dir: std.Io.Dir, 2857 db_file: std.Io.File, 2858 db: *TempDB, 2859 // map of each object id to its children 2860 parent_to_children: TempDB.HashMap(.read_write), 2861 // tracking pending parents is necessary because a child can only 2862 // be returned after all of its unconsumed parents have been returned 2863 child_to_pending_parents: TempDB.HashMap(.read_write), 2864 // queue of object ids that are ready to be returned from `next` 2865 ready_queue: TempDB.ArrayList(.read_write), 2866 ready_queue_index: u64, 2867 // the object id at the tip of this branch 2868 newest_object_id: ?[hash.byteLen(repo_opts.hash)]u8, 2869 // the object id furthest back in the history that hasn't been consumed 2870 oldest_object_id: ?[hash.byteLen(repo_opts.hash)]u8, 2871 2872 pub fn init( 2873 state: rp.Repo(repo_kind, repo_opts).State(.read_only), 2874 io: std.Io, 2875 allocator: std.mem.Allocator, 2876 consumed_object_ids: DB.HashMap(.read_only), 2877 ref: rf.Ref, 2878 ) !CommitIterator(repo_kind, repo_opts) { 2879 const db_file = try state.core.repo_dir.createFile(io, db_name, .{ .truncate = true, .lock = .exclusive, .read = true }); 2880 errdefer { 2881 db_file.close(io); 2882 state.core.repo_dir.deleteFile(io, db_name) catch {}; 2883 } 2884 2885 // cursors point at the db, so it needs a stable address 2886 const db_ptr = try allocator.create(TempDB); 2887 errdefer allocator.destroy(db_ptr); 2888 // the db is scratch state that gets deleted after iteration, so 2889 // there is nothing worth fsyncing 2890 db_ptr.* = try TempDB.init(.{ .io = io, .file = db_file, .fsync = false }); 2891 2892 const map = try TempDB.HashMap(.read_write).init(db_ptr.rootCursor()); 2893 2894 const parent_to_children_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "parent->children")); 2895 const parent_to_children = try TempDB.HashMap(.read_write).init(parent_to_children_cursor); 2896 2897 const child_to_pending_parents_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "child->pending-parents")); 2898 const child_to_pending_parents = try TempDB.HashMap(.read_write).init(child_to_pending_parents_cursor); 2899 2900 const ready_queue_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "ready-queue")); 2901 const ready_queue = try TempDB.ArrayList(.read_write).init(ready_queue_cursor); 2902 2903 var self = CommitIterator(repo_kind, repo_opts){ 2904 .repo_dir = state.core.repo_dir, 2905 .db_file = db_file, 2906 .db = db_ptr, 2907 .parent_to_children = parent_to_children, 2908 .child_to_pending_parents = child_to_pending_parents, 2909 .ready_queue = ready_queue, 2910 .ready_queue_index = 0, 2911 .newest_object_id = null, 2912 .oldest_object_id = null, 2913 }; 2914 errdefer self.deinit(io, allocator); 2915 2916 try self.collect(state, io, allocator, consumed_object_ids, ref); 2917 2918 return self; 2919 } 2920 2921 pub fn deinit(self: *CommitIterator(repo_kind, repo_opts), io: std.Io, allocator: std.mem.Allocator) void { 2922 self.db_file.close(io); 2923 self.repo_dir.deleteFile(io, db_name) catch {}; 2924 allocator.destroy(self.db); 2925 } 2926 2927 pub fn next(self: *CommitIterator(repo_kind, repo_opts)) !?[hash.byteLen(repo_opts.hash)]u8 { 2928 if (self.ready_queue_index >= try self.ready_queue.count()) return null; 2929 2930 const oid = try readOidFromList(self.ready_queue.readOnly(), self.ready_queue_index); 2931 self.ready_queue_index += 1; 2932 2933 try self.enqueueChildren(&oid); 2934 return oid; 2935 } 2936 2937 fn collect( 2938 self: *CommitIterator(repo_kind, repo_opts), 2939 state: rp.Repo(repo_kind, repo_opts).State(.read_only), 2940 io: std.Io, 2941 allocator: std.mem.Allocator, 2942 consumed_object_ids: DB.HashMap(.read_only), 2943 ref: rf.Ref, 2944 ) !void { 2945 const map = try TempDB.HashMap(.read_write).init(self.db.rootCursor()); 2946 2947 const walk_queue_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "walk-queue")); 2948 const walk_queue = try TempDB.ArrayList(.read_write).init(walk_queue_cursor); 2949 var walk_queue_index: u64 = 0; 2950 2951 const seen_object_ids_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "seen-object-ids")); 2952 const seen_object_ids = try TempDB.HashSet(.read_write).init(seen_object_ids_cursor); 2953 2954 const head_oid_hex = (try rf.readRecur(repo_kind, repo_opts, state, io, .{ .ref = ref })) orelse return error.OidNotFound; 2955 var head_oid: [hash.byteLen(repo_opts.hash)]u8 = undefined; 2956 _ = try std.fmt.hexToBytes(&head_oid, &head_oid_hex); 2957 try walk_queue.append(.{ .bytes = &head_oid }); 2958 2959 while (walk_queue_index < try walk_queue.count()) { 2960 const oid = try readOidFromList(walk_queue.readOnly(), walk_queue_index); 2961 const oid_int = hash.bytesToInt(repo_opts.hash, &oid); 2962 2963 walk_queue_index += 1; 2964 2965 // if we've seen this object id, skip it 2966 if (null != try seen_object_ids.getCursor(oid_int)) { 2967 continue; 2968 } 2969 2970 try seen_object_ids.put(oid_int, .{ .bytes = &oid }); 2971 2972 var commit_object = try obj.Object(repo_kind, repo_opts).init(state, io, allocator, &std.fmt.bytesToHex(oid, .lower)); 2973 defer commit_object.deinit(); 2974 2975 // if this object id has already been consumed, skip it 2976 if (null != try consumed_object_ids.getCursor(oid_int)) { 2977 continue; 2978 } 2979 2980 if (self.newest_object_id == null) self.newest_object_id = oid; 2981 self.oldest_object_id = oid; 2982 2983 const parent_oids = commit_object.content.commit.metadata.parent_oids orelse return error.ParentOidsNotFound; 2984 var pending_parent_count: u64 = 0; 2985 for (parent_oids) |*parent_oid| { 2986 var parent_oid_bytes: [hash.byteLen(repo_opts.hash)]u8 = undefined; 2987 _ = try std.fmt.hexToBytes(&parent_oid_bytes, parent_oid); 2988 const parent_oid_int = hash.bytesToInt(repo_opts.hash, &parent_oid_bytes); 2989 2990 // if this object id has already been consumed, skip it 2991 if (null != try consumed_object_ids.getCursor(parent_oid_int)) { 2992 continue; 2993 } 2994 2995 // if this object id hasn't already been seen, add it to the walk queue 2996 if (null == try seen_object_ids.getCursor(parent_oid_int)) { 2997 try walk_queue.append(.{ .bytes = &parent_oid_bytes }); 2998 } 2999 3000 const children_cursor = try self.parent_to_children.putCursor(parent_oid_int); 3001 const children = try TempDB.HashSet(.read_write).init(children_cursor); 3002 try children.put(oid_int, .{ .bytes = &oid }); 3003 pending_parent_count += 1; 3004 } 3005 3006 if (pending_parent_count == 0) { 3007 try self.ready_queue.append(.{ .bytes = &oid }); 3008 } else { 3009 try self.child_to_pending_parents.put(oid_int, .{ .uint = pending_parent_count }); 3010 } 3011 } 3012 } 3013 3014 fn enqueueChildren(self: *CommitIterator(repo_kind, repo_opts), oid: *const [hash.byteLen(repo_opts.hash)]u8) !void { 3015 const oid_int = hash.bytesToInt(repo_opts.hash, oid); 3016 const children_cursor = try self.parent_to_children.getCursor(oid_int) orelse return; 3017 const children = try TempDB.HashSet(.read_only).init(children_cursor); 3018 var children_iter = try children.iterator(); 3019 3020 while (try children_iter.next()) |*child_cursor| { 3021 const kv_pair = try child_cursor.readKeyValuePair(); 3022 const child_oid_int = kv_pair.hash; 3023 const pending_parent_cursor = try self.child_to_pending_parents.getCursor(child_oid_int) orelse return error.CursorNotFound; 3024 const pending_parent_count = try pending_parent_cursor.readUint(); 3025 3026 if (pending_parent_count <= 1) { 3027 _ = try self.child_to_pending_parents.remove(child_oid_int); 3028 const child_oid = hash.intToBytes(hash.HashInt(repo_opts.hash), child_oid_int); 3029 try self.ready_queue.append(.{ .bytes = &child_oid }); 3030 } else { 3031 try self.child_to_pending_parents.put(child_oid_int, .{ .uint = pending_parent_count - 1 }); 3032 } 3033 } 3034 } 3035 3036 fn readOidFromList(list: TempDB.ArrayList(.read_only), index: u64) ![hash.byteLen(repo_opts.hash)]u8 { 3037 const oid_cursor = try list.getCursor(index) orelse return error.CursorNotFound; 3038 var oid: [hash.byteLen(repo_opts.hash)]u8 = undefined; 3039 _ = try oid_cursor.readBytes(&oid); 3040 return oid; 3041 } 3042 }; 3043 } 3044