feat(C1.2): persistent content-addressed cache for compiled #source units

compileCToObjects now probes .sx-cache/c-<key>.o before invoking the
embedded clang and writes fresh objects back (per-pid temp + copy, the
main object cache's pattern). Default on for both JIT and AOT — the
temp-compile-and-delete behavior it replaces was strictly worse. A
cached entry must carry an object-file magic (Mach-O/ELF) or it falls
back to a fresh compile; no cache failure can fail a build. Cold/warm
verified via --time: the object compile disappears on the warm run.
This commit is contained in:
agra
2026-06-12 16:48:02 +03:00
parent 10f5a4318d
commit 053314b3ea
2 changed files with 98 additions and 3 deletions

View File

@@ -232,15 +232,74 @@ pub fn processCImport(
// Native C compilation (compile to .o, not LLVM module)
// ---------------------------------------------------------------------------
/// Compile C sources to native object files (in memory).
/// A cached entry must at least LOOK like an object file (Mach-O or
/// ELF magic) — a truncated or garbage entry falls back to a fresh
/// compile instead of poisoning the link with an opaque error.
pub fn objectMagicOk(data: []const u8) bool {
if (data.len < 4) return false;
if (data[0] == 0x7f and data[1] == 'E' and data[2] == 'L' and data[3] == 'F') return true;
if (data[0] == 0xcf and data[1] == 0xfa and data[2] == 0xed and data[3] == 0xfe) return true; // Mach-O 64
if (data[0] == 0xce and data[1] == 0xfa and data[2] == 0xed and data[3] == 0xfe) return true; // Mach-O 32
return false;
}
fn loadCachedObject(path: [:0]const u8) ?c.LLVMMemoryBufferRef {
var buf: c.LLVMMemoryBufferRef = null;
var err_msg: [*c]u8 = null;
if (c.LLVMCreateMemoryBufferWithContentsOfFile(path.ptr, &buf, &err_msg) != 0) {
if (err_msg != null) c.LLVMDisposeMessage(err_msg);
return null;
}
const start = c.LLVMGetBufferStart(buf);
const size = c.LLVMGetBufferSize(buf);
if (start == null or size < 4) {
c.LLVMDisposeMemoryBuffer(buf);
return null;
}
const data = @as([*]const u8, @ptrCast(start))[0..size];
if (!objectMagicOk(data)) {
c.LLVMDisposeMemoryBuffer(buf);
return null;
}
return buf;
}
// Best-effort, never fails the build: write to a per-pid temp at the
// repo root, then copy into place (copyFile's make_path creates
// .sx-cache/ if needed) — same pattern as main.zig's object cache.
fn saveCachedObject(allocator: std.mem.Allocator, obj_buf: c.LLVMMemoryBufferRef, io: std.Io, cache_path: [:0]const u8) void {
const start = c.LLVMGetBufferStart(obj_buf);
const size = c.LLVMGetBufferSize(obj_buf);
if (start == null or size == 0) return;
const data = @as([*]const u8, @ptrCast(start))[0..size];
const tmp = std.fmt.allocPrint(allocator, ".sx-c-cache-tmp-{d}", .{std.c.getpid()}) catch return;
std.Io.Dir.writeFile(.cwd(), io, .{ .sub_path = tmp, .data = data }) catch return;
std.Io.Dir.copyFile(.cwd(), tmp, .cwd(), cache_path, io, .{ .make_path = true }) catch {};
std.Io.Dir.deleteFile(.cwd(), io, tmp) catch {};
}
/// Compile C sources to native object files (in memory), through a
/// persistent content-addressed cache (`.sx-cache/c-<key>.o`, default
/// on). A `#source` unit recompiles only when its cache key changes —
/// see `cSourceCacheKey` for what participates. Any cache-machinery
/// failure (unreadable input for hashing, corrupt entry) falls back to
/// a fresh compile; the cache can never fail a build.
/// Returns list of LLVMMemoryBufferRef (each containing a .o file).
pub fn compileCToObjects(
allocator: std.mem.Allocator,
io: std.Io,
infos: []const CImportInfo,
target_config: @import("target.zig").TargetConfig,
) ![]c.LLVMMemoryBufferRef {
var obj_bufs = std.ArrayList(c.LLVMMemoryBufferRef).empty;
var ver_maj: c_uint = 0;
var ver_min: c_uint = 0;
var ver_pat: c_uint = 0;
c.LLVMGetVersion(&ver_maj, &ver_min, &ver_pat);
const llvm_version = try std.fmt.allocPrint(allocator, "{d}.{d}.{d}", .{ ver_maj, ver_min, ver_pat });
const triple_slice: ?[]const u8 = if (target_config.triple) |t| std.mem.span(t) else null;
for (infos) |info| {
if (info.sources.len == 0) continue;
@@ -267,8 +326,10 @@ pub fn compileCToObjects(
try args_list.append(allocator, sysroot.ptr);
}
}
var inc_dirs = std.ArrayList([]const u8).empty;
for (info.includes) |inc| {
const dir = dirName(inc);
try inc_dirs.append(allocator, dir);
try args_list.append(allocator, (try allocPrintZ(allocator, "-I{s}", .{dir})).ptr);
}
for (info.defines) |def| {
@@ -284,7 +345,40 @@ pub fn compileCToObjects(
null;
const args_len: c_int = @intCast(args_list.items.len);
// Declared headers participate in the cache key BY CONTENT; an
// unreadable one disables caching for this unit, never the build.
var header_bytes = std.ArrayList([]const u8).empty;
var cache_ok = true;
for (info.includes) |inc| {
const hb = std.Io.Dir.readFileAlloc(.cwd(), io, inc, allocator, .limited(64 * 1024 * 1024)) catch {
cache_ok = false;
break;
};
try header_bytes.append(allocator, hb);
}
for (info.sources) |src| {
var cache_path: ?[:0]const u8 = null;
if (cache_ok) {
if (std.Io.Dir.readFileAlloc(.cwd(), io, src, allocator, .limited(64 * 1024 * 1024))) |src_bytes| {
const key = cSourceCacheKey(
src_bytes,
header_bytes.items,
info.defines,
info.flags,
inc_dirs.items,
llvm_version,
triple_slice,
target_config.sysroot,
);
cache_path = try std.fmt.allocPrintSentinel(allocator, ".sx-cache/c-{x:0>16}.o", .{key}, 0);
if (loadCachedObject(cache_path.?)) |cached| {
try obj_bufs.append(allocator, cached);
continue;
}
} else |_| {}
}
const src_z = try allocator.dupeZ(u8, src);
var err_msg: [*c]u8 = null;
@@ -302,6 +396,7 @@ pub fn compileCToObjects(
return error.CompileError;
}
if (cache_path) |cp| saveCachedObject(allocator, obj_buf, io, cp);
try obj_bufs.append(allocator, obj_buf);
}
}

View File

@@ -325,7 +325,7 @@ fn compileCForJIT(allocator: std.mem.Allocator, io: std.Io, comp: *sx.core.Compi
const c_infos = try comp.collectCImportSources();
if (c_infos.len == 0) return .{ .allocator = allocator };
const obj_bufs = try sx.c_import.compileCToObjects(allocator, c_infos, comp.target_config);
const obj_bufs = try sx.c_import.compileCToObjects(allocator, io, c_infos, comp.target_config);
return try sx.c_import.loadCObjectsForJIT(allocator, io, obj_bufs);
}
@@ -339,7 +339,7 @@ fn compileCForBuild(allocator: std.mem.Allocator, io: std.Io, comp: *sx.core.Com
return try sx.c_import.compileCWithEmcc(allocator, io, c_infos, comp.target_config, tmp_dir);
}
const obj_bufs = try sx.c_import.compileCToObjects(allocator, c_infos, comp.target_config);
const obj_bufs = try sx.c_import.compileCToObjects(allocator, io, c_infos, comp.target_config);
return try sx.c_import.writeCObjectFiles(allocator, io, obj_bufs, tmp_dir);
}