feat(C1.2): persistent content-addressed cache for compiled #source units
compileCToObjects now probes .sx-cache/c-<key>.o before invoking the embedded clang and writes fresh objects back (per-pid temp + copy, the main object cache's pattern). Default on for both JIT and AOT — the temp-compile-and-delete behavior it replaces was strictly worse. A cached entry must carry an object-file magic (Mach-O/ELF) or it falls back to a fresh compile; no cache failure can fail a build. Cold/warm verified via --time: the object compile disappears on the warm run.
This commit is contained in:
@@ -232,15 +232,74 @@ pub fn processCImport(
|
||||
// Native C compilation (compile to .o, not LLVM module)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Compile C sources to native object files (in memory).
|
||||
/// A cached entry must at least LOOK like an object file (Mach-O or
|
||||
/// ELF magic) — a truncated or garbage entry falls back to a fresh
|
||||
/// compile instead of poisoning the link with an opaque error.
|
||||
pub fn objectMagicOk(data: []const u8) bool {
|
||||
if (data.len < 4) return false;
|
||||
if (data[0] == 0x7f and data[1] == 'E' and data[2] == 'L' and data[3] == 'F') return true;
|
||||
if (data[0] == 0xcf and data[1] == 0xfa and data[2] == 0xed and data[3] == 0xfe) return true; // Mach-O 64
|
||||
if (data[0] == 0xce and data[1] == 0xfa and data[2] == 0xed and data[3] == 0xfe) return true; // Mach-O 32
|
||||
return false;
|
||||
}
|
||||
|
||||
fn loadCachedObject(path: [:0]const u8) ?c.LLVMMemoryBufferRef {
|
||||
var buf: c.LLVMMemoryBufferRef = null;
|
||||
var err_msg: [*c]u8 = null;
|
||||
if (c.LLVMCreateMemoryBufferWithContentsOfFile(path.ptr, &buf, &err_msg) != 0) {
|
||||
if (err_msg != null) c.LLVMDisposeMessage(err_msg);
|
||||
return null;
|
||||
}
|
||||
const start = c.LLVMGetBufferStart(buf);
|
||||
const size = c.LLVMGetBufferSize(buf);
|
||||
if (start == null or size < 4) {
|
||||
c.LLVMDisposeMemoryBuffer(buf);
|
||||
return null;
|
||||
}
|
||||
const data = @as([*]const u8, @ptrCast(start))[0..size];
|
||||
if (!objectMagicOk(data)) {
|
||||
c.LLVMDisposeMemoryBuffer(buf);
|
||||
return null;
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
// Best-effort, never fails the build: write to a per-pid temp at the
|
||||
// repo root, then copy into place (copyFile's make_path creates
|
||||
// .sx-cache/ if needed) — same pattern as main.zig's object cache.
|
||||
fn saveCachedObject(allocator: std.mem.Allocator, obj_buf: c.LLVMMemoryBufferRef, io: std.Io, cache_path: [:0]const u8) void {
|
||||
const start = c.LLVMGetBufferStart(obj_buf);
|
||||
const size = c.LLVMGetBufferSize(obj_buf);
|
||||
if (start == null or size == 0) return;
|
||||
const data = @as([*]const u8, @ptrCast(start))[0..size];
|
||||
const tmp = std.fmt.allocPrint(allocator, ".sx-c-cache-tmp-{d}", .{std.c.getpid()}) catch return;
|
||||
std.Io.Dir.writeFile(.cwd(), io, .{ .sub_path = tmp, .data = data }) catch return;
|
||||
std.Io.Dir.copyFile(.cwd(), tmp, .cwd(), cache_path, io, .{ .make_path = true }) catch {};
|
||||
std.Io.Dir.deleteFile(.cwd(), io, tmp) catch {};
|
||||
}
|
||||
|
||||
/// Compile C sources to native object files (in memory), through a
|
||||
/// persistent content-addressed cache (`.sx-cache/c-<key>.o`, default
|
||||
/// on). A `#source` unit recompiles only when its cache key changes —
|
||||
/// see `cSourceCacheKey` for what participates. Any cache-machinery
|
||||
/// failure (unreadable input for hashing, corrupt entry) falls back to
|
||||
/// a fresh compile; the cache can never fail a build.
|
||||
/// Returns list of LLVMMemoryBufferRef (each containing a .o file).
|
||||
pub fn compileCToObjects(
|
||||
allocator: std.mem.Allocator,
|
||||
io: std.Io,
|
||||
infos: []const CImportInfo,
|
||||
target_config: @import("target.zig").TargetConfig,
|
||||
) ![]c.LLVMMemoryBufferRef {
|
||||
var obj_bufs = std.ArrayList(c.LLVMMemoryBufferRef).empty;
|
||||
|
||||
var ver_maj: c_uint = 0;
|
||||
var ver_min: c_uint = 0;
|
||||
var ver_pat: c_uint = 0;
|
||||
c.LLVMGetVersion(&ver_maj, &ver_min, &ver_pat);
|
||||
const llvm_version = try std.fmt.allocPrint(allocator, "{d}.{d}.{d}", .{ ver_maj, ver_min, ver_pat });
|
||||
const triple_slice: ?[]const u8 = if (target_config.triple) |t| std.mem.span(t) else null;
|
||||
|
||||
for (infos) |info| {
|
||||
if (info.sources.len == 0) continue;
|
||||
|
||||
@@ -267,8 +326,10 @@ pub fn compileCToObjects(
|
||||
try args_list.append(allocator, sysroot.ptr);
|
||||
}
|
||||
}
|
||||
var inc_dirs = std.ArrayList([]const u8).empty;
|
||||
for (info.includes) |inc| {
|
||||
const dir = dirName(inc);
|
||||
try inc_dirs.append(allocator, dir);
|
||||
try args_list.append(allocator, (try allocPrintZ(allocator, "-I{s}", .{dir})).ptr);
|
||||
}
|
||||
for (info.defines) |def| {
|
||||
@@ -284,7 +345,40 @@ pub fn compileCToObjects(
|
||||
null;
|
||||
const args_len: c_int = @intCast(args_list.items.len);
|
||||
|
||||
// Declared headers participate in the cache key BY CONTENT; an
|
||||
// unreadable one disables caching for this unit, never the build.
|
||||
var header_bytes = std.ArrayList([]const u8).empty;
|
||||
var cache_ok = true;
|
||||
for (info.includes) |inc| {
|
||||
const hb = std.Io.Dir.readFileAlloc(.cwd(), io, inc, allocator, .limited(64 * 1024 * 1024)) catch {
|
||||
cache_ok = false;
|
||||
break;
|
||||
};
|
||||
try header_bytes.append(allocator, hb);
|
||||
}
|
||||
|
||||
for (info.sources) |src| {
|
||||
var cache_path: ?[:0]const u8 = null;
|
||||
if (cache_ok) {
|
||||
if (std.Io.Dir.readFileAlloc(.cwd(), io, src, allocator, .limited(64 * 1024 * 1024))) |src_bytes| {
|
||||
const key = cSourceCacheKey(
|
||||
src_bytes,
|
||||
header_bytes.items,
|
||||
info.defines,
|
||||
info.flags,
|
||||
inc_dirs.items,
|
||||
llvm_version,
|
||||
triple_slice,
|
||||
target_config.sysroot,
|
||||
);
|
||||
cache_path = try std.fmt.allocPrintSentinel(allocator, ".sx-cache/c-{x:0>16}.o", .{key}, 0);
|
||||
if (loadCachedObject(cache_path.?)) |cached| {
|
||||
try obj_bufs.append(allocator, cached);
|
||||
continue;
|
||||
}
|
||||
} else |_| {}
|
||||
}
|
||||
|
||||
const src_z = try allocator.dupeZ(u8, src);
|
||||
var err_msg: [*c]u8 = null;
|
||||
|
||||
@@ -302,6 +396,7 @@ pub fn compileCToObjects(
|
||||
return error.CompileError;
|
||||
}
|
||||
|
||||
if (cache_path) |cp| saveCachedObject(allocator, obj_buf, io, cp);
|
||||
try obj_bufs.append(allocator, obj_buf);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,7 +325,7 @@ fn compileCForJIT(allocator: std.mem.Allocator, io: std.Io, comp: *sx.core.Compi
|
||||
const c_infos = try comp.collectCImportSources();
|
||||
if (c_infos.len == 0) return .{ .allocator = allocator };
|
||||
|
||||
const obj_bufs = try sx.c_import.compileCToObjects(allocator, c_infos, comp.target_config);
|
||||
const obj_bufs = try sx.c_import.compileCToObjects(allocator, io, c_infos, comp.target_config);
|
||||
return try sx.c_import.loadCObjectsForJIT(allocator, io, obj_bufs);
|
||||
}
|
||||
|
||||
@@ -339,7 +339,7 @@ fn compileCForBuild(allocator: std.mem.Allocator, io: std.Io, comp: *sx.core.Com
|
||||
return try sx.c_import.compileCWithEmcc(allocator, io, c_infos, comp.target_config, tmp_dir);
|
||||
}
|
||||
|
||||
const obj_bufs = try sx.c_import.compileCToObjects(allocator, c_infos, comp.target_config);
|
||||
const obj_bufs = try sx.c_import.compileCToObjects(allocator, io, c_infos, comp.target_config);
|
||||
return try sx.c_import.writeCObjectFiles(allocator, io, obj_bufs, tmp_dir);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user