Harden file I/O and widen the VIR corpus

Makefile

@@ -192,8 +192,9 @@ config:
 check: check-stage0 check-stage2 check-abi-stage0 check-abi-stage2 \
 	check-vir-core check-vir-essential check-vir-frontend-stage0 check-vir-frontend-stage2 check-vir-fuzz check-vir-stage0 check-vir-stage2 \
 	check-vir-multi-input-stage0 check-vir-multi-input-stage2 \
-	check-vir-baseline check-vir-opt-baseline \
-	check-vir-corpus-stage0 check-fuzz-stage0 check-vir-frontend-policy-stage0
+	check-vir-opt-baseline \
+	check-vir-corpus-stage0 check-fuzz-stage0 \
+	check-vir-frontend-policy-stage0 check-vir-frontend-policy-stage2
 
 # Random programs checked against the host compiler and built by shecc.
 # FUZZ_SEEDS and FUZZ_START widen or move the run; a failing program is kept
@@ -206,9 +207,9 @@ check-fuzz-stage0: $(OUT)/$(STAGE0) tests/fuzz.sh tests/fuzz-gen.c
 # Only stage 0 is part of check: under emulation, stage 2 compiles the corpus
 # twice inside qemu, which costs more than the other checks.
 .PHONY: check-vir-corpus-stage0 check-vir-corpus-stage2
-check-vir-corpus-stage0: $(OUT)/$(STAGE0) tests/vir-corpus.sh
+check-vir-corpus-stage0: $(OUT)/$(STAGE0) tests/vir-corpus.sh tests/vir-corpus-exclusions.txt tests/vir-runtime-workloads.txt
 	TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-corpus.sh 0 $(ARCH)
-check-vir-corpus-stage2: $(OUT)/$(STAGE2) tests/vir-corpus.sh
+check-vir-corpus-stage2: $(OUT)/$(STAGE2) tests/vir-corpus.sh tests/vir-corpus-exclusions.txt tests/vir-runtime-workloads.txt
 	TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-corpus.sh 2 $(ARCH)
 
 .PHONY: check-vir-multi-input-stage0 check-vir-multi-input-stage2
@@ -292,11 +293,6 @@ check-all-targets:
 		$(MAKE) check ARCH="$$target_arch"; \
 	done
 
-.PHONY: vir-baseline
-vir-baseline: $(OUT)/$(STAGE0) tests/vir-baseline.sh tests/benchmark-time.sh \
-	tests/vir-baseline-workloads.txt
-	VIR_BASELINE_RUNS=$${VIR_BASELINE_RUNS:-5} tests/vir-baseline.sh
-
 .PHONY: vir-opt-baseline
 vir-opt-baseline: $(OUT)/$(STAGE0) tests/vir-opt-baseline.sh tests/benchmark-time.sh \
 	tests/vir-opt-baseline-workloads.txt
@@ -361,9 +357,7 @@ check-stage2: $(OUT)/$(STAGE2) tests/driver.sh
 
 check-sanitizer: $(OUT)/$(STAGE0)-sanitizer tests/driver.sh check-vir-sanitizer
 	$(VECHO) "  TEST STAGE 0 (with sanitizers)\n"
-	$(Q)cp $(OUT)/$(STAGE0)-sanitizer $(OUT)/shecc
-	tests/driver.sh 0 $(DYNLINK) $(BINDING)
-	$(Q)rm $(OUT)/shecc
+	SHECC_STAGE0="$(OUT)/$(STAGE0)-sanitizer" tests/driver.sh 0 $(DYNLINK) $(BINDING)
 
 check-abi-stage0: $(OUT)/$(STAGE0)
 	tests/$(ARCH)-abi.sh 0 $(DYNLINK) $(BINDING);
@@ -421,19 +415,15 @@ check-vir-stage2: $(OUT)/$(STAGE2) tests/vir-stage0.c src/vir.c src/vir.h
 
 .PHONY: check-vir-native-regressions-stage0 check-vir-native-regressions-stage2
 check: check-vir-native-regressions-stage0 check-vir-native-regressions-stage2 check-vir-lower
-check-vir-native-regressions-stage0: $(OUT)/$(STAGE0)
+check-vir-native-regressions-stage0: $(OUT)/$(STAGE0) tests/vir-native-regressions.sh tests/vir-runtime-workloads.txt
 	TARGET_EXEC="$(TARGET_EXEC)" tests/vir-native-regressions.sh $(OUT)/$(STAGE0)
-check-vir-native-regressions-stage2: $(OUT)/$(STAGE2)
+check-vir-native-regressions-stage2: $(OUT)/$(STAGE2) tests/vir-native-regressions.sh tests/vir-runtime-workloads.txt
 	TARGET_EXEC="$(TARGET_EXEC)" tests/vir-native-regressions.sh $(TARGET_EXEC) $(OUT)/$(STAGE2)
 
 .PHONY: check-vir-lower
 check-vir-lower: $(OUT)/$(STAGE0) tests/vir-lower.c tests/vir-lower-smoke.c tests/vir-lower.sh
 	CC="$(CC)" TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-lower.sh
 
-check-vir-baseline: $(OUT)/$(STAGE0) tests/vir-baseline.sh tests/benchmark-time.sh \
-	tests/vir-baseline-workloads.txt
-	VIR_BASELINE_RUNS=1 VIR_BASELINE_NO_TIMING=1 tests/vir-baseline.sh
-
 # Each workload is compiled twice at both levels by each stage, and the two
 # compilers must agree on the output and on the VIR dump. The stats lines are
 # left out because they report host-dependent arena sizes.

docs/vir.md

@@ -356,12 +356,33 @@ types, use-def links, edges, and ordered effects.
 | `make check-vir-native-regressions-stage0 check-vir-native-regressions-stage2` | Native frontend, ABI, and runtime regression fixtures with both compiler stages. |
 | `make check-vir-multi-input-stage0 check-vir-multi-input-stage2` | Translation-unit isolation, private symbol identity, and cross-input linking. |
 | `make check-vir-corpus-stage0` | Execution equivalence of the default build against O0 across the C fixture corpus. |
+| `make check-vir-opt-baseline` | Stage-0/stage-2 binary and dump determinism at O0 and the default level, including stats validation and compiler-sized workloads. |
 | `bash tests/vir-qualifiers.sh out/shecc` | Pointer and volatile qualifier execution and graph checks at O0 and the default level. |
 
 Corpus compilation defaults to 300 seconds per build. Set
 `VIR_CORPUS_COMPILE_TIMEOUT` for slower self-compiled runs under emulation;
 program execution retains its 10-second limit.
 
+The corpus checks every standalone `tests/*.c` fixture on every target,
+including the target-named direct-emission fixtures. Compilation failures and
+missing `main` functions fail the suite. Fixtures that cannot build standalone
+(host libraries, `--dynlink` against host libc, more than eight arguments
+outside arm64) and those owned by diagnostic, multi-input, and qualifier
+harnesses are recorded with their owner in `tests/vir-corpus-exclusions.txt`.
+An optional `standalone-arch` field names a target on which an excluded
+fixture builds, and the corpus checks it there. The external-libc fixtures run
+only under the x86-64 and AArch64 direct harnesses, at the default level.
+`tests/vir-runtime-workloads.txt` owns the zero-exit runtime cases executed at
+both optimization levels by both compiler stages. The two lists must remain
+disjoint and name existing files. Adding a C fixture automatically adds it to
+the corpus unless it is listed.
+
+`make check` includes optimization policy checks at both compiler stages.
+Use `make vir-opt-baseline` for timing and memory measurements; correctness
+checks disable timing.
+Sanitizer driver runs select their compiler with `SHECC_STAGE0` and leave the
+ordinary stage-0 binary available to the other suites.
+
 Changes must preserve execution, deterministic dumps and output, a clean
 verifier, and stage-1/stage-2 bootstrap identity. The AArch64 native harness
 adds its default dynamic QEMU sysroot only when the configured runner has not

lib/c.c

@@ -732,7 +732,8 @@ FILE *fopen(const char *filename, const char *mode)
 
 int fclose(FILE *stream)
 {
-    __syscall(__syscall_close, stream);
+    if (__syscall(__syscall_close, stream) < 0)
+        return -1;
     return 0;
 }
 

src/arm.c

@@ -29,7 +29,7 @@
 #include "defs.h"
 
 /* opcode */
-typedef enum {
+enum {
     arm_and = 0,
     arm_eor = 1,
     arm_sub = 2,
@@ -45,7 +45,7 @@ typedef enum {
     arm_mov = 13,
     arm_mvn = 15,
     arm_stmdb = 16
-} arm_op_t;
+};
 
 /* Condition code Reference:
  * https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/condition-codes-1-condition-flags-and-codes

src/arm64.c

@@ -122,9 +122,8 @@ int a64_cmp_reg_insn(bool sf, int rn, int rm)
 }
 
 /* The extended-register forms of 64-bit ADD and SUB. UXTX is the plain form
- * that may name SP; SXTW widens a W register operand as it is added.
+ * that may name SP.
  */
-#define A64_EXT_SXTW 6
 #define A64_EXT_UXTX 3
 
 int a64_add_ext_insn(int rd, int rn, int rm, int option)

src/defs.h

@@ -1248,10 +1248,7 @@ typedef struct {
 
 bool op_is_binary_alu(opcode_t op);
 bool op_writes_dest(opcode_t op);
-bool op_is_integer_binary(opcode_t op);
 bool op_is_comparison(opcode_t op);
-bool op_is_scalar_unary(opcode_t op);
-bool op_is_scalar_cast(opcode_t op);
 const char *opcode_mnemonic(opcode_t op);
 basic_block_t *bb_sole_pred(const basic_block_t *bb);
 

src/elf.c

@@ -1040,8 +1040,8 @@ void elf_postprocess(void)
 #ifdef HOST_BUFFERED_STDIO
 void elf_write_all(FILE *fp, const char *buf, int len)
 {
-    if (len > 0)
-        fwrite(buf, 1, len, fp);
+    if (len > 0 && (int) fwrite(buf, 1, len, fp) != len)
+        usage_error("Unable to write output file");
 }
 #else
 void elf_write_all(FILE *fp, const char *buf, int len)
@@ -1051,7 +1051,7 @@ void elf_write_all(FILE *fp, const char *buf, int len)
     while (off < len) {
         int n = __syscall(__syscall_write, fp, buf + off, len - off);
         if (n <= 0)
-            return;
+            usage_error("Unable to write output file");
         off += n;
     }
 }
@@ -1139,7 +1139,9 @@ void elf_generate(const char *outfile)
     elf_write_all(fp, elf_shstrtab->elements, elf_shstrtab->size);
     elf_write_all(fp, elf_section_header->elements, elf_section_header->size);
 #endif
-    fclose(fp);
+    /* Buffered runtimes may defer the actual write until fclose(). */
+    if (fclose(fp) != 0)
+        usage_error("Unable to close output file");
 
     /* A generated ELF is meant to be runnable directly, but the mode it gets
      * depends on which libc opened it, not on how the output is linked: the

src/lexer.c

@@ -273,59 +273,90 @@ char read_char(strbuf_t *buf)
     return source_char_at(buf, buf->size);
 }
 
-/* Fill @dst with @len bytes of @f, returning how many arrived.
- *
- * The whole file is wanted and its size is already known, so it is asked for in
- * one piece. Reading it a line at a time instead cost a library call and a
- * second copy for every line of every source file -- and shecc's own libc has
- * no buffer behind fgets(), so each of those lines was a read(2) as well.
+/* Read up to @len bytes, distinguishing an I/O error from end of input. Static
+ * self-hosted builds use read(2) directly to avoid byte-at-a-time libc I/O.
  */
 #ifdef HOST_BUFFERED_STDIO
-int file_read_all(FILE *f, char *dst, int len)
+/* The dynamically linked compiler uses the host's stream error indicator; the
+ * embedded, unbuffered libc has no such state. Host builds take the declaration
+ * from <stdio.h>, where ferror may be a macro.
+ */
+#ifdef __SHECC__
+int ferror(FILE *stream);
+#endif
+
+int file_read_chunk(FILE *f, char *dst, int len)
 {
-    if (len <= 0)
-        return 0;
-    return fread(dst, 1, len, f);
+    int n = fread(dst, 1, len, f);
+    return ferror(f) ? -1 : n;
 }
 #else
-int file_read_all(FILE *f, char *dst, int len)
+int file_read_chunk(FILE *f, char *dst, int len)
 {
-    int got = 0;
-
-    while (got < len) {
-        int n = __syscall(__syscall_read, f, dst + got, len - got);
-        if (n <= 0)
-            return got;
-        got += n;
-    }
-    return got;
+    return __syscall(__syscall_read, f, dst, len);
 }
 #endif
 
-/* @filename's contents, or NULL when it cannot be opened. */
+/* Reports that @filename could not be opened or read: a usage error. */
+__noreturn void source_file_error(const char *action, const char *filename)
+{
+    char msg[MAX_LINE_LEN];
+
+    snprintf(msg, MAX_LINE_LEN, "cannot %s source file '%s'", action, filename);
+    usage_error(msg);
+}
+
+#define SOURCE_BUF_INIT 16384
+
+/* Set when the last try_read_file() opened its file but could not read it, so a
+ * caller can tell "cannot read" from "cannot open".
+ */
+bool source_read_failed;
+
+/* @filename's contents, or NULL when it cannot be opened or read. */
 strbuf_t *try_read_file(const char *filename)
 {
     FILE *f = fopen(filename, "rb");
     strbuf_t *src;
 
+    source_read_failed = false;
     if (!f)
         return NULL;
 
-    fseek(f, 0, SEEK_END);
-    int len = ftell(f);
-    src = strbuf_create(len + 1);
-    fseek(f, 0, SEEK_SET);
+    src = strbuf_create(SOURCE_BUF_INIT);
+    if (!src)
+        fatal("Unable to allocate source buffer");
 
-    src->size = file_read_all(f, src->elements, len);
+    /* Pipes and procfs files do not have a useful seek-derived length. Read
+     * through EOF into whatever room the buffer has, keeping one byte free for
+     * the lexer's NUL sentinel.
+     */
+    while (true) {
+        if (!strbuf_extend(src, 1))
+            fatal("Unable to grow source buffer");
+        int n = file_read_chunk(f, src->elements + src->size,
+                                src->capacity - src->size - 1);
+        if (n < 0) {
+            fclose(f);
+            strbuf_free(src);
+            source_read_failed = true;
+            return NULL;
+        }
+        if (!n)
+            break;
+        src->size += n;
+    }
 
     fclose(f);
-    src->elements[len] = '\0';
+    src->elements[src->size] = '\0';
+    /* The lexer treats the byte at capacity - 1 as the end of input. */
+    src->capacity = src->size + 1;
     return src;
 }
 
 /* @filename's contents, read once and kept for every later request; NULL when
- * it cannot be opened, so that a caller with a better place to report the
- * mistake can.
+ * it cannot be opened or read, so that a caller with a better place to report
+ * the mistake can.
  */
 strbuf_t *try_get_file_buf(char *filename)
 {
@@ -343,14 +374,8 @@ strbuf_t *get_file_buf(char *filename)
 {
     strbuf_t *buf = try_get_file_buf(filename);
 
-    /* A file that cannot be opened is a mistake in how the compiler was
-     * invoked, not a broken invariant, so it exits rather than abort()ing.
-     */
-    if (!buf) {
-        printf("[Error]: cannot open source file '%s'\n", filename);
-        fflush(NULL);
-        exit(1);
-    }
+    if (!buf)
+        source_file_error(source_read_failed ? "read" : "open", filename);
     return buf;
 }
 

src/preprocessor.c

@@ -2498,8 +2498,10 @@ static token_t *pp_preprocess_body(token_t *tk, preprocess_ctx_t *ctx)
                 continue;
             }
 
-            if (!angle_header) {
-                /* Quoted headers are relative to the physical source path. */
+            if (!angle_header && inclusion_path[0] != '/') {
+                /* Relative quoted headers use the physical source path;
+                 * absolute names already identify their own location.
+                 */
                 char path[MAX_LINE_LEN];
                 const char *file = include_tk->location.file->physical;
                 int c = strlen(file) - 1;
@@ -2528,7 +2530,8 @@ static token_t *pp_preprocess_body(token_t *tk, preprocess_ctx_t *ctx)
              * lexer, under the name the lexer will ask for.
              */
             if (!try_get_file_buf(normalize_filename(inclusion_path)))
-                error_at("Included file cannot be found",
+                error_at(source_read_failed ? "Included file cannot be read"
+                                            : "Included file cannot be found",
                          &include_tk->location);
 
             tk = pp_lex_expect_token(tk, T_newline, true);