diff --git a/autobuild/check_dataset_allowlist.py b/autobuild/check_dataset_allowlist.py new file mode 100644 index 0000000..e3d993c --- /dev/null +++ b/autobuild/check_dataset_allowlist.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Leg 4 of PyAutoBuild#126 — dataset allowlist guard. + +Asserts that every tracked file under `dataset/` in the current workspace is +covered by that workspace's `.gitignore` allowlist (the `!dataset/...` re-include +lines). This makes the `git add -f` leak (fixed in pre_build) unable to recur: if +a simulated dataset is ever force-committed again, a `pre_build` release run fails +loudly here instead of shipping ~10 MB of runtime-generated data to users. + +Allowlist-based, NOT `git check-ignore`-based: `git check-ignore` never flags an +already-tracked file (a tracked file is by definition not ignored) and mishandles +`!`-negated paths, so it cannot detect this leak. We compare tracked paths against +the parsed allowlist directly. + +Allowlist-presence-gated: a workspace whose `.gitignore` has a bare `dataset/` +rule with no `!dataset/...` re-includes (autofit_workspace, HowTo*) has not yet +adopted the allowlist regime (Group B, PyAutoBuild#126) — the guard skips it with +a notice rather than failing, until that repo opts in. + +Run from a workspace root. Exit 0 = clean/skipped, 1 = violation. +""" +import re +import subprocess +import sys +from pathlib import Path + + +def allowlist_prefixes(gitignore: Path): + prefixes = [] + has_dataset_ignore = False + if not gitignore.exists(): + return prefixes, has_dataset_ignore + for line in gitignore.read_text().splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + if re.match(r"^/?dataset(/\*\*|/)?\s*$", s): + has_dataset_ignore = True + if s.startswith("!") and "dataset" in s: + p = re.sub(r"/\*+$", "", s[1:].strip().lstrip("/")).rstrip("/") + prefixes.append(p) + return prefixes, has_dataset_ignore + + +def tracked_dataset_files(): + out = subprocess.run( + ["git", "ls-files", "dataset/"], capture_output=True, text=True + ).stdout + return [f for f in out.splitlines() if f and not f.endswith("dataset/.gitignore")] + + +def main() -> int: + prefixes, has_dataset_ignore = allowlist_prefixes(Path(".gitignore")) + tracked = tracked_dataset_files() + + if not tracked: + print("[dataset-allowlist] no tracked dataset/ files — OK") + return 0 + + if not prefixes: + # bare `dataset/` (or none) with no re-includes: Group B, not yet in regime + note = "bare `dataset/` ignore" if has_dataset_ignore else "no dataset allowlist" + print( + f"[dataset-allowlist] SKIP — {note}; workspace has not adopted the " + f"allowlist regime (Group B, PyAutoBuild#126). {len(tracked)} tracked files." + ) + return 0 + + def allowed(f: str) -> bool: + return any(f == p or f.startswith(p + "/") for p in prefixes) + + violations = sorted({"/".join(f.split("/")[:3]) for f in tracked if not allowed(f)}) + if violations: + print( + "[dataset-allowlist] FAIL — tracked dataset dirs outside the " + f".gitignore allowlist ({len(violations)}):", + file=sys.stderr, + ) + for d in violations: + print(f" {d}", file=sys.stderr) + print( + "\nThese are simulated datasets that must be generated at runtime, not " + "committed. Purge them (`git rm -r`) — they self-provision via " + "should_simulate()/`.exists()` — or add a `!dataset//**` allowlist " + "line if the data is real/committed-by-design. See PyAutoBuild#126.", + file=sys.stderr, + ) + return 1 + + print( + f"[dataset-allowlist] OK — {len(tracked)} tracked dataset files, all within " + f"the allowlist ({len(prefixes)} patterns)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pre_build.sh b/pre_build.sh index 163cb33..f6fafb0 100644 --- a/pre_build.sh +++ b/pre_build.sh @@ -60,8 +60,19 @@ run_workspace() { PYTHONPATH="$PYTHONPATH_EXTRA" python "$AUTOBUILD/generate.py" "$project" fi + echo " Checking dataset allowlist (PyAutoBuild#126 leg 4)..." + python3 "$AUTOBUILD/check_dataset_allowlist.py" || { + echo " ABORT: tracked dataset/ contains non-allowlisted simulated data." >&2 + exit 1 + } + echo " Staging safe directories..." - [ -d dataset ] && git add -f dataset/ + # Honor .gitignore for dataset/ — each workspace ignores dataset/** except a + # small allowlist of REAL observational data (un-ignored via ! lines). A prior + # `git add -f` here force-committed simulated datasets that are meant to be + # generated at runtime via al.util.dataset.should_simulate(); only allowlisted + # real data may be staged. See PyAutoBuild#126. + [ -d dataset ] && git add dataset/ for d in config notebooks scripts; do [ -d "$d" ] && git add "$d/" done