78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
"""Run the released LiveKit stages for a user-owned, non-publishable experiment."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from stage_evidence import run_stage
|
|
|
|
RUN_DIR = Path("/run")
|
|
INPUTS_DIR = Path("/inputs")
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as file:
|
|
for block in iter(lambda: file.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def verify_local_cache() -> None:
|
|
lock = json.loads((RUN_DIR / "sources.lock.json").read_text(encoding="utf-8"))
|
|
sources = lock.get("sources")
|
|
if not isinstance(sources, list) or len(sources) != 1:
|
|
raise ValueError("local source lock must contain exactly one cache inventory")
|
|
files = sources[0].get("files") if isinstance(sources[0], dict) else None
|
|
if not isinstance(files, list):
|
|
raise ValueError("local source lock has no content-addressed file inventory")
|
|
expected_paths: set[str] = set()
|
|
for item in files:
|
|
if not isinstance(item, dict):
|
|
raise ValueError("local source inventory entry is invalid")
|
|
relative = item.get("relative_path")
|
|
expected_hash = item.get("sha256")
|
|
expected_size = item.get("size_bytes")
|
|
if (
|
|
not isinstance(relative, str)
|
|
or not relative
|
|
or Path(relative).is_absolute()
|
|
or ".." in Path(relative).parts
|
|
or not isinstance(expected_hash, str)
|
|
or not isinstance(expected_size, int)
|
|
):
|
|
raise ValueError("local source inventory entry is unsafe")
|
|
path = INPUTS_DIR / relative
|
|
if not path.is_file() or path.is_symlink():
|
|
raise ValueError(f"locked local cache file is missing: {relative}")
|
|
if path.stat().st_size != expected_size or _sha256(path) != expected_hash:
|
|
raise ValueError(f"locked local cache file changed: {relative}")
|
|
expected_paths.add(relative)
|
|
actual_paths = {
|
|
path.relative_to(INPUTS_DIR).as_posix()
|
|
for path in INPUTS_DIR.rglob("*")
|
|
if path.is_file()
|
|
}
|
|
if actual_paths != expected_paths:
|
|
raise ValueError("local cache contains unsealed files")
|
|
|
|
|
|
def main(config_path: Path) -> int:
|
|
request = json.loads((RUN_DIR / "request.json").read_text(encoding="utf-8"))
|
|
payload = request.get("request", {})
|
|
if payload.get("run_kind") != "local_experiment":
|
|
raise ValueError("local experiment wrapper requires run_kind=local_experiment")
|
|
verify_local_cache()
|
|
run_stage("provenance", ["python", "/opt/jr-wakeword/record-image-provenance.py"])
|
|
for stage in ("generate", "augment", "train", "export", "eval"):
|
|
run_stage(stage, ["livekit-wakeword", stage, str(config_path)])
|
|
run_stage("finalize", ["python", "/opt/jr-wakeword/finalize-run.py", str(config_path)])
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(Path(sys.argv[1])))
|