feat: publish downloadable wakeword trainer source
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
# The matching lock records the tag provenance, platform, Python, PyTorch,
|
||||
# CUDA, and cuDNN compatibility. Never replace this with a mutable tag.
|
||||
ARG BASE_IMAGE=docker.io/pytorch/pytorch@sha256:639b8229ccfd8a3aa803cf49c33d6d6fe406750d79aaf723fe8c0eb1060d8cff
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG BASE_IMAGE
|
||||
ARG PYTHON_VERSION=3.11
|
||||
LABEL org.opencontainers.image.title="jr-wakeword-training" \
|
||||
org.opencontainers.image.description="Offline LiveKit WakeWord 0.2.1 training lane" \
|
||||
org.opencontainers.image.source="https://github.com/livekit/livekit-wakeword" \
|
||||
org.opencontainers.image.revision="1ec7f680df30ff4ca0ebae6b5983441e94b10980" \
|
||||
org.opencontainers.image.version="0.2.1"
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
NLTK_DATA=/usr/local/share/nltk_data
|
||||
|
||||
RUN case "${BASE_IMAGE}" in *@sha256:*) ;; *) echo "BASE_IMAGE must be digest-pinned" >&2; exit 64;; esac \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends espeak-ng libsndfile1 ffmpeg sox time \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& python -c "import sys; assert sys.version_info[:2] == (${PYTHON_VERSION%.*}, ${PYTHON_VERSION#*.})"
|
||||
|
||||
WORKDIR /opt/jr-wakeword
|
||||
|
||||
# The complete hash lock is generated with uv and reviewed in source control.
|
||||
# Network access is permitted only at image build time; a launched training
|
||||
# container is network-isolated and never resolves or installs dependencies.
|
||||
COPY deploy/wakeword-training/requirements.wheelhouse.lock ./requirements.wheelhouse.lock
|
||||
RUN python -m pip install --require-hashes -r requirements.wheelhouse.lock \
|
||||
&& python -c "from importlib.metadata import version; assert version('livekit-wakeword') == '0.2.1'; assert version('torch') == version('torchaudio')" \
|
||||
&& python -c "import nltk; assert nltk.download('cmudict', download_dir='/usr/local/share/nltk_data', quiet=True, raise_on_error=True)"
|
||||
|
||||
COPY scripts/wakeword_training_image.py ./wakeword_training_image.py
|
||||
COPY deploy/wakeword-training/run-commercial-pipeline.py ./run-commercial-pipeline.py
|
||||
COPY deploy/wakeword-training/run-local-experiment.py ./run-local-experiment.py
|
||||
COPY deploy/wakeword-training/run-quarantined-smoke.py ./run-quarantined-smoke.py
|
||||
COPY deploy/wakeword-training/stage_evidence.py ./stage_evidence.py
|
||||
COPY deploy/wakeword-training/record-image-provenance.py ./record-image-provenance.py
|
||||
COPY deploy/wakeword-training/finalize-run.py ./finalize-run.py
|
||||
|
||||
# Do not add listener extras, microphone SDK packages, gateway source, gateway credentials,
|
||||
# Docker socket access, or published ports in this one-shot image.
|
||||
# DockerTrainingExecutor supplies the explicit commercial/quarantined wrapper
|
||||
# as argv. CMD keeps that argv from being appended to a fixed ENTRYPOINT.
|
||||
CMD ["python", "/opt/jr-wakeword/run-commercial-pipeline.py", "/run/resolved-config.yaml"]
|
||||
@@ -0,0 +1,61 @@
|
||||
# Wakeword Training Image
|
||||
|
||||
This directory packages the one-shot, GPU-only image used by both the offline
|
||||
operator runner and downloadable local trainer. It is not part of `compose.yaml`,
|
||||
exposes no port, and contains no gateway credentials or listener dependencies.
|
||||
|
||||
The image uses LiveKit WakeWord `0.2.1`, source revision
|
||||
`1ec7f680df30ff4ca0ebae6b5983441e94b10980`, and source-distribution SHA-256
|
||||
`cf2d9cf4867812c06788f64c15e49abd909d9d6291f0a13f1c3f9cb649fa6127`.
|
||||
|
||||
The gateway's `.env` deliberately contains only `WAKEWORD_CATALOG_*` settings.
|
||||
Do not add runner GPU, image, work-root, timeout, or retention values to it: the
|
||||
host-only runner receives those as explicit invocation inputs and refuses to run
|
||||
without `--enable`. This directory is neither a Compose service nor normal CI input.
|
||||
|
||||
Before an operator builds it, run:
|
||||
|
||||
```bash
|
||||
scripts/build-wakeword-training-image \
|
||||
registry.example/jr-wakeword-training:0.2.1
|
||||
```
|
||||
|
||||
The command builds from the reviewed, complete
|
||||
`requirements.wheelhouse.lock` with SHA-256 hashes; it never rewrites that lock.
|
||||
Refresh the lock only in a separate reviewed dependency-update change, then
|
||||
commit it before building a release image. It becomes part of each run's
|
||||
provenance. The pinned base image is in `base-image.lock.json`; the image build
|
||||
may install the reviewed dependencies, but a training container runs with
|
||||
`--network none`. Do not run this setup during normal gateway CI.
|
||||
|
||||
The runtime is designed for `--network none`, a read-only `/inputs` bind mount,
|
||||
a writable `/run` bind mount, a read-only container root, and no Linux
|
||||
capabilities. A commercial run validates `sources.lock.json`, rejects default
|
||||
ACAV, unknown-license RIR mirrors, and unapproved Piper/VoxCPM outputs, copies
|
||||
only hash-locked files into `/run`, then invokes the released CLI's `augment`,
|
||||
`train`, `export`, and `eval` stages. It never calls `livekit-wakeword setup`
|
||||
or `generate` in the commercial lane.
|
||||
|
||||
For compatibility with the released CLI only, source-lock-approved
|
||||
general-negative feature bundles are copied to its fixed ACAV/validation
|
||||
filenames beneath the isolated staging directory. The recorded source lock and
|
||||
materialization manifest retain their real source IDs and hashes; no ACAV data
|
||||
is admitted or downloaded.
|
||||
|
||||
`quarantined_smoke` is a separately named wrapper and stays permanently
|
||||
non-promotable. It may exercise the upstream `run` command only after its
|
||||
operator has separately prepared test data; network isolation still prevents
|
||||
an implicit setup/download in the run container.
|
||||
|
||||
`local_experiment` is the downloadable-trainer lane. A separate, explicit setup
|
||||
container may populate the user's versioned LiveKit cache with normal network
|
||||
access. The training container then runs the released `generate`, `augment`,
|
||||
`train`, `export`, and `eval` stages with `--network none`. It produces a local
|
||||
result, never `publish-candidate.json`. See
|
||||
[`docs/wakeword-trainer.md`](../../docs/wakeword-trainer.md).
|
||||
|
||||
The source-lock format is documented by `source-lock.schema.json`; runtime
|
||||
validation additionally verifies every source/terms hash, publisher, version,
|
||||
revision, attribution, commercial disposition, approver, and partition. Raw
|
||||
inputs and features remain in `/run` evidence only and are never serving
|
||||
artifacts.
|
||||
@@ -0,0 +1,15 @@
|
||||
# Third-party notices
|
||||
|
||||
The training image integrates
|
||||
[LiveKit WakeWord 0.2.1](https://github.com/livekit/livekit-wakeword/tree/v0.2.1),
|
||||
licensed under Apache License 2.0. Its pinned source revision and source-distribution checksum are
|
||||
recorded in the image and every run's provenance.
|
||||
|
||||
The image also contains the exact Python and OS packages recorded in
|
||||
`requirements.wheelhouse.lock`, the pinned base-image lock, and per-run image provenance. Those
|
||||
components retain their own licenses.
|
||||
|
||||
Training models, speech corpora, generated speech, background audio, room impulse responses, and
|
||||
general-negative feature datasets are separate works with separate terms. The downloadable source
|
||||
archive does not contain the approximately 17 GB LiveKit setup cache and does not grant rights to
|
||||
redistribute any downloaded or generated training data.
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"image": "docker.io/pytorch/pytorch@sha256:639b8229ccfd8a3aa803cf49c33d6d6fe406750d79aaf723fe8c0eb1060d8cff",
|
||||
"source_tag": "pytorch/pytorch:2.7.1-cuda12.6-cudnn9-devel",
|
||||
"platform": "linux/amd64",
|
||||
"python": "3.11",
|
||||
"pytorch": "2.7.1+cu126",
|
||||
"cuda": "12.6",
|
||||
"cudnn": "9"
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Seal upstream outputs into the runner/catalog evidence contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
RUN_DIR = Path("/run")
|
||||
REQUIRED_EVALUATION = (
|
||||
"phrase_duration_evidence",
|
||||
"release_negative_hours",
|
||||
"release_false_activations",
|
||||
"speaker_macro_recall",
|
||||
"p95_detection_latency_ms",
|
||||
)
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _load(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{path.name} must be a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def _canonical(value: dict[str, Any]) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def _validate_onnx(path: Path) -> None:
|
||||
import onnx
|
||||
|
||||
model = onnx.load(path)
|
||||
onnx.checker.check_model(model)
|
||||
if not any(item.version == 18 for item in model.opset_import):
|
||||
raise ValueError("classifier must use ONNX opset 18")
|
||||
inputs = {item.name: item for item in model.graph.input}
|
||||
outputs = {item.name for item in model.graph.output}
|
||||
embeddings = inputs.get("embeddings")
|
||||
if embeddings is None or "score" not in outputs:
|
||||
raise ValueError("classifier must expose embeddings input and score output")
|
||||
dimensions = embeddings.type.tensor_type.shape.dim
|
||||
if embeddings.type.tensor_type.elem_type != onnx.TensorProto.FLOAT or [
|
||||
item.dim_value for item in dimensions[-2:]
|
||||
] != [16, 96]:
|
||||
raise ValueError("classifier must accept FP32 (batch,16,96) embeddings")
|
||||
|
||||
|
||||
def _upstream_eval(output_dir: Path) -> dict[str, Any] | None:
|
||||
candidates = sorted(output_dir.rglob("*_eval.json"))
|
||||
if len(candidates) != 1:
|
||||
return None
|
||||
return _load(candidates[0])
|
||||
|
||||
|
||||
def _evaluation(
|
||||
run_kind: str, primary_phrase: str, variants: list[str], output_dir: Path
|
||||
) -> dict[str, Any]:
|
||||
if run_kind == "commercial":
|
||||
value = _load(RUN_DIR / "release-evaluation.json")
|
||||
if any(key not in value for key in REQUIRED_EVALUATION):
|
||||
raise ValueError("release-evaluation.json is incomplete")
|
||||
phrases = [primary_phrase, *variants]
|
||||
durations = value["phrase_duration_evidence"]
|
||||
if not isinstance(durations, dict) or any(
|
||||
phrase not in durations
|
||||
or not isinstance(durations[phrase], list)
|
||||
or not durations[phrase]
|
||||
or any(
|
||||
not isinstance(item, (int, float)) or not 0 < item <= 1.8
|
||||
for item in durations[phrase]
|
||||
)
|
||||
for phrase in phrases
|
||||
):
|
||||
raise ValueError("phrase duration evidence is incomplete or exceeds 1.80 seconds")
|
||||
return {**value, "schema_version": 1}
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"phrase_duration_evidence": {phrase: [1.0] for phrase in [primary_phrase, *variants]},
|
||||
"release_negative_hours": 0.0,
|
||||
"release_false_activations": 0,
|
||||
"speaker_macro_recall": 0.0,
|
||||
"p95_detection_latency_ms": 0.0,
|
||||
"quarantined": run_kind == "quarantined_smoke",
|
||||
}
|
||||
upstream = _upstream_eval(output_dir)
|
||||
if upstream is not None:
|
||||
result["upstream_metrics"] = upstream
|
||||
return result
|
||||
|
||||
|
||||
def _upstream_evaluation_artifacts(output_dir: Path, *, commercial: bool) -> list[dict[str, str]]:
|
||||
"""Record the raw artefacts emitted by LiveKit's ``eval`` stage.
|
||||
|
||||
The catalog consumes our compact JSON evidence, but keeping hashes of the
|
||||
upstream output makes that evidence independently reviewable. We do not
|
||||
copy or reinterpret the files: LiveKit keeps its native filenames and
|
||||
format under the sealed output directory.
|
||||
"""
|
||||
|
||||
candidates = [
|
||||
path
|
||||
for path in output_dir.rglob("*")
|
||||
if path.is_file()
|
||||
and path.name not in {"manifest.json", "evaluation.json", "classifier.onnx"}
|
||||
and ("eval" in path.name.lower() or "det" in path.name.lower())
|
||||
]
|
||||
if commercial and not candidates:
|
||||
raise ValueError("commercial output is missing an upstream LiveKit evaluation artifact")
|
||||
return [
|
||||
{
|
||||
"path": path.relative_to(output_dir).as_posix(),
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
for path in sorted(candidates)
|
||||
]
|
||||
|
||||
|
||||
def main(config_path: Path) -> int:
|
||||
request = _load(RUN_DIR / "request.json")["request"]
|
||||
if not isinstance(request, dict):
|
||||
raise ValueError("request payload is invalid")
|
||||
run_kind = str(request["run_kind"])
|
||||
if run_kind not in {"commercial", "local_experiment", "quarantined_smoke"}:
|
||||
raise ValueError("request run_kind is invalid")
|
||||
config = _load(config_path)
|
||||
family_slug = config.get("family_slug")
|
||||
version = config.get("version")
|
||||
threshold = config.get("threshold")
|
||||
if (
|
||||
not isinstance(family_slug, str)
|
||||
or not isinstance(version, int)
|
||||
or not (isinstance(threshold, (int, float)) or threshold == "auto")
|
||||
):
|
||||
raise ValueError("effective config requires family_slug, version, and threshold")
|
||||
raw_variants = request["variants"]
|
||||
if not isinstance(raw_variants, list) or not all(
|
||||
isinstance(item, str) for item in raw_variants
|
||||
):
|
||||
raise ValueError("request variants are invalid")
|
||||
variants = list(raw_variants)
|
||||
primary_phrase = str(request["primary_phrase"])
|
||||
output_dir = RUN_DIR / "output"
|
||||
evaluation = _evaluation(run_kind, primary_phrase, variants, output_dir)
|
||||
if threshold == "auto":
|
||||
upstream_metrics = evaluation.get("upstream_metrics")
|
||||
if not isinstance(upstream_metrics, dict) or not isinstance(
|
||||
upstream_metrics.get("optimal_threshold"), (int, float)
|
||||
):
|
||||
raise ValueError("automatic threshold requires one upstream evaluation result")
|
||||
threshold = float(upstream_metrics["optimal_threshold"])
|
||||
classifiers = [path for path in output_dir.rglob("*.onnx") if path.is_file()]
|
||||
if len(classifiers) != 1:
|
||||
raise ValueError("upstream output must contain exactly one ONNX classifier")
|
||||
_validate_onnx(classifiers[0])
|
||||
classifier = output_dir / "classifier.onnx"
|
||||
if classifiers[0] != classifier:
|
||||
shutil.copyfile(classifiers[0], classifier)
|
||||
image = _load(RUN_DIR / "image-provenance.json")
|
||||
source = _load(RUN_DIR / "sources.lock.json")
|
||||
upstream_artifacts = _upstream_evaluation_artifacts(
|
||||
output_dir, commercial=run_kind == "commercial"
|
||||
)
|
||||
evaluation["upstream_cli_artifacts"] = upstream_artifacts
|
||||
publication_status = {
|
||||
"commercial": "review_required",
|
||||
"local_experiment": "local_only",
|
||||
"quarantined_smoke": "quarantined",
|
||||
}[run_kind]
|
||||
evaluation["promotion"] = {
|
||||
"eligible": False,
|
||||
"status": publication_status,
|
||||
}
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"family_slug": family_slug,
|
||||
"version": version,
|
||||
"primary_phrase": primary_phrase,
|
||||
"phrase_variants": variants,
|
||||
"language": "en",
|
||||
"threshold": float(threshold),
|
||||
"debounce_ms": 2000,
|
||||
"runtime_compatibility": {
|
||||
"format": "onnx", "architecture": "conv_attention", "precision": "fp32", "opset": 18
|
||||
},
|
||||
"runner_provenance": {
|
||||
"image_digest": request["runner_image_digest"],
|
||||
"livekit_wakeword_version": "0.2.1",
|
||||
"source_revision": "1ec7f680df30ff4ca0ebae6b5983441e94b10980",
|
||||
"dependency_lock_sha256": image["dependency_lock"]["sha256"],
|
||||
},
|
||||
"source_provenance": {
|
||||
"source_lock_sha256": _sha256(RUN_DIR / "sources.lock.json"),
|
||||
"approved": run_kind == "commercial",
|
||||
"inventory": [
|
||||
item.get("source_id", item.get("id", "unidentified"))
|
||||
for item in source.get("sources", [])
|
||||
if isinstance(item, dict)
|
||||
],
|
||||
},
|
||||
"evaluation_summary": evaluation,
|
||||
"artifacts": {"onnx_sha256": _sha256(classifier), "evaluation_sha256": ""},
|
||||
}
|
||||
evaluation_path = output_dir / "evaluation.json"
|
||||
evaluation_path.write_bytes(_canonical(evaluation))
|
||||
manifest["artifacts"]["evaluation_sha256"] = _sha256(evaluation_path)
|
||||
(RUN_DIR / "output" / "manifest.json").write_bytes(_canonical(manifest))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"preset_version": "manual-poc-v1",
|
||||
"description": "Measured JRich manual-POC profile; calibrate every new Wake Phrase independently.",
|
||||
"pipeline_config": {
|
||||
"threshold": "auto",
|
||||
"n_samples": 10000,
|
||||
"n_samples_val": 2000,
|
||||
"n_background_samples": 1000,
|
||||
"n_background_samples_val": 200,
|
||||
"tts_batch_size": 10,
|
||||
"noise_scales": [0.98],
|
||||
"noise_scale_ws": [0.98],
|
||||
"length_scales": [0.75, 1.0, 1.25],
|
||||
"slerp_weights": [0.2, 0.35, 0.5, 0.65, 0.8],
|
||||
"augmentation": {
|
||||
"clip_duration": 2.0,
|
||||
"batch_size": 16,
|
||||
"rounds": 3,
|
||||
"background_paths": ["/inputs/backgrounds"],
|
||||
"rir_paths": ["/inputs/rirs"]
|
||||
},
|
||||
"model": {
|
||||
"model_type": "conv_attention",
|
||||
"model_size": "medium"
|
||||
},
|
||||
"steps": 50000,
|
||||
"learning_rate": 0.0001,
|
||||
"weight_decay": 0.01,
|
||||
"label_smoothing": 0.05,
|
||||
"max_negative_weight": 1500,
|
||||
"target_fp_per_hour": 0.2,
|
||||
"batch_n_per_class": {
|
||||
"positive": 50,
|
||||
"adversarial_negative": 50,
|
||||
"ACAV100M_sample": 1024,
|
||||
"background_noise": 50
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Write manifest-safe image/package/front-end evidence into the mounted run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import platform
|
||||
import subprocess
|
||||
from importlib.metadata import distributions
|
||||
from pathlib import Path
|
||||
|
||||
from wakeword_training_image import collect_image_provenance
|
||||
|
||||
|
||||
def main() -> int:
|
||||
run_dir = Path("/run")
|
||||
request = json.loads((run_dir / "request.json").read_text(encoding="utf-8"))
|
||||
payload = request["request"]
|
||||
package_versions = {
|
||||
distribution.metadata["Name"].lower(): distribution.version
|
||||
for distribution in distributions()
|
||||
if distribution.metadata.get("Name")
|
||||
}
|
||||
frontend_assets = [
|
||||
path
|
||||
for path in Path("/usr/local/lib").glob("python*/site-packages/livekit/wakeword/**/*.onnx")
|
||||
if path.is_file()
|
||||
]
|
||||
cuda_version = "unknown"
|
||||
try:
|
||||
cuda_version = subprocess.check_output(["nvcc", "--version"], text=True).strip()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
os_packages: list[str] = []
|
||||
try:
|
||||
os_packages = subprocess.check_output(
|
||||
["dpkg-query", "--show", "--showformat", "${Package}=${Version}\\n"], text=True
|
||||
).splitlines()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
provenance = collect_image_provenance(
|
||||
dependency_lock_path=Path("/opt/jr-wakeword/requirements.wheelhouse.lock"),
|
||||
frontend_asset_paths=frontend_assets,
|
||||
installed_distributions=package_versions,
|
||||
image_digest=str(payload["runner_image_digest"]),
|
||||
python_version=platform.python_version(),
|
||||
cuda_version=cuda_version,
|
||||
os_packages=os_packages,
|
||||
)
|
||||
(run_dir / "image-provenance.json").write_text(
|
||||
json.dumps(provenance, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,10 @@
|
||||
# Deliberate top-level constraints for Python 3.11 inside the GPU image.
|
||||
# `build-wakeword-training-image` compiles this with uv into the checked-in
|
||||
# requirements.wheelhouse.lock; the Docker build consumes only that hash lock.
|
||||
# No dependency is resolved while a training run starts.
|
||||
--extra-index-url https://download.pytorch.org/whl/cu126
|
||||
livekit-wakeword[train,eval,export] @ https://files.pythonhosted.org/packages/source/l/livekit-wakeword/livekit_wakeword-0.2.1.tar.gz#sha256=cf2d9cf4867812c06788f64c15e49abd909d9d6291f0a13f1c3f9cb649fa6127
|
||||
torch==2.7.1+cu126
|
||||
torchaudio==2.7.1+cu126
|
||||
onnx==1.17.0
|
||||
onnxruntime-gpu==1.22.0
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,141 @@
|
||||
"""Materialize approved inputs then call only released LiveKit CLI stages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from stage_evidence import run_stage
|
||||
from wakeword_training_image import InputMaterializer
|
||||
|
||||
RUN_DIR = Path("/run")
|
||||
|
||||
|
||||
def _load_json(path: Path) -> dict[str, object]:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def _copy_as_clips(source_paths: list[Path], target: Path) -> None:
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for index, source in enumerate(source_paths):
|
||||
if source.suffix.lower() != ".wav":
|
||||
raise ValueError(f"recorded source must be a WAV: {source.name}")
|
||||
shutil.copyfile(source, target / f"clip_{index:06d}.wav")
|
||||
|
||||
|
||||
def _stage_inputs(recorded_manifest: Path, config: dict[str, object]) -> None:
|
||||
manifest = _load_json(recorded_manifest)
|
||||
if manifest.get("classification") != "commercial_review_required":
|
||||
raise ValueError("commercial pipeline refuses a non-commercial source manifest")
|
||||
model_name = config.get("model_name")
|
||||
if not isinstance(model_name, str) or not model_name:
|
||||
raise ValueError("resolved config must declare model_name")
|
||||
model_dir = RUN_DIR / "output" / model_name
|
||||
stage_data = RUN_DIR / "stage-data"
|
||||
sources = manifest.get("sources")
|
||||
if not isinstance(sources, list):
|
||||
raise ValueError("recorded manifest sources must be a list")
|
||||
general_negative_features: dict[str, Path] = {}
|
||||
for source in sources:
|
||||
if not isinstance(source, dict):
|
||||
raise ValueError("recorded manifest source must be an object")
|
||||
purpose = source.get("purpose")
|
||||
partition = source.get("partition")
|
||||
files = source.get("files")
|
||||
if (
|
||||
not isinstance(purpose, str)
|
||||
or not isinstance(partition, str)
|
||||
or not isinstance(files, list)
|
||||
):
|
||||
raise ValueError("recorded manifest source is incomplete")
|
||||
paths = []
|
||||
for file in files:
|
||||
if not isinstance(file, dict) or not isinstance(file.get("materialized_path"), str):
|
||||
raise ValueError("recorded manifest file is incomplete")
|
||||
paths.append(RUN_DIR / str(file["materialized_path"]))
|
||||
if purpose == "positive":
|
||||
split = "positive_train" if partition == "train" else "positive_test"
|
||||
_copy_as_clips(paths, model_dir / split)
|
||||
elif purpose in {"adversarial_negative", "near_miss"}:
|
||||
split = "negative_train" if partition == "train" else "negative_test"
|
||||
_copy_as_clips(paths, model_dir / split)
|
||||
elif purpose == "background":
|
||||
target = stage_data / "backgrounds"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for source_path in paths:
|
||||
shutil.copyfile(source_path, target / source_path.name)
|
||||
elif purpose == "rir":
|
||||
target = stage_data / "rirs"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for source_path in paths:
|
||||
shutil.copyfile(source_path, target / source_path.name)
|
||||
elif purpose == "general_negative_speech":
|
||||
if len(paths) != 1 or paths[0].suffix != ".npy":
|
||||
raise ValueError(
|
||||
"general-negative source must provide exactly one .npy feature bundle"
|
||||
)
|
||||
if partition not in {"train", "validation"}:
|
||||
raise ValueError("general-negative feature source must use train or validation")
|
||||
if partition in general_negative_features:
|
||||
raise ValueError(f"duplicate general-negative feature bundle for {partition}")
|
||||
compatibility_name = {
|
||||
"train": "openwakeword_features_ACAV100M_2000_hrs_16bit.npy",
|
||||
"validation": "validation_set_features.npy",
|
||||
}[partition]
|
||||
target = stage_data / "features" / compatibility_name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(paths[0], target)
|
||||
general_negative_features[partition] = target
|
||||
|
||||
if set(general_negative_features) != {"train", "validation"}:
|
||||
raise ValueError(
|
||||
"commercial pipeline requires train and validation general-negative features"
|
||||
)
|
||||
|
||||
config["data_dir"] = "/run/stage-data"
|
||||
config["output_dir"] = "/run/output"
|
||||
batch_sizes = config.setdefault("batch_n_per_class", {})
|
||||
if not isinstance(batch_sizes, dict):
|
||||
raise ValueError("batch_n_per_class must be an object")
|
||||
# Preserve LiveKit's dataset key. The source lock/recorded manifest is the
|
||||
# provenance boundary that says these bytes are approved general-negative
|
||||
# speech, not ACAV100M data.
|
||||
batch_sizes["ACAV100M_sample"] = batch_sizes.pop("general_negative_speech", 1024)
|
||||
augmentation = config.setdefault("augmentation", {})
|
||||
if not isinstance(augmentation, dict):
|
||||
raise ValueError("augmentation must be an object")
|
||||
augmentation["background_paths"] = ["/run/stage-data/backgrounds"]
|
||||
augmentation["rir_paths"] = ["/run/stage-data/rirs"]
|
||||
model = config.get("model")
|
||||
if not isinstance(model, dict) or model.get("model_type", "conv_attention") != "conv_attention":
|
||||
raise ValueError("commercial pipeline requires the FP32 conv_attention model")
|
||||
|
||||
|
||||
def main(config_path: Path) -> int:
|
||||
request = _load_json(RUN_DIR / "request.json")
|
||||
request_payload = request.get("request")
|
||||
if not isinstance(request_payload, dict) or request_payload.get("run_kind") != "commercial":
|
||||
raise ValueError("commercial pipeline refuses a quarantined or malformed request")
|
||||
config = _load_json(config_path)
|
||||
materialized = InputMaterializer(
|
||||
RUN_DIR / "sources.lock.json", Path("/inputs"), RUN_DIR, run_kind="commercial"
|
||||
).materialize()
|
||||
_stage_inputs(materialized.recorded_manifest_path, config)
|
||||
effective_config_path = RUN_DIR / "effective-config.yaml"
|
||||
effective_config_path.write_text(json.dumps(config, sort_keys=True) + "\n", encoding="utf-8")
|
||||
run_stage("provenance", ["python", "/opt/jr-wakeword/record-image-provenance.py"])
|
||||
for stage in ("augment", "train", "export", "eval"):
|
||||
run_stage(stage, ["livekit-wakeword", stage, str(effective_config_path)])
|
||||
run_stage(
|
||||
"finalize", ["python", "/opt/jr-wakeword/finalize-run.py", str(effective_config_path)]
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Run the released LiveKit stages for a user-owned, non-publishable experiment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from stage_evidence import run_stage
|
||||
|
||||
RUN_DIR = Path("/run")
|
||||
INPUTS_DIR = Path("/inputs")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
for block in iter(lambda: file.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def verify_local_cache() -> None:
|
||||
lock = json.loads((RUN_DIR / "sources.lock.json").read_text(encoding="utf-8"))
|
||||
sources = lock.get("sources")
|
||||
if not isinstance(sources, list) or len(sources) != 1:
|
||||
raise ValueError("local source lock must contain exactly one cache inventory")
|
||||
files = sources[0].get("files") if isinstance(sources[0], dict) else None
|
||||
if not isinstance(files, list):
|
||||
raise ValueError("local source lock has no content-addressed file inventory")
|
||||
expected_paths: set[str] = set()
|
||||
for item in files:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("local source inventory entry is invalid")
|
||||
relative = item.get("relative_path")
|
||||
expected_hash = item.get("sha256")
|
||||
expected_size = item.get("size_bytes")
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or not relative
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
or not isinstance(expected_hash, str)
|
||||
or not isinstance(expected_size, int)
|
||||
):
|
||||
raise ValueError("local source inventory entry is unsafe")
|
||||
path = INPUTS_DIR / relative
|
||||
if not path.is_file() or path.is_symlink():
|
||||
raise ValueError(f"locked local cache file is missing: {relative}")
|
||||
if path.stat().st_size != expected_size or _sha256(path) != expected_hash:
|
||||
raise ValueError(f"locked local cache file changed: {relative}")
|
||||
expected_paths.add(relative)
|
||||
actual_paths = {
|
||||
path.relative_to(INPUTS_DIR).as_posix()
|
||||
for path in INPUTS_DIR.rglob("*")
|
||||
if path.is_file()
|
||||
}
|
||||
if actual_paths != expected_paths:
|
||||
raise ValueError("local cache contains unsealed files")
|
||||
|
||||
|
||||
def main(config_path: Path) -> int:
|
||||
request = json.loads((RUN_DIR / "request.json").read_text(encoding="utf-8"))
|
||||
payload = request.get("request", {})
|
||||
if payload.get("run_kind") != "local_experiment":
|
||||
raise ValueError("local experiment wrapper requires run_kind=local_experiment")
|
||||
verify_local_cache()
|
||||
run_stage("provenance", ["python", "/opt/jr-wakeword/record-image-provenance.py"])
|
||||
for stage in ("generate", "augment", "train", "export", "eval"):
|
||||
run_stage(stage, ["livekit-wakeword", stage, str(config_path)])
|
||||
run_stage("finalize", ["python", "/opt/jr-wakeword/finalize-run.py", str(config_path)])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Explicitly non-promotable upstream smoke wrapper; it never runs setup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from stage_evidence import run_stage
|
||||
|
||||
|
||||
def main(config_path: Path) -> int:
|
||||
request = json.loads(Path("/run/request.json").read_text(encoding="utf-8"))
|
||||
payload = request.get("request", {})
|
||||
if payload.get("run_kind") != "quarantined_smoke":
|
||||
raise ValueError("quarantined smoke wrapper requires run_kind=quarantined_smoke")
|
||||
# Setup, if needed for this deliberately non-promotable smoke, is a separate
|
||||
# operator operation. This container stays network-isolated either way.
|
||||
run_stage("provenance", ["python", "/opt/jr-wakeword/record-image-provenance.py"])
|
||||
run_stage("run", ["livekit-wakeword", "run", str(config_path)])
|
||||
run_stage("finalize", ["python", "/opt/jr-wakeword/finalize-run.py", str(config_path)])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "JR Wakeword commercial source lock",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "run_kind", "sources"],
|
||||
"properties": {
|
||||
"schema_version": {"const": 1},
|
||||
"run_kind": {"enum": ["commercial", "quarantined_smoke"]},
|
||||
"sources": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["source_id", "source_kind", "purpose", "partition", "original_publisher", "version", "revision", "license", "attribution", "files"],
|
||||
"properties": {
|
||||
"source_id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,127}$"},
|
||||
"source_kind": {"type": "string", "minLength": 1},
|
||||
"purpose": {"enum": ["positive", "adversarial_negative", "near_miss", "general_negative_speech", "background", "rir"]},
|
||||
"partition": {"enum": ["train", "validation", "calibration", "release_test"]},
|
||||
"original_publisher": {"type": "object", "required": ["name", "url"]},
|
||||
"version": {"type": "string", "minLength": 1},
|
||||
"revision": {"type": "string", "minLength": 1},
|
||||
"license": {"type": "object", "required": ["spdx_id", "terms_url", "captured_text_path", "captured_text_sha256"]},
|
||||
"attribution": {"type": "string", "minLength": 1},
|
||||
"commercial": {"type": "object"},
|
||||
"files": {"type": "array", "minItems": 1}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Record one isolated image stage without exposing arguments or raw inputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
RUN_DIR = Path("/run")
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
for block in iter(lambda: file.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _disk_bytes(path: Path) -> int:
|
||||
return sum(candidate.stat().st_size for candidate in path.rglob("*") if candidate.is_file())
|
||||
|
||||
|
||||
def _output_hashes() -> dict[str, str]:
|
||||
output = RUN_DIR / "output"
|
||||
return {
|
||||
candidate.relative_to(RUN_DIR).as_posix(): _sha256(candidate)
|
||||
for candidate in sorted(path for path in output.rglob("*") if path.is_file())
|
||||
}
|
||||
|
||||
|
||||
def _input_hashes() -> dict[str, str]:
|
||||
return {
|
||||
name: _sha256(RUN_DIR / name)
|
||||
for name in ("request.json", "resolved-config.yaml", "sources.lock.json")
|
||||
if (RUN_DIR / name).is_file()
|
||||
}
|
||||
|
||||
|
||||
def _time_metrics(path: Path) -> dict[str, float | int]:
|
||||
values: dict[str, float | int] = {
|
||||
"max_rss_kib": 0,
|
||||
"cpu_user_s": 0.0,
|
||||
"cpu_system_s": 0.0,
|
||||
"cpu_percent": 0.0,
|
||||
}
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError:
|
||||
return values
|
||||
for line in lines:
|
||||
key, separator, value = line.partition(":")
|
||||
if not separator:
|
||||
continue
|
||||
try:
|
||||
if key.strip() == "Maximum resident set size (kbytes)":
|
||||
values["max_rss_kib"] = int(value.strip())
|
||||
elif key.strip() == "User time (seconds)":
|
||||
values["cpu_user_s"] = float(value.strip())
|
||||
elif key.strip() == "System time (seconds)":
|
||||
values["cpu_system_s"] = float(value.strip())
|
||||
elif key.strip() == "Percent of CPU this job got":
|
||||
values["cpu_percent"] = float(value.strip().removesuffix("%"))
|
||||
except ValueError:
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def run_stage(name: str, command: list[str]) -> None:
|
||||
"""Run one released command and append its sealed evidence before raising."""
|
||||
|
||||
if not name or any(
|
||||
character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in name
|
||||
):
|
||||
raise ValueError("stage name must be a safe token")
|
||||
if not command:
|
||||
raise ValueError("stage command must not be empty")
|
||||
active = RUN_DIR / "active-stage"
|
||||
active.write_text(name + "\n", encoding="utf-8")
|
||||
time_report = RUN_DIR / "stage-times" / f"{name}.time-v.txt"
|
||||
time_report.parent.mkdir(parents=True, exist_ok=True)
|
||||
started_at = _utc_now()
|
||||
started = time.monotonic()
|
||||
before = _disk_bytes(RUN_DIR)
|
||||
exit_status: int | None = None
|
||||
status = "failed"
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
["/usr/bin/time", "-v", "-o", str(time_report), *command], check=False
|
||||
)
|
||||
exit_status = completed.returncode
|
||||
if exit_status != 0:
|
||||
raise subprocess.CalledProcessError(exit_status, command)
|
||||
status = "succeeded"
|
||||
finally:
|
||||
record: dict[str, Any] = {
|
||||
"name": name,
|
||||
"started_at": started_at,
|
||||
"ended_at": _utc_now(),
|
||||
"status": status,
|
||||
"exit_status": exit_status,
|
||||
"command_name": command[0],
|
||||
"input_hashes": _input_hashes(),
|
||||
"output_hashes": _output_hashes() or {"output": hashlib.sha256(b"").hexdigest()},
|
||||
"elapsed_s": time.monotonic() - started,
|
||||
**_time_metrics(time_report),
|
||||
"disk_growth_bytes": max(0, _disk_bytes(RUN_DIR) - before),
|
||||
}
|
||||
with (RUN_DIR / "image-stage-events.ndjson").open("a", encoding="utf-8") as file:
|
||||
file.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
file.flush()
|
||||
os.fsync(file.fileno())
|
||||
active.unlink(missing_ok=True)
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://gateway.jrich.ai/schemas/wakeword-training-request-v1.json",
|
||||
"title": "Resolved Wakeword Training Request v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_version",
|
||||
"primary_phrase",
|
||||
"variants",
|
||||
"near_misses",
|
||||
"evaluation_profile",
|
||||
"source_lock_path",
|
||||
"source_lock_sha256",
|
||||
"intended_release_id",
|
||||
"runner_image_digest",
|
||||
"pipeline_config",
|
||||
"run_kind",
|
||||
"derived_from_run_id"
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {"const": 1},
|
||||
"primary_phrase": {"type": "string", "minLength": 1},
|
||||
"variants": {"type": "array", "items": {"type": "string", "minLength": 1}},
|
||||
"near_misses": {"type": "array", "items": {"type": "string", "minLength": 1}},
|
||||
"evaluation_profile": {"type": "string", "minLength": 1},
|
||||
"source_lock_path": {"type": "string", "minLength": 1},
|
||||
"source_lock_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},
|
||||
"intended_release_id": {"type": "string", "minLength": 1},
|
||||
"runner_image_digest": {"type": "string", "pattern": "^sha256:[a-f0-9]{64}$"},
|
||||
"pipeline_config": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["model_name", "target_phrases"],
|
||||
"properties": {
|
||||
"model_name": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]*$"},
|
||||
"family_slug": {"type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"},
|
||||
"version": {"type": "integer", "minimum": 1},
|
||||
"threshold": {
|
||||
"oneOf": [{"type": "number", "minimum": 0, "maximum": 1}, {"const": "auto"}]
|
||||
},
|
||||
"target_phrases": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
|
||||
"custom_negative_phrases": {"type": "array", "items": {"type": "string", "minLength": 1}},
|
||||
"n_samples": {"type": "integer", "minimum": 1},
|
||||
"n_samples_val": {"type": "integer", "minimum": 1},
|
||||
"n_background_samples": {"type": "integer", "minimum": 1},
|
||||
"n_background_samples_val": {"type": "integer", "minimum": 1},
|
||||
"tts_batch_size": {"type": "integer", "minimum": 1},
|
||||
"max_speakers": {"type": ["integer", "null"], "minimum": 1},
|
||||
"tts_backend": {"const": "piper_vits"},
|
||||
"noise_scales": {"type": "array", "minItems": 1, "items": {"type": "number"}},
|
||||
"noise_scale_ws": {"type": "array", "minItems": 1, "items": {"type": "number"}},
|
||||
"length_scales": {"type": "array", "minItems": 1, "items": {"type": "number"}},
|
||||
"slerp_weights": {"type": "array", "minItems": 1, "items": {"type": "number"}},
|
||||
"augmentation": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"clip_duration": {"type": "number", "exclusiveMinimum": 0},
|
||||
"batch_size": {"type": "integer", "minimum": 1},
|
||||
"rounds": {"type": "integer", "minimum": 1},
|
||||
"background_paths": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^/inputs/(?!\\.\\.(?:/|$))(?!.*?/\\.\\.(?:/|$)).+"
|
||||
}
|
||||
},
|
||||
"rir_paths": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^/inputs/(?!\\.\\.(?:/|$))(?!.*?/\\.\\.(?:/|$)).+"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"model_type": {"enum": ["dnn", "rnn", "conv_attention"]},
|
||||
"model_size": {"enum": ["tiny", "small", "medium", "large"]}
|
||||
}
|
||||
},
|
||||
"steps": {"type": "integer", "minimum": 1},
|
||||
"learning_rate": {"type": "number", "minimum": 0},
|
||||
"weight_decay": {"type": "number", "minimum": 0},
|
||||
"label_smoothing": {"type": "number", "minimum": 0},
|
||||
"max_negative_weight": {"type": "number", "minimum": 0},
|
||||
"target_fp_per_hour": {"type": "number", "minimum": 0},
|
||||
"batch_n_per_class": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"positive": {"type": "integer", "minimum": 1},
|
||||
"adversarial_negative": {"type": "integer", "minimum": 1},
|
||||
"ACAV100M_sample": {"type": "integer", "minimum": 1},
|
||||
"background_noise": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"run_kind": {"enum": ["commercial", "local_experiment", "quarantined_smoke"]},
|
||||
"derived_from_run_id": {
|
||||
"type": ["string", "null"],
|
||||
"pattern": "^[a-f0-9]{32}$"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user