Replace a TrainingJob
const url = 'https://example.com/apis/nodus.dev/v1beta1/namespaces/example/trainingjobs/example';const options = { method: 'PUT', headers: {'Content-Type': 'application/json'}, body: '{"apiVersion":"example","kind":"example","metadata":{"annotations":{"additionalProperty":"example"},"creationTimestamp":"example","deletionGracePeriodSeconds":1,"deletionTimestamp":"example","finalizers":["example"],"generateName":"example","generation":1,"labels":{"additionalProperty":"example"},"managedFields":[{"apiVersion":"example","fieldsType":"example","fieldsV1":{"additionalProperty":"example"},"manager":"example","operation":"example","subresource":"example","time":"example"}],"name":"example","namespace":"example","ownerReferences":[{"apiVersion":"example","blockOwnerDeletion":true,"controller":true,"kind":"example","name":"example","uid":"example"}],"resourceVersion":"example","selfLink":"example","uid":"example"},"spec":{"data":{"columns":{"additionalProperty":"example"},"format":"example","path":"example","revision":1,"validationFraction":"example","volume":"example"},"distributed":{"gpusPerNode":1,"launcher":"example","maxAssemblyRetries":1,"network":"example","nodes":1,"startupTimeout":"example","totalGPUs":1,"transport":"example"},"environment":{"heldOutTasks":1,"name":"example","seed":1,"trainTasks":1},"evaluation":{"baseline":true,"final":true,"minComparisonTasks":1},"expectedDuration":"example","exportTo":{"volume":"example"},"grading":{"maxParallel":1},"maxCostUSD":"example","mode":"example","model":{"secret":"example","uri":"example","volume":{"name":"example","revision":1}},"parameters":"example","placement":{"dedicated":true,"interruptible":"example","maxRateUSDPerHour":"example","offerings":["example"],"pool":"example","profile":"example","queueTimeout":"example","regions":["example"]},"resources":{"cpu":"example","disk":"example","gpu":{"count":1,"exact":true,"interconnect":"example","minMemory":"example","type":["example"]},"memory":"example","nodes":1},"runtime":"example","state":"example","task":"example","teacher":{"secret":"example","uri":"example","volume":{"name":"example","revision":1}},"tracking":{"mlflow":{"experiment":"example","secret":"example","uri":"example"},"wandb":{"connection":"example"}}},"status":{"compiledJob":{"activeDeadlineSeconds":1,"args":["example"],"backoffLimit":1,"command":["example"],"completeByTime":"example","completions":1,"connections":["example"],"distributed":{"gpusPerNode":1,"launcher":"example","maxAssemblyRetries":1,"network":"example","nodes":1,"startupTimeout":"example","totalGPUs":1,"transport":"example"},"env":[{"name":"example","value":"example","valueFrom":{"secretKeyRef":{"key":"example","name":"example"}}}],"expectedDuration":"example","image":"example","imagePullSecrets":["example"],"imageRef":"example","initCommand":{"command":["example"],"timeout":"example"},"inputs":[{"bucket":{"connection":"example","sha256":"example","uri":"example"},"fromStage":{"output":"example","stage":"example"},"fromWorkspace":{"name":"example","path":"example"},"name":"example","output":{"index":1,"job":"example","name":"example"},"url":{"sha256":"example","url":"example"},"volume":{"name":"example","revision":1}}],"maxCostUSD":"example","network":{"egress":{"allow":["example"],"policy":"example","presets":["example"]}},"outputs":[{"fromRanks":"example","name":"example","path":"example","sink":{"connection":"example","mode":"example","table":"example"}}],"parallelism":1,"placement":{"dedicated":true,"interruptible":"example","maxRateUSDPerHour":"example","offerings":["example"],"pool":"example","profile":"example","queueTimeout":"example","regions":["example"]},"recovery":{"checkpoint":{"format":"example","integration":"example","interval":"example","maxSize":"example","paths":["example"],"retainAfterFinish":"example"},"continuity":"example","maxAttempts":1,"minProgressDuration":"example","onInterruption":"example"},"resources":{"cpu":"example","disk":"example","gpu":{"count":1,"exact":true,"interconnect":"example","minMemory":"example","type":["example"]},"memory":"example","nodes":1},"scaledownWindow":"example","secrets":[{"name":"example","version":1}],"serviceAccountName":"example","sidecars":[{"command":["example"],"env":[{"name":"example","value":"example","valueFrom":{"secretKeyRef":{"key":"example","name":"example"}}}],"image":"example","name":"example","startupTimeout":"example"}],"source":{"blob":"example","git":{"path":"example","ref":"example","repo":"example"}},"state":"example","timeout":"example","ttlSecondsAfterFinished":1,"volumes":[{"mountPath":"example","readOnly":true,"revision":1,"volume":"example"}],"workingDir":"example"},"conditions":[{"lastTransitionTime":"example","message":"example","observedGeneration":1,"reason":"example","status":"example","type":"example"}],"cost":{"bySegment":{"bootUSD":"example","coveredByNodus":["example"],"restoreUSD":"example","runningUSD":"example","teardownUSD":"example"},"final":true,"fundedUntilTime":"example","heldUSD":"example","limitUSD":"example","rateUSDPerHour":"example","totalUSD":"example"},"diagnostics":{"rewardSignal":"example","weightsChanged":true},"estimate":{"assemblyBoundUSD":"example","blockingReasons":[{"code":"example","fix":"example","message":"example"}],"concurrentHoldUSD":"example","costUSD":{"p50":"example","p90":"example"},"gangHoldUSD":"example","holdUSD":"example","minimumChargeUSD":"example","overheadUSD":{"p50":"example","p90":"example"},"placementPreview":{"explain":"example","interruptible":true,"offering":"example","regionClass":"example","warmReuse":true},"pricebookVersion":"example","rateUSDPerHour":"example","startup":{"cold":{"p50":"example","p90":"example"},"warm":{"p50":"example","p90":"example"}},"topology":"example","validUntil":"example","warnings":[{"code":"example","message":"example"}]},"job":"example","message":"example","metrics":{"gpuMemoryBytes":1,"loss":"example","step":1,"tokensPerSecond":"example","totalSteps":1},"observedGeneration":1,"outputs":[{"name":"example","sizeBytes":1}],"phase":"example","progress":{"stage":"example"},"provenance":{"environmentDigest":"example","managed":true,"modelRevision":"example","parametersSHA256":"example","runtimeDigest":"example"},"reason":"example","summary":{"baseline":{"errors":1,"inProgress":1,"meanReward":"example","passRate":"example","passed":1,"tasks":1},"comparison":{"changePP":"example","comparable":true,"measurable":true,"note":"example","reason":"example","resolutionPP":"example"},"evaluation":{"errors":1,"inProgress":1,"meanReward":"example","passRate":"example","passed":1,"tasks":1}}}}'};
try { const response = await fetch(url, options); const data = await response.json(); console.log(data);} catch (error) { console.error(error);}curl --request PUT \ --url https://example.com/apis/nodus.dev/v1beta1/namespaces/example/trainingjobs/example \ --header 'Content-Type: application/json' \ --data '{ "apiVersion": "example", "kind": "example", "metadata": { "annotations": { "additionalProperty": "example" }, "creationTimestamp": "example", "deletionGracePeriodSeconds": 1, "deletionTimestamp": "example", "finalizers": [ "example" ], "generateName": "example", "generation": 1, "labels": { "additionalProperty": "example" }, "managedFields": [ { "apiVersion": "example", "fieldsType": "example", "fieldsV1": { "additionalProperty": "example" }, "manager": "example", "operation": "example", "subresource": "example", "time": "example" } ], "name": "example", "namespace": "example", "ownerReferences": [ { "apiVersion": "example", "blockOwnerDeletion": true, "controller": true, "kind": "example", "name": "example", "uid": "example" } ], "resourceVersion": "example", "selfLink": "example", "uid": "example" }, "spec": { "data": { "columns": { "additionalProperty": "example" }, "format": "example", "path": "example", "revision": 1, "validationFraction": "example", "volume": "example" }, "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "environment": { "heldOutTasks": 1, "name": "example", "seed": 1, "trainTasks": 1 }, "evaluation": { "baseline": true, "final": true, "minComparisonTasks": 1 }, "expectedDuration": "example", "exportTo": { "volume": "example" }, "grading": { "maxParallel": 1 }, "maxCostUSD": "example", "mode": "example", "model": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "parameters": "example", "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "runtime": "example", "state": "example", "task": "example", "teacher": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "tracking": { "mlflow": { "experiment": "example", "secret": "example", "uri": "example" }, "wandb": { "connection": "example" } } }, "status": { "compiledJob": { "activeDeadlineSeconds": 1, "args": [ "example" ], "backoffLimit": 1, "command": [ "example" ], "completeByTime": "example", "completions": 1, "connections": [ "example" ], "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "expectedDuration": "example", "image": "example", "imagePullSecrets": [ "example" ], "imageRef": "example", "initCommand": { "command": [ "example" ], "timeout": "example" }, "inputs": [ { "bucket": { "connection": "example", "sha256": "example", "uri": "example" }, "fromStage": { "output": "example", "stage": "example" }, "fromWorkspace": { "name": "example", "path": "example" }, "name": "example", "output": { "index": 1, "job": "example", "name": "example" }, "url": { "sha256": "example", "url": "example" }, "volume": { "name": "example", "revision": 1 } } ], "maxCostUSD": "example", "network": { "egress": { "allow": [ "example" ], "policy": "example", "presets": [ "example" ] } }, "outputs": [ { "fromRanks": "example", "name": "example", "path": "example", "sink": { "connection": "example", "mode": "example", "table": "example" } } ], "parallelism": 1, "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "recovery": { "checkpoint": { "format": "example", "integration": "example", "interval": "example", "maxSize": "example", "paths": [ "example" ], "retainAfterFinish": "example" }, "continuity": "example", "maxAttempts": 1, "minProgressDuration": "example", "onInterruption": "example" }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "scaledownWindow": "example", "secrets": [ { "name": "example", "version": 1 } ], "serviceAccountName": "example", "sidecars": [ { "command": [ "example" ], "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "image": "example", "name": "example", "startupTimeout": "example" } ], "source": { "blob": "example", "git": { "path": "example", "ref": "example", "repo": "example" } }, "state": "example", "timeout": "example", "ttlSecondsAfterFinished": 1, "volumes": [ { "mountPath": "example", "readOnly": true, "revision": 1, "volume": "example" } ], "workingDir": "example" }, "conditions": [ { "lastTransitionTime": "example", "message": "example", "observedGeneration": 1, "reason": "example", "status": "example", "type": "example" } ], "cost": { "bySegment": { "bootUSD": "example", "coveredByNodus": [ "example" ], "restoreUSD": "example", "runningUSD": "example", "teardownUSD": "example" }, "final": true, "fundedUntilTime": "example", "heldUSD": "example", "limitUSD": "example", "rateUSDPerHour": "example", "totalUSD": "example" }, "diagnostics": { "rewardSignal": "example", "weightsChanged": true }, "estimate": { "assemblyBoundUSD": "example", "blockingReasons": [ { "code": "example", "fix": "example", "message": "example" } ], "concurrentHoldUSD": "example", "costUSD": { "p50": "example", "p90": "example" }, "gangHoldUSD": "example", "holdUSD": "example", "minimumChargeUSD": "example", "overheadUSD": { "p50": "example", "p90": "example" }, "placementPreview": { "explain": "example", "interruptible": true, "offering": "example", "regionClass": "example", "warmReuse": true }, "pricebookVersion": "example", "rateUSDPerHour": "example", "startup": { "cold": { "p50": "example", "p90": "example" }, "warm": { "p50": "example", "p90": "example" } }, "topology": "example", "validUntil": "example", "warnings": [ { "code": "example", "message": "example" } ] }, "job": "example", "message": "example", "metrics": { "gpuMemoryBytes": 1, "loss": "example", "step": 1, "tokensPerSecond": "example", "totalSteps": 1 }, "observedGeneration": 1, "outputs": [ { "name": "example", "sizeBytes": 1 } ], "phase": "example", "progress": { "stage": "example" }, "provenance": { "environmentDigest": "example", "managed": true, "modelRevision": "example", "parametersSHA256": "example", "runtimeDigest": "example" }, "reason": "example", "summary": { "baseline": { "errors": 1, "inProgress": 1, "meanReward": "example", "passRate": "example", "passed": 1, "tasks": 1 }, "comparison": { "changePP": "example", "comparable": true, "measurable": true, "note": "example", "reason": "example", "resolutionPP": "example" }, "evaluation": { "errors": 1, "inProgress": 1, "meanReward": "example", "passRate": "example", "passed": 1, "tasks": 1 } } } }'Parameters
Section titled “ Parameters ”Path Parameters
Section titled “ Path Parameters ”The project.
The object’s name.
Query Parameters
Section titled “ Query Parameters ”Header Parameters
Section titled “ Header Parameters ”Request Bodyrequired
Section titled “ Request Bodyrequired ”TrainingJob is pre-training, fine-tuning, post-training or an RL run on a TrainingRuntime (Beta). It compiles to one Job, a multi-node gang when it asks for more than one node, and mirrors that Job’s phase.
object
object
object
object
object
object
object
TrainingJobSpec describes one training run.
object
TrainingData is a dataset in a Volume.
object
Columns maps the runtime’s column roles (prompt, completion, chosen, rejected, text) to dataset columns.
object
Format is Auto (default), JSONL, JSON or CSV.
Path is a file or directory inside the Volume.
Revision pins a revision.
ValidationFraction is the share held out for validation, as a decimal string (default “0.05”).
Volume is the Volume holding the data.
Distributed runs the Job as a gang of nodes that start together and restart together (Beta).
object
GPUsPerNode is 1, 2, 4 or 8 and equals resources.gpu.count when both are set.
Launcher is Plain (env contract only), Torchrun, Ray, Verl, Accelerate or Deepspeed.
MaxAssemblyRetries is how many times a timed-out assembly is re-placed (0–5, default 2).
Network is Colocated (one provider and region), Regional (one region class) or Global.
Nodes is the gang size (2–16; the beta caps it at 8). Exactly one of nodes and totalGPUs.
StartupTimeout is the gang assembly budget (5m–60m, default 15m).
TotalGPUs is resolved into status.topology at admission (2–64).
Transport is Direct (private or direct paths only) or Auto (relayed paths allowed).
EnvironmentBinding selects an Environment and its task counts.
object
HeldOutTasks is the number of held-out tasks evaluated before and after training.
Name is nodus/
Seed selects the tasks deterministically (default 0).
TrainTasks is the number of training tasks.
Evaluation controls the baseline and final evaluation.
object
Baseline evaluates the model before training (default true).
Final evaluates the model after training (default true).
MinComparisonTasks is the smallest held-out set a comparison is reported on (default and minimum 16).
ExpectedDuration overrides the runtime estimate of steps × seconds per step.
ExportTarget is a Volume the outputs are also written to.
object
Volume is the Volume.
GradingPolicy bounds grading.
object
MaxParallel is the most grading Sandboxes at once (default 4, at most 32).
MaxCostUSD caps the run, its compiled Job and its grading Sandboxes together; it can only be raised.
Mode is Train (default) or Evaluate, and must be one of the runtime’s modes.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Parameters are validated against the runtime’s JSON Schema (at most 64 KiB) over the runtime defaults.
Placement constrains where work may run.
object
Dedicated is reserved; only true is accepted.
Interruptible is Never, Allow (the scheduler decides by expected cost) or Prefer.
MaxRateUSDPerHour filters out offerings above this customer rate.
Offerings restricts placement to these Offerings.
Pool is a BYOC Pool in the project.
Profile is Balanced, Cost or Speed.
QueueTimeout is how long the Job may wait for capacity before failing with CapacityUnavailable (default 24h; 0s waits forever).
Regions are region classes (us, eu, …); empty means any.
Resources are capacity floors. For GPU work the offering’s host shape applies if larger.
object
CPU is the vCPU floor.
Disk is the ephemeral disk floor.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Memory is the host memory floor.
Nodes above 1 is shorthand for spec.distributed.nodes and is stored in that form.
Runtime is a Ready TrainingRuntime: nodus/
State is the desired state: Running (default), Suspended, or the terminal stop state.
Task overrides the runtime’s task; Pretrain on sft trains on raw text with packing.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Tracking names customer-owned trackers; they are never the source of truth for comparisons.
object
MLflowTracking is an MLflow server.
object
Experiment is the experiment name.
Secret holds MLFLOW_TRACKING_TOKEN or username and password.
URI is the tracking server (https://).
WandBTracking is a Weights & Biases project.
object
Connection is a WandB Connection.
TrainingJobStatus is written by the TrainingJob controller.
object
JobSpec describes a Job. Everything is immutable after create except parallelism, backoffLimit, timeout, maxCostUSD (raise only), recovery.maxAttempts, recovery.checkpoint.interval and retainAfterFinish, ttlSecondsAfterFinished and state.
object
ActiveDeadlineSeconds is accepted as an alias of timeout on write and folded into it.
BackoffLimit is how many customer failures (non-zero exits, OOM) across all indexes are retried before the Job fails (0–100).
CompleteByTime is a finish-by hint for placement and checkpoint cadence. It never stops the Job.
Completions is how many indexes must succeed (1–10000). Each index gets NODUS_INDEX and JOB_COMPLETION_INDEX.
Distributed runs the Job as a gang of nodes that start together and restart together (Beta).
object
GPUsPerNode is 1, 2, 4 or 8 and equals resources.gpu.count when both are set.
Launcher is Plain (env contract only), Torchrun, Ray, Verl, Accelerate or Deepspeed.
MaxAssemblyRetries is how many times a timed-out assembly is re-placed (0–5, default 2).
Network is Colocated (one provider and region), Regional (one region class) or Global.
Nodes is the gang size (2–16; the beta caps it at 8). Exactly one of nodes and totalGPUs.
StartupTimeout is the gang assembly budget (5m–60m, default 15m).
TotalGPUs is resolved into status.topology at admission (2–64).
Transport is Direct (private or direct paths only) or Auto (relayed paths allowed).
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
ExpectedDuration is the declared runtime the estimate uses first.
InitCommand is a preflight run before the command.
object
Command is the argv.
Timeout is at most 30m.
Inputs are materialized read-only under /nodus/inputs/
JobInput is one input: exactly one of volume, output, bucket, url and fromWorkspace. It is mounted at /nodus/inputs/
object
BucketInput is an object in customer storage.
object
Connection is an S3 Connection that can read it.
SHA256 is checked at end of stream when set.
URI is s3://bucket/key.
StageInput names an output of a Pipeline stage.
object
Output is the output name.
Stage is the producing stage.
WorkspaceInput mounts a Workspace home.
object
Name is the Workspace.
Path is a subdirectory of the home.
Name matches ^[A-Za-z][A-Za-z0-9_]{0,63}$ and is unique among inputs.
OutputInput names a committed output of another Job.
object
Index selects an index of an Indexed Job (default 0).
Job is the producing Job.
Name is the output.
URLInput is a file downloaded over HTTPS.
object
SHA256 is the expected digest.
URL is https:// only.
VolumeInput names a Volume revision.
object
Name is the Volume.
Revision pins a revision.
MaxCostUSD caps what the Job may spend. Reaching it suspends the Job; it can only be raised.
Network sets egress for the container.
object
Egress is the outbound network policy.
object
Allow lists DNS names or *.suffix patterns (AllowList only; at most 128).
Policy is Deny, AllowList or Open.
Presets add curated host lists such as python-packages or huggingface.
Outputs are files or directories collected when an index succeeds (at most 32). Everything under /nodus/outputs is also collected as the output named “outputs”.
JobOutput is a declared output.
object
FromRanks is Zero (rank 0 only) or All (every rank under rank-
Name matches ^[a-z0-9._-]{1,64}$; the nodus. prefix is reserved.
Path is an absolute file or directory path; directories are archived as tar.zst.
OutputSink loads an output into a customer database table.
object
Connection is a Postgres, Neon or Supabase Connection.
Mode is Append (default) or Replace.
Table is the destination table.
Parallelism is how many indexes run at once (1 to min(completions, 256)).
Placement constrains where work may run.
object
Dedicated is reserved; only true is accepted.
Interruptible is Never, Allow (the scheduler decides by expected cost) or Prefer.
MaxRateUSDPerHour filters out offerings above this customer rate.
Offerings restricts placement to these Offerings.
Pool is a BYOC Pool in the project.
Profile is Balanced, Cost or Speed.
QueueTimeout is how long the Job may wait for capacity before failing with CapacityUnavailable (default 24h; 0s waits forever).
Regions are region classes (us, eu, …); empty means any.
Recovery says how progress survives lost capacity.
object
CheckpointPolicy configures checkpoints.
object
Format is Snapshot, or Dcp for torch.distributed.checkpoint (the default for distributed Jobs).
Integration is Auto, None or HFTrainer.
Interval is auto or a duration from 1m to 6h.
MaxSize is checked before upload (default 1Ti, at most 2Ti).
Paths are saved together in one checkpoint (at most 64; default /nodus/state).
RetainAfterFinish keeps checkpoints this long after the Job finishes (default 168h).
Continuity is Checkpointed (restore the latest checkpoint), Restartable (progress cursor) or Ephemeral.
MaxAttempts is how many infrastructure replacements each index may use (1–32; gangs count restarts).
MinProgressDuration is the run time that counts as progress (default 2m).
OnInterruption is Resume, or Fail to fail instead of recovering or suspending.
Resources are capacity floors. For GPU work the offering’s host shape applies if larger.
object
CPU is the vCPU floor.
Disk is the ephemeral disk floor.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Memory is the host memory floor.
Nodes above 1 is shorthand for spec.distributed.nodes and is stored in that form.
ScaledownWindow keeps the capacity warm for reuse this long after the Job ends (0s–20m).
SecretMount mounts one Secret, optionally pinned to a version. The bare name is accepted on write.
object
Name is the Secret.
Version pins a version; unset pins the latest at admission.
Sidecar is a helper container with native-sidecar semantics.
object
Command is the sidecar’s argv.
Env adds environment variables.
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
Image defaults to the Job’s image.
Name is unique within the Job.
StartupTimeout bounds how long the sidecar may take to become ready.
Source is the code for the container: exactly one of git and blob.
object
Blob is sha256:
GitSource is a repository checkout.
object
Path is the checkout path (default workingDir).
Ref is a branch, tag or commit.
Repo is owner/name or an https:// URL.
State is the desired state: Running (the default), Suspended, or the cancel state that stops the Job for good.
Timeout is the wall-clock limit counted from the first Provisioning, excluding time spent Suspended. The Job then fails with DeadlineExceeded.
TTLSecondsAfterFinished deletes the Job, its outputs and checkpoints this long after it finishes.
VolumeMount mounts a Volume.
object
MountPath is the absolute mount point; system paths are refused.
ReadOnly mounts it read-only.
Revision pins a revision; only with readOnly.
Volume names the Volume.
Conditions are the latest observations.
object
Cost is status.cost on every kind that spends money. It is a display copy of the ledger: amounts are decimal USD strings, and nothing is charged from these fields.
object
CostSegments splits a charge by billing segment.
object
BootUSD is the charge from the start of billing until the program started.
CoveredByNodus lists the segments (Boot, Running, Restore, Teardown) with time Nodus paid for instead of charging it, such as the start and teardown of a run that failed because of Nodus.
RestoreUSD is the charge for bringing a replacement up after capacity was lost.
RunningUSD is the charge while the program ran.
TeardownUSD is the charge from stop until deletion was confirmed, with the billing increment.
Final is true once the final charge has posted.
FundedUntilTime is when the current holds stop paying for the object.
HeldUSD is what is currently held for it.
LimitUSD is the object’s maxCostUSD, if it has one.
RateUSDPerHour is the sum of the hourly rates of its capacity that is still billing.
TotalUSD is what has been charged for this object so far.
TrainingDiagnostics interpret training.
object
RewardSignal is ContrastObserved, NoRewardContrast or InsufficientEvidence.
WeightsChanged reports whether the trainer reported changed weights.
Estimate is returned in status.estimate by a dry-run (?dryRun=All) and on create for every compute kind (ADR-031). Amounts are decimal USD strings; the ETag of the dry-run response binds a later create to it.
object
AssemblyBoundUSD is the most a multi-node Job can cost if its members never assemble.
BlockingReasons say why a real create would fail now, with the amounts and a fix.
BlockingReason says why a create would fail now.
object
Code is the error code the create would return, for example InsufficientCredits.
Fix says what to do.
Message states the reason with the amounts involved.
ConcurrentHoldUSD is the total of the holds for every slot that starts at once.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
GangHoldUSD is the total of the member holds of a multi-node Job.
HoldUSD is the first hold that must fit your balance, every matching Budget and the object’s own cap.
MinimumChargeUSD is the billing increment rounding charged once per allocation.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
PlacementPreview is the customer-safe description of the capacity a create would use.
object
Explain is the scheduler’s explanation, the same shape as Attempt status.placement.explain.
Interruptible reports whether the capacity can be reclaimed.
Offering is the offering id, for example h100-sxm-80g-x8-us.
RegionClass is the region class of the offering.
WarmReuse reports whether warm capacity would be reused.
PricebookVersion is the price book the estimate used.
RateUSDPerHour is the hourly rate of the chosen capacity.
StartupEstimate is the expected time to start.
object
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
Topology is the resolved multi-node topology of a gang Job.
ValidUntil is when the prices this estimate used stop being current. A create bound by If-Match is admitted after it too, and never placed above rateUSDPerHour plus 10 %.
Warnings are non-blocking notes, for example ColdStart or InterruptibleSelected.
EstimateWarning is a non-blocking note on an estimate.
object
Code is the CamelCase warning code.
Message explains the warning.
Job is the compiled Job’s name (label nodus.dev/trainingjob).
Message explains Reason.
TrainingMetrics are the latest metrics the trainer reported.
object
GPUMemoryBytes is the peak GPU memory.
Loss is the latest training loss, as a decimal string.
Step is the current optimizer step.
TokensPerSecond is the latest throughput, as a decimal string.
TotalSteps is the planned number of steps.
ObservedGeneration is the generation the status describes.
Outputs are the committed results, such as results.json and the adapter.
TrainingOutput is one committed result of the compiled Job.
object
Name is the output name.
SizeBytes is the committed size.
Phase mirrors the compiled Job in the run-to-completion vocabulary.
TrainingProgress is the current stage.
object
Stage is Baseline, Training, Evaluating or Exporting.
Provenance identifies exactly what ran.
object
EnvironmentDigest is the environment image digest.
Managed is true when a catalog runtime ran without image or command overrides.
ModelRevision is the pinned model revision.
ParametersSHA256 hashes the canonical merged parameters.
RuntimeDigest is the runtime image digest.
Reason says why the run failed, stopped or is suspended.
TrainingSummary is folded incrementally from task events.
object
PhaseSummary counts one phase’s latest report per task attempt.
object
Errors counts attempts that ended in an infrastructure or grading error; they never count as failed.
InProgress counts started attempts with no completion yet.
MeanReward is the mean finite reward, as a decimal string.
PassRate is passed divided by tasks, as a decimal string.
Passed is how many passed.
Tasks is the number of completed, scored task attempts.
Comparison is the comparability gate: like-for-like tasks, at least minComparisonTasks, not all 0 % or all 100 %, and no missing events.
object
ChangePP is the pass-rate change in percentage points, as a decimal string.
Comparable is true when baseline and evaluation scored the same tasks and no events are missing.
Measurable is true when the tasks can show a change (comparable and not all failed or all passed).
Note explains Reason in one sentence.
Reason is Comparable, SmallestStep, TooFewTasks, DifferentTasks, NonePassed, AllPassed or IncompleteEvents.
ResolutionPP is the smallest change the task set can show (100 / tasks), as a decimal string.
PhaseSummary counts one phase’s latest report per task attempt.
object
Errors counts attempts that ended in an infrastructure or grading error; they never count as failed.
InProgress counts started attempts with no completion yet.
MeanReward is the mean finite reward, as a decimal string.
PassRate is passed divided by tasks, as a decimal string.
Passed is how many passed.
Tasks is the number of completed, scored task attempts.
Example generated
{ "apiVersion": "example", "kind": "example", "metadata": { "annotations": { "additionalProperty": "example" }, "creationTimestamp": "example", "deletionGracePeriodSeconds": 1, "deletionTimestamp": "example", "finalizers": [ "example" ], "generateName": "example", "generation": 1, "labels": { "additionalProperty": "example" }, "managedFields": [ { "apiVersion": "example", "fieldsType": "example", "fieldsV1": { "additionalProperty": "example" }, "manager": "example", "operation": "example", "subresource": "example", "time": "example" } ], "name": "example", "namespace": "example", "ownerReferences": [ { "apiVersion": "example", "blockOwnerDeletion": true, "controller": true, "kind": "example", "name": "example", "uid": "example" } ], "resourceVersion": "example", "selfLink": "example", "uid": "example" }, "spec": { "data": { "columns": { "additionalProperty": "example" }, "format": "example", "path": "example", "revision": 1, "validationFraction": "example", "volume": "example" }, "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "environment": { "heldOutTasks": 1, "name": "example", "seed": 1, "trainTasks": 1 }, "evaluation": { "baseline": true, "final": true, "minComparisonTasks": 1 }, "expectedDuration": "example", "exportTo": { "volume": "example" }, "grading": { "maxParallel": 1 }, "maxCostUSD": "example", "mode": "example", "model": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "parameters": "example", "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "runtime": "example", "state": "example", "task": "example", "teacher": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "tracking": { "mlflow": { "experiment": "example", "secret": "example", "uri": "example" }, "wandb": { "connection": "example" } } }, "status": { "compiledJob": { "activeDeadlineSeconds": 1, "args": [ "example" ], "backoffLimit": 1, "command": [ "example" ], "completeByTime": "example", "completions": 1, "connections": [ "example" ], "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "expectedDuration": "example", "image": "example", "imagePullSecrets": [ "example" ], "imageRef": "example", "initCommand": { "command": [ "example" ], "timeout": "example" }, "inputs": [ { "bucket": { "connection": "example", "sha256": "example", "uri": "example" }, "fromStage": { "output": "example", "stage": "example" }, "fromWorkspace": { "name": "example", "path": "example" }, "name": "example", "output": { "index": 1, "job": "example", "name": "example" }, "url": { "sha256": "example", "url": "example" }, "volume": { "name": "example", "revision": 1 } } ], "maxCostUSD": "example", "network": { "egress": { "allow": [ "example" ], "policy": "example", "presets": [ "example" ] } }, "outputs": [ { "fromRanks": "example", "name": "example", "path": "example", "sink": { "connection": "example", "mode": "example", "table": "example" } } ], "parallelism": 1, "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "recovery": { "checkpoint": { "format": "example", "integration": "example", "interval": "example", "maxSize": "example", "paths": [ "example" ], "retainAfterFinish": "example" }, "continuity": "example", "maxAttempts": 1, "minProgressDuration": "example", "onInterruption": "example" }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "scaledownWindow": "example", "secrets": [ { "name": "example", "version": 1 } ], "serviceAccountName": "example", "sidecars": [ { "command": [ "example" ], "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "image": "example", "name": "example", "startupTimeout": "example" } ], "source": { "blob": "example", "git": { "path": "example", "ref": "example", "repo": "example" } }, "state": "example", "timeout": "example", "ttlSecondsAfterFinished": 1, "volumes": [ { "mountPath": "example", "readOnly": true, "revision": 1, "volume": "example" } ], "workingDir": "example" }, "conditions": [ { "lastTransitionTime": "example", "message": "example", "observedGeneration": 1, "reason": "example", "status": "example", "type": "example" } ], "cost": { "bySegment": { "bootUSD": "example", "coveredByNodus": [ "example" ], "restoreUSD": "example", "runningUSD": "example", "teardownUSD": "example" }, "final": true, "fundedUntilTime": "example", "heldUSD": "example", "limitUSD": "example", "rateUSDPerHour": "example", "totalUSD": "example" }, "diagnostics": { "rewardSignal": "example", "weightsChanged": true }, "estimate": { "assemblyBoundUSD": "example", "blockingReasons": [ { "code": "example", "fix": "example", "message": "example" } ], "concurrentHoldUSD": "example", "costUSD": { "p50": "example", "p90": "example" }, "gangHoldUSD": "example", "holdUSD": "example", "minimumChargeUSD": "example", "overheadUSD": { "p50": "example", "p90": "example" }, "placementPreview": { "explain": "example", "interruptible": true, "offering": "example", "regionClass": "example", "warmReuse": true }, "pricebookVersion": "example", "rateUSDPerHour": "example", "startup": { "cold": { "p50": "example", "p90": "example" }, "warm": { "p50": "example", "p90": "example" } }, "topology": "example", "validUntil": "example", "warnings": [ { "code": "example", "message": "example" } ] }, "job": "example", "message": "example", "metrics": { "gpuMemoryBytes": 1, "loss": "example", "step": 1, "tokensPerSecond": "example", "totalSteps": 1 }, "observedGeneration": 1, "outputs": [ { "name": "example", "sizeBytes": 1 } ], "phase": "example", "progress": { "stage": "example" }, "provenance": { "environmentDigest": "example", "managed": true, "modelRevision": "example", "parametersSHA256": "example", "runtimeDigest": "example" }, "reason": "example", "summary": { "baseline": { "errors": 1, "inProgress": 1, "meanReward": "example", "passRate": "example", "passed": 1, "tasks": 1 }, "comparison": { "changePP": "example", "comparable": true, "measurable": true, "note": "example", "reason": "example", "resolutionPP": "example" }, "evaluation": { "errors": 1, "inProgress": 1, "meanReward": "example", "passRate": "example", "passed": 1, "tasks": 1 } } }}Responses
Section titled “ Responses ”OK
TrainingJob is pre-training, fine-tuning, post-training or an RL run on a TrainingRuntime (Beta). It compiles to one Job, a multi-node gang when it asks for more than one node, and mirrors that Job’s phase.
object
object
object
object
object
object
object
TrainingJobSpec describes one training run.
object
TrainingData is a dataset in a Volume.
object
Columns maps the runtime’s column roles (prompt, completion, chosen, rejected, text) to dataset columns.
object
Format is Auto (default), JSONL, JSON or CSV.
Path is a file or directory inside the Volume.
Revision pins a revision.
ValidationFraction is the share held out for validation, as a decimal string (default “0.05”).
Volume is the Volume holding the data.
Distributed runs the Job as a gang of nodes that start together and restart together (Beta).
object
GPUsPerNode is 1, 2, 4 or 8 and equals resources.gpu.count when both are set.
Launcher is Plain (env contract only), Torchrun, Ray, Verl, Accelerate or Deepspeed.
MaxAssemblyRetries is how many times a timed-out assembly is re-placed (0–5, default 2).
Network is Colocated (one provider and region), Regional (one region class) or Global.
Nodes is the gang size (2–16; the beta caps it at 8). Exactly one of nodes and totalGPUs.
StartupTimeout is the gang assembly budget (5m–60m, default 15m).
TotalGPUs is resolved into status.topology at admission (2–64).
Transport is Direct (private or direct paths only) or Auto (relayed paths allowed).
EnvironmentBinding selects an Environment and its task counts.
object
HeldOutTasks is the number of held-out tasks evaluated before and after training.
Name is nodus/
Seed selects the tasks deterministically (default 0).
TrainTasks is the number of training tasks.
Evaluation controls the baseline and final evaluation.
object
Baseline evaluates the model before training (default true).
Final evaluates the model after training (default true).
MinComparisonTasks is the smallest held-out set a comparison is reported on (default and minimum 16).
ExpectedDuration overrides the runtime estimate of steps × seconds per step.
ExportTarget is a Volume the outputs are also written to.
object
Volume is the Volume.
GradingPolicy bounds grading.
object
MaxParallel is the most grading Sandboxes at once (default 4, at most 32).
MaxCostUSD caps the run, its compiled Job and its grading Sandboxes together; it can only be raised.
Mode is Train (default) or Evaluate, and must be one of the runtime’s modes.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Parameters are validated against the runtime’s JSON Schema (at most 64 KiB) over the runtime defaults.
Placement constrains where work may run.
object
Dedicated is reserved; only true is accepted.
Interruptible is Never, Allow (the scheduler decides by expected cost) or Prefer.
MaxRateUSDPerHour filters out offerings above this customer rate.
Offerings restricts placement to these Offerings.
Pool is a BYOC Pool in the project.
Profile is Balanced, Cost or Speed.
QueueTimeout is how long the Job may wait for capacity before failing with CapacityUnavailable (default 24h; 0s waits forever).
Regions are region classes (us, eu, …); empty means any.
Resources are capacity floors. For GPU work the offering’s host shape applies if larger.
object
CPU is the vCPU floor.
Disk is the ephemeral disk floor.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Memory is the host memory floor.
Nodes above 1 is shorthand for spec.distributed.nodes and is stored in that form.
Runtime is a Ready TrainingRuntime: nodus/
State is the desired state: Running (default), Suspended, or the terminal stop state.
Task overrides the runtime’s task; Pretrain on sft trains on raw text with packing.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Tracking names customer-owned trackers; they are never the source of truth for comparisons.
object
MLflowTracking is an MLflow server.
object
Experiment is the experiment name.
Secret holds MLFLOW_TRACKING_TOKEN or username and password.
URI is the tracking server (https://).
WandBTracking is a Weights & Biases project.
object
Connection is a WandB Connection.
TrainingJobStatus is written by the TrainingJob controller.
object
JobSpec describes a Job. Everything is immutable after create except parallelism, backoffLimit, timeout, maxCostUSD (raise only), recovery.maxAttempts, recovery.checkpoint.interval and retainAfterFinish, ttlSecondsAfterFinished and state.
object
ActiveDeadlineSeconds is accepted as an alias of timeout on write and folded into it.
BackoffLimit is how many customer failures (non-zero exits, OOM) across all indexes are retried before the Job fails (0–100).
CompleteByTime is a finish-by hint for placement and checkpoint cadence. It never stops the Job.
Completions is how many indexes must succeed (1–10000). Each index gets NODUS_INDEX and JOB_COMPLETION_INDEX.
Distributed runs the Job as a gang of nodes that start together and restart together (Beta).
object
GPUsPerNode is 1, 2, 4 or 8 and equals resources.gpu.count when both are set.
Launcher is Plain (env contract only), Torchrun, Ray, Verl, Accelerate or Deepspeed.
MaxAssemblyRetries is how many times a timed-out assembly is re-placed (0–5, default 2).
Network is Colocated (one provider and region), Regional (one region class) or Global.
Nodes is the gang size (2–16; the beta caps it at 8). Exactly one of nodes and totalGPUs.
StartupTimeout is the gang assembly budget (5m–60m, default 15m).
TotalGPUs is resolved into status.topology at admission (2–64).
Transport is Direct (private or direct paths only) or Auto (relayed paths allowed).
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
ExpectedDuration is the declared runtime the estimate uses first.
InitCommand is a preflight run before the command.
object
Command is the argv.
Timeout is at most 30m.
Inputs are materialized read-only under /nodus/inputs/
JobInput is one input: exactly one of volume, output, bucket, url and fromWorkspace. It is mounted at /nodus/inputs/
object
BucketInput is an object in customer storage.
object
Connection is an S3 Connection that can read it.
SHA256 is checked at end of stream when set.
URI is s3://bucket/key.
StageInput names an output of a Pipeline stage.
object
Output is the output name.
Stage is the producing stage.
WorkspaceInput mounts a Workspace home.
object
Name is the Workspace.
Path is a subdirectory of the home.
Name matches ^[A-Za-z][A-Za-z0-9_]{0,63}$ and is unique among inputs.
OutputInput names a committed output of another Job.
object
Index selects an index of an Indexed Job (default 0).
Job is the producing Job.
Name is the output.
URLInput is a file downloaded over HTTPS.
object
SHA256 is the expected digest.
URL is https:// only.
VolumeInput names a Volume revision.
object
Name is the Volume.
Revision pins a revision.
MaxCostUSD caps what the Job may spend. Reaching it suspends the Job; it can only be raised.
Network sets egress for the container.
object
Egress is the outbound network policy.
object
Allow lists DNS names or *.suffix patterns (AllowList only; at most 128).
Policy is Deny, AllowList or Open.
Presets add curated host lists such as python-packages or huggingface.
Outputs are files or directories collected when an index succeeds (at most 32). Everything under /nodus/outputs is also collected as the output named “outputs”.
JobOutput is a declared output.
object
FromRanks is Zero (rank 0 only) or All (every rank under rank-
Name matches ^[a-z0-9._-]{1,64}$; the nodus. prefix is reserved.
Path is an absolute file or directory path; directories are archived as tar.zst.
OutputSink loads an output into a customer database table.
object
Connection is a Postgres, Neon or Supabase Connection.
Mode is Append (default) or Replace.
Table is the destination table.
Parallelism is how many indexes run at once (1 to min(completions, 256)).
Placement constrains where work may run.
object
Dedicated is reserved; only true is accepted.
Interruptible is Never, Allow (the scheduler decides by expected cost) or Prefer.
MaxRateUSDPerHour filters out offerings above this customer rate.
Offerings restricts placement to these Offerings.
Pool is a BYOC Pool in the project.
Profile is Balanced, Cost or Speed.
QueueTimeout is how long the Job may wait for capacity before failing with CapacityUnavailable (default 24h; 0s waits forever).
Regions are region classes (us, eu, …); empty means any.
Recovery says how progress survives lost capacity.
object
CheckpointPolicy configures checkpoints.
object
Format is Snapshot, or Dcp for torch.distributed.checkpoint (the default for distributed Jobs).
Integration is Auto, None or HFTrainer.
Interval is auto or a duration from 1m to 6h.
MaxSize is checked before upload (default 1Ti, at most 2Ti).
Paths are saved together in one checkpoint (at most 64; default /nodus/state).
RetainAfterFinish keeps checkpoints this long after the Job finishes (default 168h).
Continuity is Checkpointed (restore the latest checkpoint), Restartable (progress cursor) or Ephemeral.
MaxAttempts is how many infrastructure replacements each index may use (1–32; gangs count restarts).
MinProgressDuration is the run time that counts as progress (default 2m).
OnInterruption is Resume, or Fail to fail instead of recovering or suspending.
Resources are capacity floors. For GPU work the offering’s host shape applies if larger.
object
CPU is the vCPU floor.
Disk is the ephemeral disk floor.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Memory is the host memory floor.
Nodes above 1 is shorthand for spec.distributed.nodes and is stored in that form.
ScaledownWindow keeps the capacity warm for reuse this long after the Job ends (0s–20m).
SecretMount mounts one Secret, optionally pinned to a version. The bare name is accepted on write.
object
Name is the Secret.
Version pins a version; unset pins the latest at admission.
Sidecar is a helper container with native-sidecar semantics.
object
Command is the sidecar’s argv.
Env adds environment variables.
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
Image defaults to the Job’s image.
Name is unique within the Job.
StartupTimeout bounds how long the sidecar may take to become ready.
Source is the code for the container: exactly one of git and blob.
object
Blob is sha256:
GitSource is a repository checkout.
object
Path is the checkout path (default workingDir).
Ref is a branch, tag or commit.
Repo is owner/name or an https:// URL.
State is the desired state: Running (the default), Suspended, or the cancel state that stops the Job for good.
Timeout is the wall-clock limit counted from the first Provisioning, excluding time spent Suspended. The Job then fails with DeadlineExceeded.
TTLSecondsAfterFinished deletes the Job, its outputs and checkpoints this long after it finishes.
VolumeMount mounts a Volume.
object
MountPath is the absolute mount point; system paths are refused.
ReadOnly mounts it read-only.
Revision pins a revision; only with readOnly.
Volume names the Volume.
Conditions are the latest observations.
object
Cost is status.cost on every kind that spends money. It is a display copy of the ledger: amounts are decimal USD strings, and nothing is charged from these fields.
object
CostSegments splits a charge by billing segment.
object
BootUSD is the charge from the start of billing until the program started.
CoveredByNodus lists the segments (Boot, Running, Restore, Teardown) with time Nodus paid for instead of charging it, such as the start and teardown of a run that failed because of Nodus.
RestoreUSD is the charge for bringing a replacement up after capacity was lost.
RunningUSD is the charge while the program ran.
TeardownUSD is the charge from stop until deletion was confirmed, with the billing increment.
Final is true once the final charge has posted.
FundedUntilTime is when the current holds stop paying for the object.
HeldUSD is what is currently held for it.
LimitUSD is the object’s maxCostUSD, if it has one.
RateUSDPerHour is the sum of the hourly rates of its capacity that is still billing.
TotalUSD is what has been charged for this object so far.
TrainingDiagnostics interpret training.
object
RewardSignal is ContrastObserved, NoRewardContrast or InsufficientEvidence.
WeightsChanged reports whether the trainer reported changed weights.
Estimate is returned in status.estimate by a dry-run (?dryRun=All) and on create for every compute kind (ADR-031). Amounts are decimal USD strings; the ETag of the dry-run response binds a later create to it.
object
AssemblyBoundUSD is the most a multi-node Job can cost if its members never assemble.
BlockingReasons say why a real create would fail now, with the amounts and a fix.
BlockingReason says why a create would fail now.
object
Code is the error code the create would return, for example InsufficientCredits.
Fix says what to do.
Message states the reason with the amounts involved.
ConcurrentHoldUSD is the total of the holds for every slot that starts at once.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
GangHoldUSD is the total of the member holds of a multi-node Job.
HoldUSD is the first hold that must fit your balance, every matching Budget and the object’s own cap.
MinimumChargeUSD is the billing increment rounding charged once per allocation.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
PlacementPreview is the customer-safe description of the capacity a create would use.
object
Explain is the scheduler’s explanation, the same shape as Attempt status.placement.explain.
Interruptible reports whether the capacity can be reclaimed.
Offering is the offering id, for example h100-sxm-80g-x8-us.
RegionClass is the region class of the offering.
WarmReuse reports whether warm capacity would be reused.
PricebookVersion is the price book the estimate used.
RateUSDPerHour is the hourly rate of the chosen capacity.
StartupEstimate is the expected time to start.
object
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
Topology is the resolved multi-node topology of a gang Job.
ValidUntil is when the prices this estimate used stop being current. A create bound by If-Match is admitted after it too, and never placed above rateUSDPerHour plus 10 %.
Warnings are non-blocking notes, for example ColdStart or InterruptibleSelected.
EstimateWarning is a non-blocking note on an estimate.
object
Code is the CamelCase warning code.
Message explains the warning.
Job is the compiled Job’s name (label nodus.dev/trainingjob).
Message explains Reason.
TrainingMetrics are the latest metrics the trainer reported.
object
GPUMemoryBytes is the peak GPU memory.
Loss is the latest training loss, as a decimal string.
Step is the current optimizer step.
TokensPerSecond is the latest throughput, as a decimal string.
TotalSteps is the planned number of steps.
ObservedGeneration is the generation the status describes.
Outputs are the committed results, such as results.json and the adapter.
TrainingOutput is one committed result of the compiled Job.
object
Name is the output name.
SizeBytes is the committed size.
Phase mirrors the compiled Job in the run-to-completion vocabulary.
TrainingProgress is the current stage.
object
Stage is Baseline, Training, Evaluating or Exporting.
Provenance identifies exactly what ran.
object
EnvironmentDigest is the environment image digest.
Managed is true when a catalog runtime ran without image or command overrides.
ModelRevision is the pinned model revision.
ParametersSHA256 hashes the canonical merged parameters.
RuntimeDigest is the runtime image digest.
Reason says why the run failed, stopped or is suspended.
TrainingSummary is folded incrementally from task events.
object
PhaseSummary counts one phase’s latest report per task attempt.
object
Errors counts attempts that ended in an infrastructure or grading error; they never count as failed.
InProgress counts started attempts with no completion yet.
MeanReward is the mean finite reward, as a decimal string.
PassRate is passed divided by tasks, as a decimal string.
Passed is how many passed.
Tasks is the number of completed, scored task attempts.
Comparison is the comparability gate: like-for-like tasks, at least minComparisonTasks, not all 0 % or all 100 %, and no missing events.
object
ChangePP is the pass-rate change in percentage points, as a decimal string.
Comparable is true when baseline and evaluation scored the same tasks and no events are missing.
Measurable is true when the tasks can show a change (comparable and not all failed or all passed).
Note explains Reason in one sentence.
Reason is Comparable, SmallestStep, TooFewTasks, DifferentTasks, NonePassed, AllPassed or IncompleteEvents.
ResolutionPP is the smallest change the task set can show (100 / tasks), as a decimal string.
PhaseSummary counts one phase’s latest report per task attempt.
object
Errors counts attempts that ended in an infrastructure or grading error; they never count as failed.
InProgress counts started attempts with no completion yet.
MeanReward is the mean finite reward, as a decimal string.
PassRate is passed divided by tasks, as a decimal string.
Passed is how many passed.
Tasks is the number of completed, scored task attempts.
Example generated
{ "apiVersion": "example", "kind": "example", "metadata": { "annotations": { "additionalProperty": "example" }, "creationTimestamp": "example", "deletionGracePeriodSeconds": 1, "deletionTimestamp": "example", "finalizers": [ "example" ], "generateName": "example", "generation": 1, "labels": { "additionalProperty": "example" }, "managedFields": [ { "apiVersion": "example", "fieldsType": "example", "fieldsV1": { "additionalProperty": "example" }, "manager": "example", "operation": "example", "subresource": "example", "time": "example" } ], "name": "example", "namespace": "example", "ownerReferences": [ { "apiVersion": "example", "blockOwnerDeletion": true, "controller": true, "kind": "example", "name": "example", "uid": "example" } ], "resourceVersion": "example", "selfLink": "example", "uid": "example" }, "spec": { "data": { "columns": { "additionalProperty": "example" }, "format": "example", "path": "example", "revision": 1, "validationFraction": "example", "volume": "example" }, "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "environment": { "heldOutTasks": 1, "name": "example", "seed": 1, "trainTasks": 1 }, "evaluation": { "baseline": true, "final": true, "minComparisonTasks": 1 }, "expectedDuration": "example", "exportTo": { "volume": "example" }, "grading": { "maxParallel": 1 }, "maxCostUSD": "example", "mode": "example", "model": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "parameters": "example", "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "runtime": "example", "state": "example", "task": "example", "teacher": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "tracking": { "mlflow": { "experiment": "example", "secret": "example", "uri": "example" }, "wandb": { "connection": "example" } } }, "status": { "compiledJob": { "activeDeadlineSeconds": 1, "args": [ "example" ], "backoffLimit": 1, "command": [ "example" ], "completeByTime": "example", "completions": 1, "connections": [ "example" ], "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "expectedDuration": "example", "image": "example", "imagePullSecrets": [ "example" ], "imageRef": "example", "initCommand": { "command": [ "example" ], "timeout": "example" }, "inputs": [ { "bucket": { "connection": "example", "sha256": "example", "uri": "example" }, "fromStage": { "output": "example", "stage": "example" }, "fromWorkspace": { "name": "example", "path": "example" }, "name": "example", "output": { "index": 1, "job": "example", "name": "example" }, "url": { "sha256": "example", "url": "example" }, "volume": { "name": "example", "revision": 1 } } ], "maxCostUSD": "example", "network": { "egress": { "allow": [ "example" ], "policy": "example", "presets": [ "example" ] } }, "outputs": [ { "fromRanks": "example", "name": "example", "path": "example", "sink": { "connection": "example", "mode": "example", "table": "example" } } ], "parallelism": 1, "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "recovery": { "checkpoint": { "format": "example", "integration": "example", "interval": "example", "maxSize": "example", "paths": [ "example" ], "retainAfterFinish": "example" }, "continuity": "example", "maxAttempts": 1, "minProgressDuration": "example", "onInterruption": "example" }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "scaledownWindow": "example", "secrets": [ { "name": "example", "version": 1 } ], "serviceAccountName": "example", "sidecars": [ { "command": [ "example" ], "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "image": "example", "name": "example", "startupTimeout": "example" } ], "source": { "blob": "example", "git": { "path": "example", "ref": "example", "repo": "example" } }, "state": "example", "timeout": "example", "ttlSecondsAfterFinished": 1, "volumes": [ { "mountPath": "example", "readOnly": true, "revision": 1, "volume": "example" } ], "workingDir": "example" }, "conditions": [ { "lastTransitionTime": "example", "message": "example", "observedGeneration": 1, "reason": "example", "status": "example", "type": "example" } ], "cost": { "bySegment": { "bootUSD": "example", "coveredByNodus": [ "example" ], "restoreUSD": "example", "runningUSD": "example", "teardownUSD": "example" }, "final": true, "fundedUntilTime": "example", "heldUSD": "example", "limitUSD": "example", "rateUSDPerHour": "example", "totalUSD": "example" }, "diagnostics": { "rewardSignal": "example", "weightsChanged": true }, "estimate": { "assemblyBoundUSD": "example", "blockingReasons": [ { "code": "example", "fix": "example", "message": "example" } ], "concurrentHoldUSD": "example", "costUSD": { "p50": "example", "p90": "example" }, "gangHoldUSD": "example", "holdUSD": "example", "minimumChargeUSD": "example", "overheadUSD": { "p50": "example", "p90": "example" }, "placementPreview": { "explain": "example", "interruptible": true, "offering": "example", "regionClass": "example", "warmReuse": true }, "pricebookVersion": "example", "rateUSDPerHour": "example", "startup": { "cold": { "p50": "example", "p90": "example" }, "warm": { "p50": "example", "p90": "example" } }, "topology": "example", "validUntil": "example", "warnings": [ { "code": "example", "message": "example" } ] }, "job": "example", "message": "example", "metrics": { "gpuMemoryBytes": 1, "loss": "example", "step": 1, "tokensPerSecond": "example", "totalSteps": 1 }, "observedGeneration": 1, "outputs": [ { "name": "example", "sizeBytes": 1 } ], "phase": "example", "progress": { "stage": "example" }, "provenance": { "environmentDigest": "example", "managed": true, "modelRevision": "example", "parametersSHA256": "example", "runtimeDigest": "example" }, "reason": "example", "summary": { "baseline": { "errors": 1, "inProgress": 1, "meanReward": "example", "passRate": "example", "passed": 1, "tasks": 1 }, "comparison": { "changePP": "example", "comparable": true, "measurable": true, "note": "example", "reason": "example", "resolutionPP": "example" }, "evaluation": { "errors": 1, "inProgress": 1, "meanReward": "example", "passRate": "example", "passed": 1, "tasks": 1 } } }}default
Section titled “ default ”An error: a metav1.Status whose reason is a registered code.
Status is the error body of every API response: a Kubernetes metav1.Status (so kubectl and client-go understand it) plus three top-level extensions that those clients ignore (ADR-028).
object
object
object
Docs is the URL of the code’s docs page.
Fix says what to do next, for example a CLI command or the field to change.
object
object
RequestID identifies the request in logs and support tickets.
Example generated
{ "apiVersion": "example", "code": 1, "details": { "causes": [ { "field": "example", "message": "example", "reason": "example" } ], "group": "example", "kind": "example", "name": "example", "retryAfterSeconds": 1, "uid": "example" }, "docs": "example", "fix": "example", "kind": "example", "message": "example", "metadata": { "continue": "example", "remainingItemCount": 1, "resourceVersion": "example", "selfLink": "example", "shardInfo": { "selector": "example" } }, "reason": "example", "requestId": "example", "status": "example"}