Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Checkpointing progress on task builder.

Browse Source
This commit is contained in:
Ian Roddis
2021-08-02 11:59:43 -03:00
parent 54e8170c68
commit a5b4e6ce42
3 changed files with 68 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
examples/sample_dag.json
View File

@@ -3,24 +3,26 @@
{
"name": "pull_data_a",
"max_retries": 3,
"retry_delay_seconds": 600,
"command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_A",
"verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_A",
"timeout_seconds": 30
"retry_interval_seconds": 600,
"if": "/path/to/should_pull.sh --date {{DATE}} --source {{SOURCE}}_A",
"command": "/path/to/pull.sh --date {{DATE}} --source {{SOURCE}}_A",
"verification_command": "/path/to/pull_verify.sh --date {{DATE}} --source {{SOURCE}}_A",
"timeout_seconds": 30,
"children": [ "merge_data" ]
},
{
"name": "pull_data_b",
"max_retries": 3,
"retry_delay_seconds": 600,
"command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_B",
"verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_B",
"timeout_seconds": 30
"retry_interval_seconds": 600,
"if": "/path/to/should_pull.sh --date {{DATE}} --source {{SOURCE}}_B",
"command": "/path/to/pull.sh --date {{DATE}} --source {{SOURCE}}_B",
"verification_command": "/path/to/pull_verify.sh --date {{DATE}} --source {{SOURCE}}_B",
"timeout_seconds": 30,
"children": [ "merge_data" ]
},
{
"name": "merge_data",
"command": "/path/to/merge.sh --left {SOURCE}_A --right {SOURCE}_B"
"command": "/path/to/merge.sh --left {{SOURCE}}_A --right {{SOURCE}}_B"
}
]
}