Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixing test cases on daggyr for new polling

Browse Source
This commit is contained in:
Ian Roddis
2022-01-04 17:03:11 -04:00
parent 5a4c6d0756
commit f1479a72d9
3 changed files with 52 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
examples/sample_dag.json
View File

@@ -1,43 +1,11 @@
{
"parameters": {
"SOURCE": [
"a",
"b",
"c"
],
"DATE": [
"2021-01-01",
"2021-01-02"
]
},
"tasks": [
{
"name": "pull_data_a",
"max_retries": 3,
"retry_interval_seconds": 600,
"if": "/path/to/should_pull.sh --date {{DATE}} --source {{SOURCE}}_A",
"command": "/path/to/pull.sh --date {{DATE}} --source {{SOURCE}}_A",
"verification_command": "/path/to/pull_verify.sh --date {{DATE}} --source {{SOURCE}}_A",
"timeout_seconds": 30,
"children": [
"merge_data"
]
},
{
"name": "pull_data_b",
"max_retries": 3,
"retry_interval_seconds": 600,
"if": "/path/to/should_pull.sh --date {{DATE}} --source {{SOURCE}}_B",
"command": "/path/to/pull.sh --date {{DATE}} --source {{SOURCE}}_B",
"verification_command": "/path/to/pull_verify.sh --date {{DATE}} --source {{SOURCE}}_B",
"timeout_seconds": 30,
"children": [
"merge_data"
]
},
{
"name": "merge_data",
"command": "/path/to/merge.sh --left {{SOURCE}}_A --right {{SOURCE}}_B"
}
]
"tag": "testdag",
"tasks": {
"A": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "b", "c", "d" ]},
"b": { "job": { "command": [ "/usr/bin/sleep", "2" ], "cores": "1", "memoryMB": "100" }, "children": [ "d" ]},
"c": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "e", "f" ]},
"d": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "f" ]},
"e": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "f" ]},
"f": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }}
}
}