Skip to content

Commit fe71258

Browse files
hvuhsghvuhsg
authored
Merge pull request #13 from thewh1teagle/feat/on-check-failure
Add on_failure to health check
2 parents b23badb + cc9d4be commit fe71258

File tree

7 files changed

+123
-23
lines changed

7 files changed

+123
-23
lines changed

‎README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,8 +210,10 @@ services:
210210
headers:
211211
Host: domain.org
212212
Authorization: "Bearer abc123"
213-
# on_failure options will be added in the future
214-
213+
214+
# on_failure runs a shell command if the check fails. Expands $date, $error, $check_name.
215+
on_failure: |
216+
curl -d "Health check '$check_name' failed at $date due to: $error" ntfy.sh/gatego
215217
cache: true # Cache responses that has cache headers (Cache-Control and Expire)
216218
217219
```

‎checker.go‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,22 @@ import (
44
"fmt"
55
"log"
66
"net/http"
7+
"os/exec"
8+
"strings"
79
"time"
810

911
"github.com/google/uuid"
1012
"github.com/hvuhsg/gatego/pkg/cron"
1113
)
1214

1315
type Check struct {
14-
Name string
15-
Cron string
16-
URL string
17-
Method string
18-
Timeout time.Duration
19-
Headers map[string]string
16+
Name string
17+
Cron string
18+
URL string
19+
Method string
20+
Timeout time.Duration
21+
Headers map[string]string
22+
OnFailure string
2023
}
2124

2225
func (c Check) run(onFailure func(error)) func() {
@@ -57,18 +60,40 @@ func (c Check) run(onFailure func(error)) func() {
5760
}
5861
}
5962

63+
func handleFailure(check Check, err error) error {
64+
// Expand command
65+
command := check.OnFailure
66+
date := time.Now().UTC().Format("2006-01-02 15:04:05")
67+
command = strings.ReplaceAll(command, "$date", date)
68+
command = strings.ReplaceAll(command, "$error", err.Error())
69+
command = strings.ReplaceAll(command, "$check_name", check.Name)
70+
71+
// Run it
72+
args := strings.Split(command, " ")
73+
cmd := exec.Command(args[0], args[1:]...)
74+
if err := cmd.Start(); err != nil {
75+
return err
76+
}
77+
return nil
78+
}
79+
6080
type Checker struct {
6181
Delay time.Duration
6282
Checks []Check
6383
scheduler *cron.Cron
64-
OnFailure func(error)
6584
}
6685

6786
func (c Checker) Start() error {
6887
c.scheduler = cron.New()
6988

7089
for _, check := range c.Checks {
71-
err := c.scheduler.Add(uuid.NewString(), check.Cron, check.run(c.OnFailure))
90+
err := c.scheduler.Add(uuid.NewString(), check.Cron, check.run(func(err error) {
91+
if check.OnFailure != "" {
92+
if err := handleFailure(check, err); err != nil {
93+
log.Default().Printf("Failed to spawn on_failure command: %s\n", err)
94+
}
95+
}
96+
}))
7297
if err != nil {
7398
return err
7499
}

‎checker_test.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package gatego
22

33
import (
4+
"errors"
45
"net/http"
56
"net/http/httptest"
67
"testing"
@@ -140,6 +141,67 @@ func TestChecker_Start(t *testing.T) {
140141
}
141142
}
142143

144+
func TestChecker_OnFailure(t *testing.T) {
145+
tests := []struct {
146+
name string
147+
checker Checker
148+
expectedError bool
149+
}{
150+
{
151+
name: "on failure command with valid command",
152+
checker: Checker{
153+
Delay: 1 * time.Second,
154+
Checks: []Check{
155+
{
156+
Name: "test-check-failure",
157+
Cron: "* * * * *",
158+
Method: "GET",
159+
URL: "http://example.com",
160+
Timeout: 5 * time.Second,
161+
OnFailure: "echo check '$check_name' failed at $date: $error",
162+
},
163+
},
164+
},
165+
expectedError: false,
166+
},
167+
{
168+
name: "on failure command with invalid command",
169+
checker: Checker{
170+
Delay: 1 * time.Second,
171+
Checks: []Check{
172+
{
173+
Name: "test-check-failure",
174+
Cron: "* * * * *",
175+
Method: "GET",
176+
URL: "http://example.com",
177+
Timeout: 5 * time.Second,
178+
OnFailure: "invalidCommand $error",
179+
},
180+
},
181+
},
182+
expectedError: true,
183+
},
184+
}
185+
186+
for _, tt := range tests {
187+
t.Run(tt.name, func(t *testing.T) {
188+
// Simulate a failure scenario by injecting an error
189+
err := errors.New("Connection timeout")
190+
err = handleFailure(tt.checker.Checks[0], err)
191+
192+
// Check if an error was returned and if it matches the expected result
193+
if (err != nil) != tt.expectedError {
194+
t.Errorf("handleFailure() error = %v, expectedError %v", err, tt.expectedError)
195+
}
196+
197+
// Clean up scheduler if it was created
198+
if tt.checker.scheduler != nil {
199+
tt.checker.scheduler.Stop()
200+
}
201+
})
202+
}
203+
}
204+
143205
// TestCheckWithMockServer tests the Check struct with a mock HTTP server
144206
func TestCheckWithMockServer(t *testing.T) {
145207
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

‎cmd/config.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ services:
4444
headers:
4545
Host: domain.org
4646
Authorization: "Bearer abc123"
47+
on_failure: |
48+
echo Health check '$check_name' failed at $date with error: $error
4749
4850
omit_headers: [Authorization, X-API-Key, X-Secret-Token]
4951

‎config-schema.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,13 @@
227227
"Authorization": "Bearer abc123"
228228
}
229229
]
230+
},
231+
"on_failure": {
232+
"type": "string",
233+
"description": "Shell command to execute if the health check fails. Supports variable expansion: $date, $error, and $check_name.",
234+
"examples": [
235+
"echo Health check '$check_name' failed at $date with error: $error"
236+
]
230237
}
231238
}
232239
}

‎config/config.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,13 @@ func (b Backend) validate() error {
5151
}
5252

5353
type Check struct {
54-
Name string `yaml:"name"`
55-
Cron string `yaml:"cron"`
56-
URL string `yaml:"url"`
57-
Method string `yaml:"method"`
58-
Timeout time.Duration `yaml:"timeout"`
59-
Headers map[string]string `yaml:"headers"`
54+
Name string `yaml:"name"`
55+
Cron string `yaml:"cron"`
56+
URL string `yaml:"url"`
57+
Method string `yaml:"method"`
58+
Timeout time.Duration `yaml:"timeout"`
59+
Headers map[string]string `yaml:"headers"`
60+
OnFailure string `yaml:"on_failure"`
6061
}
6162

6263
func (c Check) validate() error {

‎gatego.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,18 +71,19 @@ func (gg GateGo) Run() error {
7171
}
7272

7373
func createChecker(services []config.Service) *Checker {
74-
checker := &Checker{Delay: 5 * time.Second, OnFailure: func(err error) {}}
74+
checker := &Checker{Delay: 5 * time.Second}
7575

7676
for _, service := range services {
7777
for _, path := range service.Paths {
7878
for _, checkConfig := range path.Checks {
7979
check := Check{
80-
Name: checkConfig.Name,
81-
Cron: checkConfig.Cron,
82-
URL: checkConfig.URL,
83-
Method: checkConfig.Method,
84-
Timeout: checkConfig.Timeout,
85-
Headers: checkConfig.Headers,
80+
Name: checkConfig.Name,
81+
Cron: checkConfig.Cron,
82+
URL: checkConfig.URL,
83+
Method: checkConfig.Method,
84+
Timeout: checkConfig.Timeout,
85+
Headers: checkConfig.Headers,
86+
OnFailure: checkConfig.OnFailure,
8687
}
8788

8889
checker.Checks = append(checker.Checks, check)

0 commit comments

Comments
 (0)