monitのif failed urlのコードリーディング

2017-02-20

はじめに

以下のページで紹介されているような if failed url ... の挙動をコードリーディングしてみたメモです。

check host fak3r.com with address fak3r.com
if failed url http://fak3r.com
timeout 10 seconds for 1 cycles then alert
then alert

コードリーディングの対象のmonitのバージョンは5.11.0です。 monit 5.11.0のソースコード

if failed urlの設定ファイルの文法

src/p.y#L1009-L1029

connection の文法定義。

connection      : IF FAILED host port type protocol urloption nettimeout retry rate1
                  THEN action1 recovery {
                    portset.timeout = $<number>8;
                    portset.retry = $<number>9;
                    /* This is a workaround to support content match without having to create
                     an URL object. 'urloption' creates the Request_T object we need minus the
                     URL object, but with enough information to perform content test. 
                     TODO: Parser is in need of refactoring */
                    portset.url_request = urlrequest;
                    addeventaction(&(portset).action, $<number>12, $<number>13);
                    addport(&portset);
                  }
                | IF FAILED URL URLOBJECT urloption nettimeout retry rate1
                  THEN action1 recovery {
                    prepare_urlrequest($<url>4);
                    portset.timeout = $<number>6;
                    portset.retry = $<number>7;
                    addeventaction(&(portset).action, $<number>10, $<number>11);
                    addport(&portset);
                  }
                ;

nettimeout の文法定義。 src/p.y#L1421-L1427

nettimeout      : /* EMPTY */ {
                   $<number>$ = NET_TIMEOUT; // timeout is in milliseconds
                  }
                | TIMEOUT NUMBER SECOND {
                   $<number>$ = $2 * 1000; // net timeout is in milliseconds internally
                  }
                ;

NET_TIMEOUT の定義。 src/net.h#L40-L44

40
41
42
43
44
/**
 * Standard milliseconds to wait for a socket connection or for socket read
 * i/o before aborting
 */
#define NET_TIMEOUT 5000

ということで if failed url ... のデフォルトのタイムアウトは 5秒。

retry の文法定義。 src/p.y#L1429-L1435

retry           : /* EMPTY */ {
                   $<number>$ = 1;
                  }
                | RETRY NUMBER {
                   $<number>$ = $2;
                  }
                ;

デフォルトのリトライ回数は1。

rate1 の文法定義。 src/p.y#L1765-L1780

rate1           : /* EMPTY */
                | NUMBER CYCLE {
                    rate1.count  = $<number>1;
                    rate1.cycles = $<number>1;
                    if (rate1.cycles < 1 || rate1.cycles > BITMAP_MAX)
                      yyerror2("The number of cycles must be between 1 and %d", BITMAP_MAX);
                  }
                | NUMBER NUMBER CYCLE {
                    rate1.count  = $<number>1;
                    rate1.cycles = $<number>2;
                    if (rate1.cycles < 1 || rate1.cycles > BITMAP_MAX)
                      yyerror2("The number of cycles must be between 1 and %d", BITMAP_MAX);
                    if (rate1.count < 1 || rate1.count > rate1.cycles)
                      yyerror2("The number of events must be bigger then 0 and less than poll cycles");
                  }
                ;

rate1 の変数定義 src/p.y#L186

186
static struct myrate rate1 = {1, 1};

myrate 構造体の定義。 src/p.y#L132-L135

132
133
134
135
struct myrate {
  unsigned count;
  unsigned cycles;
};

addeventaction 関数の実装。 src/p.y#L3116-L3147

3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
/*
 * Set EventAction object
 */
static void addeventaction(EventAction_T *_ea, int failed, int succeeded) {
  EventAction_T ea;

  ASSERT(_ea);

  NEW(ea);
  NEW(ea->failed);
  NEW(ea->succeeded);

  ea->failed->id     = failed;
  ea->failed->count  = rate1.count;
  ea->failed->cycles = rate1.cycles;
  if (failed == ACTION_EXEC) {
    ASSERT(command1);
    ea->failed->exec = command1;
    command1 = NULL;
  }

  ea->succeeded->id     = succeeded;
  ea->succeeded->count  = rate2.count;
  ea->succeeded->cycles = rate2.cycles;
  if (succeeded == ACTION_EXEC) {
    ASSERT(command2);
    ea->succeeded->exec = command2;
    command2 = NULL;
  }
  *_ea = ea;
  reset_rateset();
}

addport 関数の実装。 src/p.y#L2543-L2602

2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
/*
 * Add the given portset to the current service's portlist
 */
static void addport(Port_T port) {
  Port_T p;

  ASSERT(port);

  NEW(p);
  p->port               = port->port;
  p->type               = port->type;
  p->socket             = port->socket;
  p->family             = port->family;
  p->action             = port->action;
  p->timeout            = port->timeout;
  p->retry              = port->retry;
  p->request            = port->request;
  p->generic            = port->generic;
  p->protocol           = port->protocol;
  p->pathname           = port->pathname;
  p->hostname           = port->hostname;
  p->url_request        = port->url_request;
  p->request_checksum   = port->request_checksum;
  p->request_hostheader = port->request_hostheader;
  p->http_headers       = port->http_headers;
  p->version            = port->version;
  p->operator           = port->operator;
  p->status             = port->status;
  memcpy(&p->ApacheStatus, &port->ApacheStatus, sizeof(struct apache_status));

  if (p->request_checksum) {
    cleanup_hash_string(p->request_checksum);
    if (strlen(p->request_checksum) == 32)
      p->request_hashtype = HASH_MD5;
    else if (strlen(p->request_checksum) == 40)
      p->request_hashtype = HASH_SHA1;
    else
      yyerror2("invalid checksum [%s]", p->request_checksum);
  } else
    p->request_hashtype = 0;

  if (port->SSL.use_ssl == TRUE) {
    if (!have_ssl()) {
      yyerror("ssl check cannot be activated. SSL is not supported");
    } else {
      if (port->SSL.certmd5 != NULL) {
        p->SSL.certmd5 = port->SSL.certmd5;
        cleanup_hash_string(p->SSL.certmd5);
      }
      p->SSL.use_ssl = TRUE;
      p->SSL.version = port->SSL.version;
    }
  }
  p->maxforward = port->maxforward;
  p->next = current->portlist;
  current->portlist = p;

  reset_portset();

}

Port_T の型定義。 src/monit.h#L457-L510

457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
/** Defines a port object */
typedef struct myport {
        char *hostname;                                     /**< Hostname to check */
        List_T http_headers; /**< Optional list of HTTP headers to send with request */
        char *request;                              /**< Specific protocol request */
        char *request_checksum;     /**< The optional checksum for a req. document */
        char *request_hostheader;            /**< The optional Host: header to use. Deprecated */
        char *pathname;                   /**< Pathname, in case of an UNIX socket */
        Generic_T generic;                                /**< Generic test handle */
        volatile int socket;                       /**< Socket used for connection */
        int type;                   /**< Socket type used for connection (UDP/TCP) */
        int family;             /**< Socket family used for connection (INET/UNIX) */
        int port;                                                  /**< Portnumber */
        int request_hashtype;   /**< The optional type of hash for a req. document */
        int maxforward;            /**< Optional max forward for protocol checking */
        int timeout; /**< The timeout in millseconds to wait for connect or read i/o */
        int retry;       /**< Number of connection retry before reporting an error */
        int is_available;                /**< TRUE if the server/port is available */
        int version;                                         /**< Protocol version */
        Operator_Type operator;                           /**< Comparison operator */
        int status;                                           /**< Protocol status */
        double response;                      /**< Socket connection response time */
        EventAction_T action;  /**< Description of the action upon event occurence */
        /** Apache-status specific parameters */
        struct apache_status {
                short loglimit;                  /**< Max percentage of logging processes */
                short loglimitOP;                                  /**< loglimit operator */
                short closelimit;             /**< Max percentage of closinging processes */
                short closelimitOP;                              /**< closelimit operator */
                short dnslimit;         /**< Max percentage of processes doing DNS lookup */
                short dnslimitOP;                                  /**< dnslimit operator */
                short keepalivelimit;          /**< Max percentage of keepalive processes */
                short keepalivelimitOP;                      /**< keepalivelimit operator */
                short replylimit;               /**< Max percentage of replying processes */
                short replylimitOP;                              /**< replylimit operator */
                short requestlimit;     /**< Max percentage of processes reading requests */
                short requestlimitOP;                          /**< requestlimit operator */
                short startlimit;            /**< Max percentage of processes starting up */
                short startlimitOP;                              /**< startlimit operator */
                short waitlimit;  /**< Min percentage of processes waiting for connection */
                short waitlimitOP;                                /**< waitlimit operator */
                short gracefullimit;/**< Max percentage of processes gracefully finishing */
                short gracefullimitOP;                        /**< gracefullimit operator */
                short cleanuplimit;      /**< Max percentage of processes in idle cleanup */
                short cleanuplimitOP;                          /**< cleanuplimit operator */
        } ApacheStatus;

        Ssl_T SSL;                                             /**< SSL definition */
        Protocol_T protocol;     /**< Protocol object for testing a port's service */
        Request_T url_request;             /**< Optional url client request object */

        /** For internal use */
        struct myport *next;                               /**< next port in chain */
} *Port_T;

check_process 関数の実装

check_process 関数の実装。 src/validate.c#L989-L1037

 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
/**
 * Validate a given process service s. Events are posted according to
 * its configuration. In case of a fatal event FALSE is returned.
 */
int check_process(Service_T s) {
        pid_t  pid = -1;
        Port_T pp = NULL;
        Resource_T pr = NULL;
        ASSERT(s);
        /* Test for running process */
        if (!(pid = Util_isProcessRunning(s, FALSE))) {
                Event_post(s, Event_Nonexist, STATE_FAILED, s->action_NONEXIST, "process is not running");
                return FALSE;
        } else {
                Event_post(s, Event_Nonexist, STATE_SUCCEEDED, s->action_NONEXIST, "process is running with pid %d", (int)pid);
        }
        /* Reset the exec and timeout errors if active ... the process is running (most probably after manual intervention) */
        if (IS_EVENT_SET(s->error, Event_Exec))
                Event_post(s, Event_Exec, STATE_SUCCEEDED, s->action_EXEC, "process is running after previous exec error (slow starting or manually recovered?)");
        if (IS_EVENT_SET(s->error, Event_Timeout))
                for (ActionRate_T ar = s->actionratelist; ar; ar = ar->next)
                        Event_post(s, Event_Timeout, STATE_SUCCEEDED, ar->action, "process is running after previous restart timeout (manually recovered?)");
        if (Run.doprocess) {
                if (update_process_data(s, ptree, ptreesize, pid)) {
                        check_process_state(s);
                        check_process_pid(s);
                        check_process_ppid(s);
                        if (s->uid)
                                check_uid(s);
                        if (s->euid)
                                check_euid(s);
                        if (s->gid)
                                check_gid(s);
                        if (s->uptimelist)
                                check_uptime(s);
                        for (pr = s->resourcelist; pr; pr = pr->next)
                                check_process_resources(s, pr);
                } else
                        LogError("'%s' failed to get service data\n", s->name);
        }
        /* Test each host:port and protocol in the service's portlist */
        if (s->portlist)
                /* skip further tests during startup timeout */
                if (s->start)
                        if (s->inf->priv.process.uptime < s->start->timeout) return TRUE;
                for (pp = s->portlist; pp; pp = pp->next)
                        check_connection(s, pp);
        return TRUE;
}

check_connection 関数の実装。 src/validate.c#L138-L206

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
/**
 * Test the connection and protocol
 */
static void check_connection(Service_T s, Port_T p) {
        Socket_T socket;
        volatile int retry_count = p->retry;
        volatile int rv = TRUE;
        char buf[STRLEN];
        char report[STRLEN] = {};
        struct timeval t1;
        struct timeval t2;
        
        ASSERT(s && p);
retry:
        /* Get time of connection attempt beginning */
        gettimeofday(&t1, NULL);
        
        /* Open a socket to the destination INET[hostname:port] or UNIX[pathname] */
        socket = socket_create(p);
        if (!socket) {
                snprintf(report, STRLEN, "failed, cannot open a connection to %s", Util_portDescription(p, buf, sizeof(buf)));
                rv = FALSE;
                goto error;
        } else {
                DEBUG("'%s' succeeded connecting to %s\n", s->name, Util_portDescription(p, buf, sizeof(buf)));
        }

        if (p->protocol->check == check_default) {
                if (socket_is_udp(socket)) {
                        // Only test "connected" UDP sockets without protocol, TCP connect is verified on create
                        if (! socket_is_ready(socket)) {
                                snprintf(report, STRLEN, "connection failed, %s is not ready for i|o -- %s", Util_portDescription(p, buf, sizeof(buf)), STRERROR);
                                rv = FALSE;
                                goto error;
                        }
                }
        }
        /* Run the protocol verification routine through the socket */
        if (! p->protocol->check(socket)) {
                snprintf(report, STRLEN, "failed protocol test [%s] at %s -- %s", p->protocol->name, Util_portDescription(p, buf, sizeof(buf)), socket_getError(socket));
                rv = FALSE;
                goto error;
        } else {
                DEBUG("'%s' succeeded testing protocol [%s] at %s\n", s->name, p->protocol->name, Util_portDescription(p, buf, sizeof(buf)));
        }
        
        /* Get time of connection attempt finish */
        gettimeofday(&t2, NULL);
        
        /* Get the response time */
        p->response = (double)(t2.tv_sec - t1.tv_sec) + (double)(t2.tv_usec - t1.tv_usec)/1000000;
        
error:
        if (socket)
                socket_free(&socket);
        if (!rv) {
                if (retry_count-- > 1) {
                        DEBUG("'%s' %s (attempt %d/%d)\n", s->name, report, p->retry - retry_count, p->retry);
                        goto retry;
                }
                p->response = -1;
                p->is_available = FALSE;
                Event_post(s, Event_Connection, STATE_FAILED, p->action, "%s", report);
        } else {
                p->is_available = TRUE;
                Event_post(s, Event_Connection, STATE_SUCCEEDED, p->action, "connection succeeded to %s", Util_portDescription(p, buf, sizeof(buf)));
        }
        
}

上記の194行目で retry_count-- > 1 という条件を満たすと retry ラベルに飛んでソケット通信を再度行います。 retry_count-- と後置演算子を使っているので、まず今の値で比較した後に retry_count を 1 減らします。

retry_count は143行目で volatile int retry_count = p->retry; のように初期化されています。もし p->retry が 1 だった場合は1度ソケット通信を行った後の retry_count-- > 1 が 1 > 1 で false になるので、再度ソケット通信は行われずに200行目で STATE_FAILED で Event_post が呼ばれることになります。

retry という名前から、まず1度通信してみて失敗したら追加で retry 回分の通信を試みるのかと想像していたのですが、上記のコードだと全体の通信回数が retry 回になるということですね。

195行目の DEBUG を見ても (attempt %d/%d) の値に対応する部分が p->retry - retry_count と p->retry となっていて、分母の全体の試行回数が p->retry ですのでこの理解で間違いなさそうです。

上記の200行と203行で呼んでいる Event_post 関数やイベントループの処理も気になりますが、この記事が長くなりすぎるので別記事にします。

Protocol_T の型定義。 src/monit.h#L438-L442

438
439
440
441
442
/** Defines a protocol object with protocol functions */
typedef struct Protocol_T {
        const char *name;                                       /**< Protocol name */
        int(*check)(Socket_T);                 /**< Protocol verification function */
} *Protocol_T;

Protocol_T 型の配列の値定義。monitで扱うプロトコル一覧。 src/protocols/protocol.c#L41-L83

41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
static Protocol_T protocols[] = {
        &(struct Protocol_T){"DEFAULT",         check_default},
        &(struct Protocol_T){"HTTP",            check_http},
        &(struct Protocol_T){"FTP",             check_ftp},
        &(struct Protocol_T){"SMTP",            check_smtp},
        &(struct Protocol_T){"POP",             check_pop},
        &(struct Protocol_T){"IMAP",            check_imap},
        &(struct Protocol_T){"NNTP",            check_nntp},
        &(struct Protocol_T){"SSH",             check_ssh},
        &(struct Protocol_T){"DWP",             check_dwp},
        &(struct Protocol_T){"LDAP2",           check_ldap2},
        &(struct Protocol_T){"LDAP3",           check_ldap3},
        &(struct Protocol_T){"RDATE",           check_rdate},
        &(struct Protocol_T){"RSYNC",           check_rsync},
        &(struct Protocol_T){"generic",         check_generic},
        &(struct Protocol_T){"APACHESTATUS",    check_apache_status},
        &(struct Protocol_T){"NTP3",            check_ntp3},
        &(struct Protocol_T){"MYSQL",           check_mysql},
        &(struct Protocol_T){"DNS",             check_dns},
        &(struct Protocol_T){"POSTFIX-POLICY",  check_postfix_policy},
        &(struct Protocol_T){"TNS",             check_tns},
        &(struct Protocol_T){"PGSQL",           check_pgsql},
        &(struct Protocol_T){"CLAMAV",          check_clamav},
        &(struct Protocol_T){"SIP",             check_sip},
        &(struct Protocol_T){"LMTP",            check_lmtp},
        &(struct Protocol_T){"GPS",             check_gps},
        &(struct Protocol_T){"RADIUS",          check_radius},
        &(struct Protocol_T){"MEMCACHE",        check_memcache},
        &(struct Protocol_T){"WEBSOCKET",       check_websocket},
        &(struct Protocol_T){"REDIS",           check_redis},
        &(struct Protocol_T){"MONGODB",         check_mongodb},
        &(struct Protocol_T){"SIEVE",           check_sieve}
};

/* ------------------------------------------------------------------ Public */

Protocol_T Protocol_get(Protocol_Type type) {
        if (type >= sizeof(protocols)/sizeof(protocols[0]))
                return protocols[0];
        return protocols[type];
}

check_http 関数の実装

check_http 関数の実装。 src/protocols/http.c#L276-L321

276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
int check_http(Socket_T socket) {
        Port_T P;
        char host[STRLEN];
        char auth[STRLEN] = {};
        const char *request = NULL;
        const char *hostheader = NULL;

        ASSERT(socket);

        P = socket_get_Port(socket);

        ASSERT(P);

        request = P->request ? P->request : "/";

        hostheader = _findHostHeaderIn(P->http_headers);
        hostheader = hostheader ? hostheader : P->request_hostheader
                                ? P->request_hostheader : Util_getHTTPHostHeader(socket, host, STRLEN); // Otherwise use deprecated request_hostheader or default host
        StringBuffer_T sb = StringBuffer_create(168);
        StringBuffer_append(sb,
                            "GET %s HTTP/1.1\r\n"
                            "Host: %s\r\n"
                            "Accept: */*\r\n"
                            "User-Agent: Monit/%s\r\n"
                            "%s",
                            request, hostheader, VERSION,
                            get_auth_header(P, auth, STRLEN));
        // Add headers if we have them
        if (P->http_headers) {
                for (list_t p = P->http_headers->head; p; p = p->next) {
                        char *header = p->e;
                        if (Str_startsWith(header, "Host")) // Already set contrived above
                                continue;
                        StringBuffer_append(sb, "%s\r\n", header);
                }
        }
        StringBuffer_append(sb, "\r\n");
        int send_status = socket_write(socket, (void*)StringBuffer_toString(sb), StringBuffer_length(sb));
        StringBuffer_free(&sb);
        if (send_status < 0) {
                socket_setError(socket, "HTTP: error sending data -- %s", STRERROR);
                return FALSE;
        }

        return check_request(socket, P);
}

Socket_T の型定義。 src/socket.c#L85-L103

 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
#define TYPE_LOCAL   0
#define TYPE_ACCEPT  1
// One TCP frame data size
#define RBUFFER_SIZE 1500

struct Socket_T {
        int port;
        int type;
        int socket;
        char *host;
        Port_T Port;
        int timeout; // milliseconds
        int connection_type;
        ssl_connection *ssl;
        ssl_server_connection *sslserver;
        int length;
        int offset;
        unsigned char buffer[RBUFFER_SIZE + 1];
};

check_request 関数の実装。 src/protocols/http.c#L199-L242

199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
/**
 * Check that the server returns a valid HTTP response as well as checksum
 * or content regex if required
 * @param s A socket
 * @return TRUE if the response is valid otherwise FALSE
 */
static int check_request(Socket_T socket, Port_T P) {
        int status, content_length = -1;
        char buf[LINE_SIZE];
        if (! socket_readln(socket, buf, LINE_SIZE)) {
                socket_setError(socket, "HTTP: Error receiving data -- %s", STRERROR);
                return FALSE;
        }
        Str_chomp(buf);
        if (! sscanf(buf, "%*s %d", &status)) {
                socket_setError(socket, "HTTP error: Cannot parse HTTP status in response: %s", buf);
                return FALSE;
        }
        if (! Util_evalQExpression(P->operator, status, P->status)) {
                socket_setError(socket, "HTTP error: Server returned status %d", status);
                return FALSE;
        }
        /* Get Content-Length header value */
        while (socket_readln(socket, buf, LINE_SIZE)) {
                if ((buf[0] == '\r' && buf[1] == '\n') || (buf[0] == '\n'))
                        break;
                Str_chomp(buf);
                if (Str_startsWith(buf, "Content-Length")) {
                        if (! sscanf(buf, "%*s%*[: ]%d", &content_length)) {
                                socket_setError(socket, "HTTP error: Parsing Content-Length response header '%s'", buf);
                                return FALSE;
                        }
                        if (content_length < 0) {
                                socket_setError(socket, "HTTP error: Illegal Content-Length response header '%s'", buf);
                                return FALSE;
                        }
                }
        }
        if (P->url_request && P->url_request->regex && ! do_regex(socket, content_length, P->url_request))
                return FALSE;
        if (P->request_checksum)
                return check_request_checksum(socket, content_length, P->request_checksum, P->request_hashtype);
        return TRUE;
}

monitのイベントループのコードリーディングに続きます。