1    	/*
2    	   SSSD
3    	
4    	   Fail over helper functions.
5    	
6    	   Authors:
7    	        Martin Nagy <mnagy@redhat.com>
8    	        Jakub Hrozek <jhrozek@redhat.com>
9    	
10   	   Copyright (C) Red Hat, Inc 2010
11   	
12   	   This program is free software; you can redistribute it and/or modify
13   	   it under the terms of the GNU General Public License as published by
14   	   the Free Software Foundation; either version 3 of the License, or
15   	   (at your option) any later version.
16   	
17   	   This program is distributed in the hope that it will be useful,
18   	   but WITHOUT ANY WARRANTY; without even the implied warranty of
19   	   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20   	   GNU General Public License for more details.
21   	
22   	   You should have received a copy of the GNU General Public License
23   	   along with this program.  If not, see <http://www.gnu.org/licenses/>.
24   	*/
25   	
26   	#include <sys/time.h>
27   	
28   	#include <errno.h>
29   	#include <stdbool.h>
30   	#include <strings.h>
31   	#include <talloc.h>
32   	
33   	#include "util/dlinklist.h"
34   	#include "util/refcount.h"
35   	#include "util/util.h"
36   	#include "providers/fail_over.h"
37   	#include "resolv/async_resolv.h"
38   	
39   	#define STATUS_DIFF(p, now) ((now).tv_sec - (p)->last_status_change.tv_sec)
40   	#define SERVER_NAME(s) ((s)->common ? (s)->common->name : "(no name)")
41   	
42   	#define DEFAULT_PORT_STATUS PORT_NEUTRAL
43   	#define DEFAULT_SERVER_STATUS SERVER_NAME_NOT_RESOLVED
44   	#define DEFAULT_SRV_STATUS SRV_NEUTRAL
45   	
46   	enum srv_lookup_status {
47   	    SRV_NEUTRAL,        /* We didn't try this SRV lookup yet */
48   	    SRV_RESOLVED,       /* This SRV lookup is resolved       */
49   	    SRV_NOT_RESOLVED,   /* Could not resolve this SRV lookup */
50   	    SRV_EXPIRED         /* Need to refresh the SRV query     */
51   	};
52   	
53   	struct fo_ctx {
54   	    struct fo_service *service_list;
55   	    struct server_common *server_common_list;
56   	
57   	    struct fo_options *opts;
58   	};
59   	
60   	struct fo_service {
61   	    struct fo_service *prev;
62   	    struct fo_service *next;
63   	
64   	    struct fo_ctx *ctx;
65   	    char *name;
66   	    struct fo_server *active_server;
67   	    struct fo_server *last_tried_server;
68   	    struct fo_server *server_list;
69   	};
70   	
71   	struct fo_server {
72   	    struct fo_server *prev;
73   	    struct fo_server *next;
74   	
75   	    void *user_data;
76   	    int port;
77   	    int port_status;
78   	    struct srv_data *srv_data;
79   	    struct fo_service *service;
80   	    struct timeval last_status_change;
81   	    struct server_common *common;
82   	};
83   	
84   	struct server_common {
85   	    REFCOUNT_COMMON;
86   	
87   	    struct fo_ctx *ctx;
88   	
89   	    struct server_common *prev;
90   	    struct server_common *next;
91   	
92   	    char *name;
93   	    struct hostent *hostent;
94   	    struct resolve_service_request *request_list;
95   	    int server_status;
96   	    struct timeval last_status_change;
97   	};
98   	
99   	struct srv_data {
100  	    char *domain;
101  	    char *proto;
102  	    char *srv;
103  	
104  	    struct fo_server *meta;
105  	
106  	    int srv_lookup_status;
107  	    struct timeval last_status_change;
108  	};
109  	
110  	struct resolve_service_request {
111  	    struct resolve_service_request *prev;
112  	    struct resolve_service_request *next;
113  	
114  	    struct server_common *server_common;
115  	    struct tevent_req *req;
116  	};
117  	
118  	struct status {
119  	    int value;
120  	    struct timeval last_change;
121  	};
122  	
123  	struct fo_ctx *
124  	fo_context_init(TALLOC_CTX *mem_ctx, struct fo_options *opts)
125  	{
126  	    struct fo_ctx *ctx;
127  	
128  	    ctx = talloc_zero(mem_ctx, struct fo_ctx);
129  	    if (ctx == NULL) {
130  	        DEBUG(1, ("No memory\n"));
131  	        return NULL;
132  	    }
133  	    ctx->opts = talloc_zero(ctx, struct fo_options);
134  	    if (ctx->opts == NULL) {
135  	        DEBUG(1, ("No memory\n"));
136  	        return NULL;
137  	    }
138  	
139  	    ctx->opts->srv_retry_timeout = opts->srv_retry_timeout;
140  	    ctx->opts->retry_timeout = opts->retry_timeout;
141  	    ctx->opts->family_order  = opts->family_order;
142  	
143  	    DEBUG(3, ("Created new fail over context, retry timeout is %d\n",
144  	              ctx->opts->retry_timeout));
145  	    return ctx;
146  	}
147  	
148  	static const char *
149  	str_port_status(enum port_status status)
150  	{
151  	    switch (status) {
152  	    case PORT_NEUTRAL:
153  	        return "neutral";
154  	    case PORT_WORKING:
155  	        return "working";
156  	    case PORT_NOT_WORKING:
157  	        return "not working";
158  	    }
159  	
160  	    return "unknown port status";
161  	}
162  	
163  	static const char *
164  	str_srv_data_status(enum srv_lookup_status status)
165  	{
166  	    switch (status) {
167  	    case SRV_NEUTRAL:
168  	        return "neutral";
169  	    case SRV_RESOLVED:
170  	        return "resolved";
171  	    case SRV_NOT_RESOLVED:
172  	        return "not resolved";
173  	    case SRV_EXPIRED:
174  	        return "expired";
175  	    }
176  	
177  	    return "unknown SRV lookup status";
178  	}
179  	
180  	static const char *
181  	str_server_status(enum server_status status)
182  	{
183  	    switch (status) {
184  	    case SERVER_NAME_NOT_RESOLVED:
185  	        return "name not resolved";
186  	    case SERVER_RESOLVING_NAME:
187  	        return "resolving name";
188  	    case SERVER_NAME_RESOLVED:
189  	        return "name resolved";
190  	    case SERVER_WORKING:
191  	        return "working";
192  	    case SERVER_NOT_WORKING:
193  	        return "not working";
194  	    }
195  	
196  	    return "unknown server status";
197  	}
198  	
199  	int fo_is_srv_lookup(struct fo_server *s)
200  	{
201  	    return s && s->srv_data;
202  	}
203  	
204  	static char *
205  	get_srv_query(TALLOC_CTX *mem_ctx, struct fo_server *server)
206  	{
207  	    char *query;
208  	
209  	    if (!fo_is_srv_lookup(server)) {
210  	        return NULL;
211  	    }
212  	
213  	    query = talloc_asprintf(mem_ctx, "_%s._%s.%s", server->srv_data->srv,
214  	                                                   server->srv_data->proto,
215  	                                                   server->srv_data->domain);
216  	    return query;
217  	}
218  	
219  	static struct fo_server *
220  	collapse_srv_lookup(struct fo_server *server)
221  	{
222  	    struct fo_server *tmp, *meta;
223  	
224  	    meta = server->srv_data->meta;
225  	    DEBUG(4, ("Need to refresh SRV lookup for domain %s\n", meta->srv_data->domain))
226  	
227  	    if (server != meta) {
228  	        while (server->prev && server->prev->srv_data == meta->srv_data) {
229  	            tmp = server->prev;
230  	            DLIST_REMOVE(server->service->server_list, tmp);
231  	            talloc_zfree(tmp);
232  	        }
233  	        while (server->next && server->next->srv_data == meta->srv_data) {
234  	            tmp = server->next;
235  	            DLIST_REMOVE(server->service->server_list, tmp);
236  	            talloc_zfree(tmp);
237  	        }
238  	
239  	        if (server == server->service->active_server) {
240  	            server->service->active_server = NULL;
241  	        }
242  	        if (server == server->service->last_tried_server) {
243  	            server->service->last_tried_server = meta;
244  	        }
245  	
246  	        /* add back the meta server to denote SRV lookup */
247  	        DLIST_ADD_AFTER(server->service->server_list, meta, server);
248  	        DLIST_REMOVE(server->service->server_list, server);
249  	        talloc_zfree(server);
250  	    }
251  	
252  	    meta->srv_data->srv_lookup_status = SRV_NEUTRAL;
253  	    meta->srv_data->last_status_change.tv_sec = 0;
254  	
255  	    return meta;
256  	}
257  	
258  	static enum srv_lookup_status
259  	get_srv_data_status(struct srv_data *data)
260  	{
261  	    struct timeval tv;
262  	    time_t timeout;
263  	
264  	    timeout = data->meta->service->ctx->opts->srv_retry_timeout;
265  	    gettimeofday(&tv, NULL);
266  	
267  	    if (timeout && STATUS_DIFF(data, tv) > timeout) {
268  	        switch(data->srv_lookup_status) {
269  	        case SRV_EXPIRED:
270  	        case SRV_NEUTRAL:
271  	            break;
272  	        case SRV_RESOLVED:
273  	            data->srv_lookup_status = SRV_EXPIRED;
274  	            data->last_status_change.tv_sec = 0;
275  	            break;
276  	        case SRV_NOT_RESOLVED:
277  	            data->srv_lookup_status = SRV_NEUTRAL;
278  	            data->last_status_change.tv_sec = 0;
279  	            break;
280  	        default:
281  	            DEBUG(1, ("Unknown state for SRV server!\n"));
282  	        }
283  	    }
284  	
285  	    return data->srv_lookup_status;
286  	}
287  	
288  	static void
289  	set_srv_data_status(struct srv_data *data, enum srv_lookup_status status)
290  	{
291  	    DEBUG(4, ("Marking SRV lookup of service '%s' as '%s'\n",
292  	              data->meta->service->name, str_srv_data_status(status)));
293  	
294  	    gettimeofday(&data->last_status_change, NULL);
295  	    data->srv_lookup_status = status;
296  	}
297  	
298  	/*
299  	 * This function will return the status of the server. If the status was
300  	 * last updated a long time ago, we will first reset the status.
301  	 */
302  	static enum server_status
303  	get_server_status(struct fo_server *server)
304  	{
305  	    struct timeval tv;
306  	    time_t timeout;
307  	
308  	    if (server->common == NULL)
309  	        return SERVER_NAME_RESOLVED;
310  	
311  	    DEBUG(7, ("Status of server '%s' is '%s'\n", SERVER_NAME(server),
312  	              str_server_status(server->common->server_status)));
313  	
314  	    timeout = server->service->ctx->opts->retry_timeout;
315  	    if (timeout != 0 && server->common->server_status == SERVER_NOT_WORKING) {
316  	        gettimeofday(&tv, NULL);
317  	        if (STATUS_DIFF(server->common, tv) > timeout) {
318  	            DEBUG(4, ("Reseting the server status of '%s'\n",
319  	                      SERVER_NAME(server)));
320  	            server->common->server_status = SERVER_NAME_NOT_RESOLVED;
321  	            server->last_status_change.tv_sec = 0;
322  	        }
323  	    }
324  	
325  	    return server->common->server_status;
326  	}
327  	
328  	/*
329  	 * This function will return the status of the service. If the status was
330  	 * last updated a long time ago, we will first reset the status.
331  	 */
332  	static enum port_status
333  	get_port_status(struct fo_server *server)
334  	{
335  	    struct timeval tv;
336  	    time_t timeout;
337  	
338  	    DEBUG(7, ("Port status of port %d for server '%s' is '%s'\n", server->port,
339  	              SERVER_NAME(server), str_port_status(server->port_status)));
340  	
341  	    timeout = server->service->ctx->opts->retry_timeout;
342  	    if (timeout != 0 && server->port_status == PORT_NOT_WORKING) {
343  	        gettimeofday(&tv, NULL);
344  	        if (STATUS_DIFF(server, tv) > timeout) {
345  	            DEBUG(4, ("Reseting the status of port %d for server '%s'\n",
346  	                      server->port, SERVER_NAME(server)));
347  	            server->port_status = PORT_NEUTRAL;
348  	            server->last_status_change.tv_sec = tv.tv_sec;
349  	        }
350  	    }
351  	
352  	    return server->port_status;
353  	}
354  	
355  	static int
356  	server_works(struct fo_server *server)
357  	{
358  	    if (get_server_status(server) == SERVER_NOT_WORKING)
359  	        return 0;
360  	
361  	    return 1;
362  	}
363  	
364  	static int
365  	service_works(struct fo_server *server)
366  	{
367  	    if (!server_works(server))
368  	        return 0;
369  	    if (get_port_status(server) == PORT_NOT_WORKING)
370  	        return 0;
371  	
372  	    return 1;
373  	}
374  	
375  	static int
376  	service_destructor(struct fo_service *service)
377  	{
378  	    DLIST_REMOVE(service->ctx->service_list, service);
379  	    return 0;
380  	}
381  	
382  	int
383  	fo_new_service(struct fo_ctx *ctx, const char *name,
384  	               struct fo_service **_service)
385  	{
386  	    struct fo_service *service;
387  	    int ret;
388  	
389  	    DEBUG(3, ("Creating new service '%s'\n", name));
390  	    ret = fo_get_service(ctx, name, &service);
391  	    if (ret == EOK) {
392  	        DEBUG(5, ("Service '%s' already exists\n", name));
393  	        if (_service) {
394  	                *_service = service;
395  	        }
396  	        return EEXIST;
397  	    } else if (ret != ENOENT) {
398  	        return ret;
399  	    }
400  	
401  	    service = talloc_zero(ctx, struct fo_service);
402  	    if (service == NULL)
403  	        return ENOMEM;
404  	
405  	    service->name = talloc_strdup(service, name);
406  	    if (service->name == NULL) {
407  	        talloc_free(service);
408  	        return ENOMEM;
409  	    }
410  	
411  	    service->ctx = ctx;
412  	    DLIST_ADD(ctx->service_list, service);
413  	
414  	    talloc_set_destructor(service, service_destructor);
415  	    if (_service) {
416  	        *_service = service;
417  	    }
418  	
419  	    return EOK;
420  	}
421  	
422  	int
423  	fo_get_service(struct fo_ctx *ctx, const char *name,
424  	               struct fo_service **_service)
425  	{
426  	    struct fo_service *service;
427  	
428  	    DLIST_FOR_EACH(service, ctx->service_list) {
429  	        if (!strcmp(name, service->name)) {
430  	            *_service = service;
431  	            return EOK;
432  	        }
433  	    }
434  	
435  	    return ENOENT;
436  	}
437  	
438  	static int
439  	get_server_common(TALLOC_CTX *mem_ctx, struct fo_ctx *ctx, const char *name,
440  	                  struct server_common **_common)
441  	{
442  	    struct server_common *common;
443  	
444  	    DLIST_FOR_EACH(common, ctx->server_common_list) {
445  	        if (!strcasecmp(name, common->name)) {
446  	            *_common = rc_reference(mem_ctx, struct server_common, common);
447  	            if (_common == NULL)
448  	                return ENOMEM;
449  	            return EOK;
450  	        }
451  	    }
452  	
453  	    return ENOENT;
454  	}
455  	
456  	static int server_common_destructor(void *memptr)
457  	{
458  	    struct server_common *common;
459  	
460  	    common = talloc_get_type(memptr, struct server_common);
461  	    if (common->request_list) {
462  	        DEBUG(1, ("BUG: pending requests still associated with this server\n"));
463  	        return -1;
464  	    }
465  	    DLIST_REMOVE(common->ctx->server_common_list, common);
466  	
467  	    return 0;
468  	}
469  	
470  	static struct server_common *
471  	create_server_common(TALLOC_CTX *mem_ctx, struct fo_ctx *ctx, const char *name)
472  	{
473  	    struct server_common *common;
474  	
475  	    common = rc_alloc(mem_ctx, struct server_common);
476  	    if (common == NULL)
477  	        return NULL;
478  	
479  	    common->name = talloc_strdup(common, name);
480  	    if (common->name == NULL) {
481  	        talloc_free(common);
482  	        return NULL;
483  	    }
484  	
485  	    common->ctx = ctx;
486  	    common->prev = NULL;
487  	    common->next = NULL;
488  	    common->hostent = NULL;
489  	    common->request_list = NULL;
490  	    common->server_status = DEFAULT_SERVER_STATUS;
491  	    common->last_status_change.tv_sec = 0;
492  	    common->last_status_change.tv_usec = 0;
493  	
494  	    talloc_set_destructor((TALLOC_CTX *) common, server_common_destructor);
495  	    DLIST_ADD_END(ctx->server_common_list, common, struct server_common *);
496  	    return common;
497  	}
498  	
499  	int
500  	fo_add_srv_server(struct fo_service *service, const char *srv,
501  	                  const char *domain, const char *proto, void *user_data)
502  	{
503  	    struct fo_server *server;
504  	
505  	    DEBUG(3, ("Adding new SRV server in domain '%s', to service '%s'\n",
506  	              domain, service->name));
507  	
508  	    DLIST_FOR_EACH(server, service->server_list) {
509  	        if (server->user_data != user_data)
510  	            continue;
511  	
512  	        if (fo_is_srv_lookup(server)) {
513  	            if (strcasecmp(server->srv_data->domain, domain) == 0 &&
514  	                strcasecmp(server->srv_data->proto, proto) == 0) {
515  	                return EEXIST;
516  	            }
517  	        }
518  	    }
519  	
520  	    server = talloc_zero(service, struct fo_server);
521  	    if (server == NULL)
522  	        return ENOMEM;
523  	
524  	    server->user_data = user_data;
525  	    server->service = service;
526  	    server->port_status = DEFAULT_PORT_STATUS;
527  	
528  	    /* add the SRV-specific data */
529  	    server->srv_data = talloc_zero(service, struct srv_data);
530  	    if (server->srv_data == NULL)
531  	        return ENOMEM;
532  	
533  	    server->srv_data->domain = talloc_strdup(server->srv_data, domain);
534  	    server->srv_data->proto = talloc_strdup(server->srv_data, proto);
535  	    server->srv_data->srv = talloc_strdup(server->srv_data, srv);
536  	    if (server->srv_data->domain == NULL ||
537  	        server->srv_data->proto == NULL ||
538  	        server->srv_data->srv == NULL)
539  	        return ENOMEM;
540  	
541  	    server->srv_data->meta = server;
542  	    server->srv_data->srv_lookup_status = DEFAULT_SRV_STATUS;
543  	    server->srv_data->last_status_change.tv_sec = 0;
544  	
545  	    DLIST_ADD_END(service->server_list, server, struct fo_server *);
546  	    return EOK;
547  	}
548  	
549  	static struct fo_server *
550  	create_fo_server(struct fo_service *service, const char *name,
551  	                 int port, void *user_data)
552  	{
553  	    struct fo_server *server;
554  	    int ret;
555  	
556  	    server = talloc_zero(service, struct fo_server);
557  	    if (server == NULL)
558  	        return NULL;
559  	
560  	    server->port = port;
561  	    server->user_data = user_data;
562  	    server->service = service;
563  	    server->port_status = DEFAULT_PORT_STATUS;
564  	
565  	    if (name != NULL) {
566  	        ret = get_server_common(server, service->ctx, name, &server->common);
567  	        if (ret == ENOENT) {
568  	            server->common = create_server_common(server, service->ctx, name);
569  	            if (server->common == NULL) {
570  	                talloc_free(server);
571  	                return NULL;
572  	            }
573  	        } else if (ret != EOK) {
574  	            talloc_free(server);
575  	            return NULL;
576  	        }
577  	    }
578  	
579  	    return server;
580  	}
581  	
582  	int
583  	fo_add_server(struct fo_service *service, const char *name, int port,
584  	              void *user_data)
585  	{
586  	    struct fo_server *server;
587  	
588  	    DEBUG(3, ("Adding new server '%s', to service '%s'\n",
589  	              name ? name : "(no name)", service->name));
590  	    DLIST_FOR_EACH(server, service->server_list) {
591  	        if (server->port != port || server->user_data != user_data)
592  	            continue;
593  	        if (name == NULL && server->common == NULL) {
594  	            return EEXIST;
595  	        } else if (name != NULL && server->common != NULL) {
596  	            if (!strcasecmp(name, server->common->name))
597  	                return EEXIST;
598  	        }
599  	    }
600  	
601  	    server = create_fo_server(service, name, port, user_data);
602  	    if (!server) {
603  	        return ENOMEM;
604  	    }
605  	
606  	    DLIST_ADD_END(service->server_list, server, struct fo_server *);
607  	
608  	    return EOK;
609  	}
610  	
611  	static int
612  	get_first_server_entity(struct fo_service *service, struct fo_server **_server)
613  	{
614  	    struct fo_server *server;
615  	
616  	    /* If we already have a working server, use that one. */
617  	    server = service->active_server;
618  	    if (server != NULL) {
619  	        if (service_works(server)) {
620  	            goto done;
621  	        }
622  	        service->active_server = NULL;
623  	    }
624  	
625  	    /*
626  	     * Otherwise iterate through the server list.
627  	     */
628  	
629  	    /* First, try servers after the last one we tried. */
630  	    if (service->last_tried_server != NULL) {
631  	        DLIST_FOR_EACH(server, service->last_tried_server->next) {
632  	            if (service_works(server)) {
633  	                goto done;
634  	            }
635  	        }
636  	    }
637  	
638  	    /* If none were found, try at the start. */
639  	    DLIST_FOR_EACH(server, service->server_list) {
640  	        if (service_works(server)) {
641  	            goto done;
642  	        }
643  	        if (server == service->last_tried_server) {
644  	            break;
645  	        }
646  	    }
647  	
648  	    service->last_tried_server = NULL;
649  	    return ENOENT;
650  	
651  	done:
652  	    service->last_tried_server = server;
653  	    *_server = server;
654  	    return EOK;
655  	}
656  	
657  	static int
658  	resolve_service_request_destructor(struct resolve_service_request *request)
659  	{
660  	    DLIST_REMOVE(request->server_common->request_list, request);
661  	    return 0;
662  	}
663  	
664  	static int
665  	set_lookup_hook(struct fo_server *server, struct tevent_req *req)
666  	{
667  	    struct resolve_service_request *request;
668  	
669  	    request = talloc(req, struct resolve_service_request);
670  	    if (request == NULL) {
671  	        DEBUG(1, ("No memory\n"));
672  	        talloc_free(request);
673  	        return ENOMEM;
674  	    }
675  	    request->server_common = rc_reference(request, struct server_common,
676  	                                          server->common);
677  	    if (request->server_common == NULL) {
678  	        talloc_free(request);
679  	        return ENOMEM;
680  	    }
681  	    request->req = req;
682  	    DLIST_ADD(server->common->request_list, request);
683  	    talloc_set_destructor(request, resolve_service_request_destructor);
684  	
685  	    return EOK;
686  	}
687  	
688  	/*******************************************************************
689  	 * Get server to connect to.                                       *
690  	 *******************************************************************/
691  	
692  	struct resolve_service_state {
693  	    struct fo_server *server;
694  	
695  	    struct resolv_ctx *resolv;
696  	    struct tevent_context *ev;
697  	    struct fo_ctx *fo_ctx;
698  	};
699  	
700  	
701  	static void fo_resolve_service_cont(struct tevent_req *subreq);
702  	static void fo_resolve_service_done(struct tevent_req *subreq);
703  	static bool fo_resolve_service_server(struct tevent_req *req);
704  	
705  	/* Forward declarations for SRV resolving */
706  	static struct tevent_req *
707  	resolve_srv_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
708  	                    struct resolv_ctx *resolv, struct fo_ctx *ctx,
709  	                    struct fo_server *server);
710  	static int
711  	resolve_srv_recv(struct tevent_req *req, struct fo_server **server);
712  	
713  	struct tevent_req *
714  	fo_resolve_service_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
715  	                        struct resolv_ctx *resolv, struct fo_ctx *ctx,
716  	                        struct fo_service *service)
717  	{
718  	    int ret;
719  	    struct fo_server *server;
720  	    struct tevent_req *req;
721  	    struct tevent_req *subreq;
722  	    struct resolve_service_state *state;
723  	
724  	    DEBUG(4, ("Trying to resolve service '%s'\n", service->name));
725  	    req = tevent_req_create(mem_ctx, &state, struct resolve_service_state);
726  	    if (req == NULL)
727  	        return NULL;
728  	
729  	    state->resolv = resolv;
730  	    state->ev = ev;
731  	    state->fo_ctx = ctx;
732  	
733  	    ret = get_first_server_entity(service, &server);
734  	    if (ret != EOK) {
735  	        DEBUG(1, ("No available servers for service '%s'\n", service->name));
736  	        goto done;
737  	    }
738  	
739  	    if (fo_is_srv_lookup(server)) {
740  	        /* Don't know the server yet, must do a SRV lookup */
741  	        subreq = resolve_srv_send(state, ev, resolv,
742  	                                  ctx, server);
743  	        if (subreq == NULL) {
744  	            ret = ENOMEM;
745  	            goto done;
746  	        }
747  	
748  	        tevent_req_set_callback(subreq,
749  	                                fo_resolve_service_cont,
750  	                                req);
751  	        return req;
752  	    }
753  	
754  	    /* This is a regular server, just do hostname lookup */
755  	    state->server = server;
756  	    if (fo_resolve_service_server(req)) {
757  	        tevent_req_post(req, ev);
758  	    }
759  	
760  	    ret = EOK;
761  	done:
762  	    if (ret != EOK) {
763  	        tevent_req_error(req, ret);
764  	        tevent_req_post(req, ev);
765  	    }
766  	    return req;
767  	}
768  	
769  	static void set_server_common_status(struct server_common *common,
770  	                                     enum server_status status);
771  	
772  	/* SRV resolving finished, see if we got server to work with */
773  	static void
774  	fo_resolve_service_cont(struct tevent_req *subreq)
775  	{
776  	    struct tevent_req *req = tevent_req_callback_data(subreq,
777  	                                                      struct tevent_req);
778  	    struct resolve_service_state *state = tevent_req_data(req,
779  	                                        struct resolve_service_state);
780  	    int ret;
781  	
782  	    ret = resolve_srv_recv(subreq, &state->server);
783  	    talloc_zfree(subreq);
784  	
785  	    if (ret) {
786  	        tevent_req_error(req, ret);
787  	        return;
788  	    }
789  	
790  	    fo_resolve_service_server(req);
791  	}
792  	
793  	static bool
794  	fo_resolve_service_server(struct tevent_req *req)
795  	{
796  	    struct resolve_service_state *state = tevent_req_data(req,
797  	                                        struct resolve_service_state);
798  	    struct tevent_req *subreq;
799  	    int ret;
800  	
801  	    switch (get_server_status(state->server)) {
802  	    case SERVER_NAME_NOT_RESOLVED: /* Request name resolution. */
803  	        subreq = resolv_gethostbyname_send(state->server->common,
804  	                                           state->ev, state->resolv,
805  	                                           state->server->common->name,
806  	                                           state->fo_ctx->opts->family_order);
807  	        if (subreq == NULL) {
808  	            tevent_req_error(req, ENOMEM);
809  	            return true;
810  	        }
811  	        tevent_req_set_callback(subreq, fo_resolve_service_done, req);
812  	        fo_set_server_status(state->server, SERVER_RESOLVING_NAME);
813  	        /* FALLTHROUGH */
814  	    case SERVER_RESOLVING_NAME:
815  	        /* Name resolution is already under way. Just add ourselves into the
816  	         * waiting queue so we get notified after the operation is finished. */
817  	        ret = set_lookup_hook(state->server, req);
818  	        if (ret != EOK) {
819  	            tevent_req_error(req, ret);
820  	            return true;
821  	        }
822  	        break;
823  	    default: /* The name is already resolved. Return immediately. */
824  	        tevent_req_done(req);
825  	        return true;
826  	    }
827  	
828  	    return false;
829  	}
830  	
831  	static void
832  	fo_resolve_service_done(struct tevent_req *subreq)
833  	{
834  	    struct tevent_req *req = tevent_req_callback_data(subreq,
835  	                                                      struct tevent_req);
836  	    struct resolve_service_state *state = tevent_req_data(req,
837  	                                                struct resolve_service_state);
838  	    struct server_common *common;
839  	    int resolv_status;
840  	    struct resolve_service_request *request;
841  	    int ret;
842  	
843  	    if (state->server->common->hostent != NULL) {
844  	        talloc_zfree(state->server->common->hostent);
845  	    }
846  	
847  	    ret = resolv_gethostbyname_recv(subreq, state->server->common,
848  	                                    &resolv_status, NULL,
849  	                                    &state->server->common->hostent);
850  	    talloc_zfree(subreq);
851  	    if (ret != EOK) {
852  	        DEBUG(1, ("Failed to resolve server '%s': %s\n",
853  	                  state->server->common->name,
854  	                  resolv_strerror(resolv_status)));
855  	        set_server_common_status(state->server->common, SERVER_NOT_WORKING);
856  	    } else {
857  	        set_server_common_status(state->server->common, SERVER_NAME_RESOLVED);
858  	    }
859  	
860  	    /* Take care of all requests for this server. */
861  	    common = state->server->common; /* state can disappear now */
862  	    while ((request = common->request_list) != NULL) {
863  	        DLIST_REMOVE(common->request_list, request);
864  	        if (resolv_status) {
865  	            /* FIXME FIXME: resolv_status is an ARES error.
866  	             * but any caller will expect classic error codes.
867  	             * also the send() function may return ENOENT, so this mix
868  	             * IS explosive (ENOENT = 2 = ARES_EFORMER) */
869  	            tevent_req_error(request->req, resolv_status);
870  	        } else {
871  	            tevent_req_done(request->req);
872  	        }
873  	    }
874  	}
875  	
876  	int
877  	fo_resolve_service_recv(struct tevent_req *req, struct fo_server **server)
878  	{
879  	    struct resolve_service_state *state;
880  	
881  	    state = tevent_req_data(req, struct resolve_service_state);
882  	
883  	    /* always return the server if asked for, otherwise the caller
884  	     * cannot mark it as faulty in case we return an error */
885  	    if (server)
886  	        *server = state->server;
887  	
888  	    TEVENT_REQ_RETURN_ON_ERROR(req);
889  	
890  	    return EOK;
891  	}
892  	
893  	/*******************************************************************
894  	 * Resolve the server to connect to using a SRV query.             *
895  	 *******************************************************************/
896  	
897  	static void resolve_srv_done(struct tevent_req *subreq);
898  	
899  	struct resolve_srv_state {
900  	    struct fo_server *meta;
901  	    struct fo_service *service;
902  	
903  	    struct fo_server *out;
904  	
905  	    struct resolv_ctx *resolv;
906  	    struct tevent_context *ev;
907  	    struct fo_ctx *fo_ctx;
908  	};
909  	
910  	static struct tevent_req *
911  	resolve_srv_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
912  	                 struct resolv_ctx *resolv, struct fo_ctx *ctx,
913  	                 struct fo_server *server)
914  	{
915  	    int ret;
916  	    char *query;
917  	    struct tevent_req *req;
918  	    struct tevent_req *subreq;
919  	    struct resolve_srv_state *state;
920  	    int status;
921  	
922  	    req = tevent_req_create(mem_ctx, &state, struct resolve_srv_state);
923  	    if (req == NULL)
924  	        return NULL;
925  	
926  	    state->service = server->service;
927  	    state->ev = ev;
928  	    state->resolv = resolv;
929  	    state->fo_ctx = ctx;
930  	    state->meta = server;
931  	
932  	    status = get_srv_data_status(server->srv_data);
933  	    DEBUG(6, ("The status of SRV lookup is %s\n",
934  	              str_srv_data_status(status)));
935  	    switch(status) {
936  	    case SRV_EXPIRED: /* Need a refresh */
937  	        state->meta = collapse_srv_lookup(server);
938  	        /* FALLTHROUGH */
939  	    case SRV_NEUTRAL: /* Request SRV lookup */
940  	        query = get_srv_query(state, state->meta);
941  	        if (!query) {
942  	            ret = ENOMEM;
943  	            goto done;
944  	        }
945  	        DEBUG(4, ("Searching for servers via SRV query '%s'\n", query));
946  	
947  	        subreq = resolv_getsrv_send(state, ev, resolv, query);
948  	        if (subreq == NULL) {
949  	            ret = ENOMEM;
950  	            goto done;
951  	        }
952  	        tevent_req_set_callback(subreq, resolve_srv_done, req);
953  	        break;
954  	    case SRV_NOT_RESOLVED: /* query could not be resolved but don't retry yet */
955  	        ret = EIO;
956  	        goto done;
957  	    case SRV_RESOLVED:  /* The query is resolved and valid. Return. */
958  	        state->out = server;
959  	        tevent_req_done(req);
960  	        tevent_req_post(req, state->ev);
961  	        return req;
962  	    default:
963  	        DEBUG(1, ("Unexpected status %d for a SRV server\n", status));
964  	        ret = EIO;
965  	        break;
966  	    }
967  	
968  	    ret = EOK;
969  	done:
970  	    if (ret != EOK) {
971  	        tevent_req_error(req, ret);
972  	        tevent_req_post(req, ev);
973  	    }
974  	    return req;
975  	}
976  	
977  	static void
978  	resolve_srv_done(struct tevent_req *subreq)
979  	{
980  	    struct tevent_req *req = tevent_req_callback_data(subreq,
981  	                                                      struct tevent_req);
982  	    struct resolve_srv_state *state = tevent_req_data(req,
983  	                                                struct resolve_srv_state);
984  	    struct ares_srv_reply *reply_list;
985  	    struct ares_srv_reply *reply;
986  	    struct fo_server *server = NULL;
987  	    struct fo_server *srv_list = NULL;
988  	    int ret;
989  	    int resolv_status;
990  	
991  	    ret = resolv_getsrv_recv(state, subreq,
992  	                             &resolv_status, NULL, &reply_list);
993  	    talloc_free(subreq);
994  	    if (ret != EOK) {
995  	        DEBUG(1, ("SRV query failed %s\n",
996  	                  resolv_strerror(resolv_status)));
997  	        fo_set_port_status(state->meta, PORT_NOT_WORKING);
998  	        goto fail;
999  	    }
1000 	
1001 	    ret = resolv_sort_srv_reply(state, &reply_list);
1002 	    if (ret != EOK) {
1003 	        DEBUG(1, ("Could not sort the answers from DNS [%d]: %s\n",
1004 	                  ret, strerror(ret)));
1005 	        fo_set_port_status(state->meta, PORT_NOT_WORKING);
1006 	        goto fail;
1007 	    }
1008 	
1009 	    for (reply = reply_list; reply; reply = reply->next) {
1010 	        ret = EOK;
1011 	        DLIST_FOR_EACH(server, state->service->server_list) {
1012 	            if (server->port == reply->port) {
1013 	                ret = EEXIST;
1014 	                break;
1015 	            }
1016 	        }
1017 	        if (ret == EEXIST) continue;
1018 	
1019 	        server = create_fo_server(state->service, reply->host,
1020 	                                  reply->port, state->meta->user_data);
1021 	        if (!server) {
1022 	            ret = ENOMEM;
1023 	            goto fail;
1024 	        }
1025 	        server->srv_data = state->meta->srv_data;
1026 	
1027 	        DLIST_ADD_END(srv_list, server, struct fo_server *);
1028 	        DEBUG(6, ("Inserted server '%s:%d' for service %s\n",
1029 	                  server->common->name,
1030 	                  server->port,
1031 	                  state->service->name));
1032 	    }
1033 	
1034 	    if (srv_list) {
1035 	        DLIST_ADD_LIST_AFTER(state->service->server_list, state->meta,
1036 	                             srv_list, struct fo_server *);
1037 	
1038 	        DLIST_REMOVE(state->service->server_list, state->meta);
1039 	        if (state->service->last_tried_server == state->meta) {
1040 	            state->service->last_tried_server = srv_list;
1041 	        }
1042 	
1043 	        state->out = srv_list;
1044 	        set_srv_data_status(state->meta->srv_data, SRV_RESOLVED);
1045 	        tevent_req_done(req);
1046 	        return;
1047 	    } else {
1048 	        ret = EIO;
1049 	        goto fail;
1050 	    }
1051 	
1052 	fail:
1053 	    state->out = state->meta;
1054 	    set_srv_data_status(state->meta->srv_data, SRV_NOT_RESOLVED);
1055 	    tevent_req_error(req, ret);
1056 	}
1057 	
1058 	static int
1059 	resolve_srv_recv(struct tevent_req *req, struct fo_server **server)
1060 	{
1061 	    struct resolve_srv_state *state = tevent_req_data(req,
1062 	                                                struct resolve_srv_state);
1063 	
1064 	    /* always return the server if asked for, otherwise the caller
1065 	     * cannot mark it as faulty in case we return an error */
1066 	    if (server) {
1067 	        *server = state->out;
1068 	    }
1069 	
1070 	    TEVENT_REQ_RETURN_ON_ERROR(req);
1071 	
1072 	    return EOK;
1073 	}
1074 	
1075 	static void
1076 	set_server_common_status(struct server_common *common,
1077 	                         enum server_status status)
1078 	{
1079 	    DEBUG(4, ("Marking server '%s' as '%s'\n", common->name,
1080 	              str_server_status(status)));
1081 	
1082 	    common->server_status = status;
1083 	    gettimeofday(&common->last_status_change, NULL);
1084 	}
1085 	
1086 	void
1087 	fo_set_server_status(struct fo_server *server, enum server_status status)
1088 	{
1089 	    if (server->common == NULL) {
1090 	        DEBUG(1, ("Bug: Trying to set server status of a name-less server\n"));
1091 	        return;
1092 	    }
1093 	
1094 	    set_server_common_status(server->common, status);
1095 	}
1096 	
1097 	void
1098 	fo_set_port_status(struct fo_server *server, enum port_status status)
1099 	{
1100 	    DEBUG(4, ("Marking port %d of server '%s' as '%s'\n", server->port,
1101 	              SERVER_NAME(server), str_port_status(status)));
1102 	
1103 	    server->port_status = status;
1104 	    gettimeofday(&server->last_status_change, NULL);
1105 	    if (status == PORT_WORKING) {
1106 	        fo_set_server_status(server, SERVER_WORKING);
1107 	        server->service->active_server = server;
1108 	    }
1109 	}
1110 	
1111 	void *
1112 	fo_get_server_user_data(struct fo_server *server)
1113 	{
1114 	    return server->user_data;
1115 	}
1116 	
1117 	int
1118 	fo_get_server_port(struct fo_server *server)
1119 	{
1120 	    return server->port;
1121 	}
1122 	
1123 	const char *fo_get_server_name(struct fo_server *server)
1124 	{
Event var_compare_op: Comparing "server->common" to null implies that "server->common" might be null.
Also see events: [var_deref_op]
At conditional (1): "!server->common": Taking true branch.
At conditional (2): "fo_is_srv_lookup(server)": Taking false branch.
1125 	    if (!server->common && fo_is_srv_lookup(server)) {
1126 	        return "SRV lookup meta-server";
1127 	    }
1128 	
Event var_deref_op: Dereferencing null variable "server->common".
Also see events: [var_compare_op]
1129 	    return server->common->name;
1130 	}
1131 	
1132 	struct hostent *
1133 	fo_get_server_hostent(struct fo_server *server)
1134 	{
1135 	    if (server->common == NULL) {
1136 	        DEBUG(1, ("Bug: Trying to get hostent from a name-less server\n"));
1137 	        return NULL;
1138 	    }
1139 	    return server->common->hostent;
1140 	}