corosync  2.3.5
totemudpu.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2005 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8 
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <assert.h>
39 #include <sys/mman.h>
40 #include <sys/types.h>
41 #include <sys/stat.h>
42 #include <sys/socket.h>
43 #include <netdb.h>
44 #include <sys/un.h>
45 #include <sys/ioctl.h>
46 #include <sys/param.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <sched.h>
55 #include <time.h>
56 #include <sys/time.h>
57 #include <sys/poll.h>
58 #include <sys/uio.h>
59 #include <limits.h>
60 
61 #include <qb/qbdefs.h>
62 #include <qb/qbloop.h>
63 
64 #include <corosync/sq.h>
65 #include <corosync/list.h>
66 #include <corosync/swab.h>
67 #define LOGSYS_UTILS_ONLY 1
68 #include <corosync/logsys.h>
69 #include "totemudpu.h"
70 
71 #include "util.h"
72 #include "totemcrypto.h"
73 
74 #include <nss.h>
75 #include <pk11pub.h>
76 #include <pkcs11.h>
77 #include <prerror.h>
78 
79 #ifndef MSG_NOSIGNAL
80 #define MSG_NOSIGNAL 0
81 #endif
82 
83 #define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
84 #define NETIF_STATE_REPORT_UP 1
85 #define NETIF_STATE_REPORT_DOWN 2
86 
87 #define BIND_STATE_UNBOUND 0
88 #define BIND_STATE_REGULAR 1
89 #define BIND_STATE_LOOPBACK 2
90 
92  struct list_head list;
94  int fd;
95  int active;
96 };
97 
100 
102 
104 
106 
108 
109  void *context;
110 
111  void (*totemudpu_deliver_fn) (
112  void *context,
113  const void *msg,
114  unsigned int msg_len);
115 
116  void (*totemudpu_iface_change_fn) (
117  void *context,
118  const struct totem_ip_address *iface_address);
119 
120  void (*totemudpu_target_set_completed) (void *context);
121 
122  /*
123  * Function and data used to log messages
124  */
126 
128 
130 
132 
134 
136 
137  void (*totemudpu_log_printf) (
138  int level,
139  int subsys,
140  const char *function,
141  const char *file,
142  int line,
143  const char *format,
144  ...)__attribute__((format(printf, 6, 7)));
145 
146  void *udpu_context;
147 
148  char iov_buffer[FRAME_SIZE_MAX];
149 
150  struct iovec totemudpu_iov_recv;
151 
152  struct list_head member_list;
153 
155 
157 
159 
161 
163 
164  struct timeval stats_tv_start;
165 
166  struct totem_ip_address my_id;
167 
168  int firstrun;
169 
170  qb_loop_timer_handle timer_netif_check_timeout;
171 
172  unsigned int my_memb_entries;
173 
175 
177 
178  struct totem_ip_address token_target;
179 
181 
182  qb_loop_timer_handle timer_merge_detect_timeout;
183 
185 
187 };
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
195 static int totemudpu_build_sockets (
196  struct totemudpu_instance *instance,
197  struct totem_ip_address *bindnet_address,
198  struct totem_ip_address *bound_to);
199 
200 static int totemudpu_create_sending_socket(
201  void *udpu_context,
202  const struct totem_ip_address *member);
203 
205  void *udpu_context);
206 
207 static void totemudpu_start_merge_detect_timeout(
208  void *udpu_context);
209 
210 static void totemudpu_stop_merge_detect_timeout(
211  void *udpu_context);
212 
213 static struct totem_ip_address localhost;
214 
215 static void totemudpu_instance_initialize (struct totemudpu_instance *instance)
216 {
217  memset (instance, 0, sizeof (struct totemudpu_instance));
218 
220 
221  instance->totemudpu_iov_recv.iov_base = instance->iov_buffer;
222 
223  instance->totemudpu_iov_recv.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
224 
225  /*
226  * There is always atleast 1 processor
227  */
228  instance->my_memb_entries = 1;
229 
230  list_init (&instance->member_list);
231 }
232 
233 #define log_printf(level, format, args...) \
234 do { \
235  instance->totemudpu_log_printf ( \
236  level, instance->totemudpu_subsys_id, \
237  __FUNCTION__, __FILE__, __LINE__, \
238  (const char *)format, ##args); \
239 } while (0);
240 #define LOGSYS_PERROR(err_num, level, fmt, args...) \
241 do { \
242  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
243  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
244  instance->totemudpu_log_printf ( \
245  level, instance->totemudpu_subsys_id, \
246  __FUNCTION__, __FILE__, __LINE__, \
247  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
248  } while(0)
249 
251  void *udpu_context,
252  const char *cipher_type,
253  const char *hash_type)
254 {
255 
256  return (0);
257 }
258 
259 
260 static inline void ucast_sendmsg (
261  struct totemudpu_instance *instance,
262  struct totem_ip_address *system_to,
263  const void *msg,
264  unsigned int msg_len)
265 {
266  struct msghdr msg_ucast;
267  int res = 0;
268  size_t buf_out_len;
269  unsigned char buf_out[FRAME_SIZE_MAX];
270  struct sockaddr_storage sockaddr;
271  struct iovec iovec;
272  int addrlen;
273 
274  if (msg_len + crypto_get_current_sec_header_size(instance->crypto_inst) > sizeof(buf_out)) {
275  log_printf(LOGSYS_LEVEL_CRIT, "UDPU message for ucast is too big. Ignoring message");
276 
277  return ;
278  }
279 
280  /*
281  * Encrypt and digest the message
282  */
284  instance->crypto_inst,
285  (const unsigned char *)msg,
286  msg_len,
287  buf_out,
288  &buf_out_len) != 0) {
289  log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
290  return;
291  }
292 
293  iovec.iov_base = (void *)buf_out;
294  iovec.iov_len = buf_out_len;
295 
296  /*
297  * Build unicast message
298  */
300  instance->totem_interface->ip_port, &sockaddr, &addrlen);
301  memset(&msg_ucast, 0, sizeof(msg_ucast));
302  msg_ucast.msg_name = &sockaddr;
303  msg_ucast.msg_namelen = addrlen;
304  msg_ucast.msg_iov = (void *)&iovec;
305  msg_ucast.msg_iovlen = 1;
306 #ifdef HAVE_MSGHDR_CONTROL
307  msg_ucast.msg_control = 0;
308 #endif
309 #ifdef HAVE_MSGHDR_CONTROLLEN
310  msg_ucast.msg_controllen = 0;
311 #endif
312 #ifdef HAVE_MSGHDR_FLAGS
313  msg_ucast.msg_flags = 0;
314 #endif
315 #ifdef HAVE_MSGHDR_ACCRIGHTS
316  msg_ucast.msg_accrights = NULL;
317 #endif
318 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
319  msg_ucast.msg_accrightslen = 0;
320 #endif
321 
322 
323  /*
324  * Transmit unicast message
325  * An error here is recovered by totemsrp
326  */
327  res = sendmsg (instance->token_socket, &msg_ucast, MSG_NOSIGNAL);
328  if (res < 0) {
329  LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
330  "sendmsg(ucast) failed (non-critical)");
331  }
332 }
333 
334 static inline void mcast_sendmsg (
335  struct totemudpu_instance *instance,
336  const void *msg,
337  unsigned int msg_len,
338  int only_active)
339 {
340  struct msghdr msg_mcast;
341  int res = 0;
342  size_t buf_out_len;
343  unsigned char buf_out[FRAME_SIZE_MAX];
344  struct iovec iovec;
345  struct sockaddr_storage sockaddr;
346  int addrlen;
347  struct list_head *list;
348  struct totemudpu_member *member;
349 
350  if (msg_len + crypto_get_current_sec_header_size(instance->crypto_inst) > sizeof(buf_out)) {
351  log_printf(LOGSYS_LEVEL_CRIT, "UDPU message for mcast is too big. Ignoring message");
352 
353  return ;
354  }
355 
356  /*
357  * Encrypt and digest the message
358  */
360  instance->crypto_inst,
361  (const unsigned char *)msg,
362  msg_len,
363  buf_out,
364  &buf_out_len) != 0) {
365  log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
366  return;
367  }
368 
369  iovec.iov_base = (void *)buf_out;
370  iovec.iov_len = buf_out_len;
371 
372  memset(&msg_mcast, 0, sizeof(msg_mcast));
373  /*
374  * Build multicast message
375  */
376  for (list = instance->member_list.next;
377  list != &instance->member_list;
378  list = list->next) {
379 
380  member = list_entry (list,
381  struct totemudpu_member,
382  list);
383 
384  /*
385  * Do not send multicast message if message is not "flush", member
386  * is inactive and timeout for sending merge message didn't expired.
387  */
388  if (only_active && !member->active && !instance->send_merge_detect_message)
389  continue ;
390 
392  instance->totem_interface->ip_port, &sockaddr, &addrlen);
393  msg_mcast.msg_name = &sockaddr;
394  msg_mcast.msg_namelen = addrlen;
395  msg_mcast.msg_iov = (void *)&iovec;
396  msg_mcast.msg_iovlen = 1;
397  #ifdef HAVE_MSGHDR_CONTROL
398  msg_mcast.msg_control = 0;
399  #endif
400  #ifdef HAVE_MSGHDR_CONTROLLEN
401  msg_mcast.msg_controllen = 0;
402  #endif
403  #ifdef HAVE_MSGHDR_FLAGS
404  msg_mcast.msg_flags = 0;
405  #endif
406  #ifdef HAVE_MSGHDR_ACCRIGHTS
407  msg_mcast.msg_accrights = NULL;
408  #endif
409  #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
410  msg_mcast.msg_accrightslen = 0;
411  #endif
412 
413  /*
414  * Transmit multicast message
415  * An error here is recovered by totemsrp
416  */
417  res = sendmsg (member->fd, &msg_mcast, MSG_NOSIGNAL);
418  if (res < 0) {
419  LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
420  "sendmsg(mcast) failed (non-critical)");
421  }
422  }
423 
424  if (!only_active || instance->send_merge_detect_message) {
425  /*
426  * Current message was sent to all nodes
427  */
429  instance->send_merge_detect_message = 0;
430  }
431 }
432 
434  void *udpu_context)
435 {
436  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
437  int res = 0;
438 
439  if (instance->token_socket > 0) {
440  qb_loop_poll_del (instance->totemudpu_poll_handle,
441  instance->token_socket);
442  close (instance->token_socket);
443  }
444 
445  totemudpu_stop_merge_detect_timeout(instance);
446 
447  return (res);
448 }
449 
450 static int net_deliver_fn (
451  int fd,
452  int revents,
453  void *data)
454 {
455  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
456  struct msghdr msg_recv;
457  struct iovec *iovec;
458  struct sockaddr_storage system_from;
459  int bytes_received;
460  int res = 0;
461 
462  iovec = &instance->totemudpu_iov_recv;
463 
464  /*
465  * Receive datagram
466  */
467  msg_recv.msg_name = &system_from;
468  msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
469  msg_recv.msg_iov = iovec;
470  msg_recv.msg_iovlen = 1;
471 #ifdef HAVE_MSGHDR_CONTROL
472  msg_recv.msg_control = 0;
473 #endif
474 #ifdef HAVE_MSGHDR_CONTROLLEN
475  msg_recv.msg_controllen = 0;
476 #endif
477 #ifdef HAVE_MSGHDR_FLAGS
478  msg_recv.msg_flags = 0;
479 #endif
480 #ifdef HAVE_MSGHDR_ACCRIGHTS
481  msg_recv.msg_accrights = NULL;
482 #endif
483 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
484  msg_recv.msg_accrightslen = 0;
485 #endif
486 
487  bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
488  if (bytes_received == -1) {
489  return (0);
490  } else {
491  instance->stats_recv += bytes_received;
492  }
493 
494  /*
495  * Authenticate and if authenticated, decrypt datagram
496  */
497 
498  res = crypto_authenticate_and_decrypt (instance->crypto_inst, iovec->iov_base, &bytes_received);
499  if (res == -1) {
500  log_printf (instance->totemudpu_log_level_security, "Received message has invalid digest... ignoring.");
502  "Invalid packet data");
503  iovec->iov_len = FRAME_SIZE_MAX;
504  return 0;
505  }
506  iovec->iov_len = bytes_received;
507 
508  /*
509  * Handle incoming message
510  */
511  instance->totemudpu_deliver_fn (
512  instance->context,
513  iovec->iov_base,
514  iovec->iov_len);
515 
516  iovec->iov_len = FRAME_SIZE_MAX;
517  return (0);
518 }
519 
520 static int netif_determine (
521  struct totemudpu_instance *instance,
522  struct totem_ip_address *bindnet,
523  struct totem_ip_address *bound_to,
524  int *interface_up,
525  int *interface_num)
526 {
527  int res;
528 
529  res = totemip_iface_check (bindnet, bound_to,
530  interface_up, interface_num,
531  instance->totem_config->clear_node_high_bit);
532 
533 
534  return (res);
535 }
536 
537 
538 /*
539  * If the interface is up, the sockets for totem are built. If the interface is down
540  * this function is requeued in the timer list to retry building the sockets later.
541  */
542 static void timer_function_netif_check_timeout (
543  void *data)
544 {
545  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
546  int interface_up;
547  int interface_num;
548  struct totem_ip_address *bind_address;
549 
550  /*
551  * Build sockets for every interface
552  */
553  netif_determine (instance,
554  &instance->totem_interface->bindnet,
555  &instance->totem_interface->boundto,
556  &interface_up, &interface_num);
557  /*
558  * If the network interface isn't back up and we are already
559  * in loopback mode, add timer to check again and return
560  */
561  if ((instance->netif_bind_state == BIND_STATE_LOOPBACK &&
562  interface_up == 0) ||
563 
564  (instance->my_memb_entries == 1 &&
565  instance->netif_bind_state == BIND_STATE_REGULAR &&
566  interface_up == 1)) {
567 
568  qb_loop_timer_add (instance->totemudpu_poll_handle,
569  QB_LOOP_MED,
570  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
571  (void *)instance,
572  timer_function_netif_check_timeout,
573  &instance->timer_netif_check_timeout);
574 
575  /*
576  * Add a timer to check for a downed regular interface
577  */
578  return;
579  }
580 
581  if (instance->token_socket > 0) {
582  qb_loop_poll_del (instance->totemudpu_poll_handle,
583  instance->token_socket);
584  close (instance->token_socket);
585  }
586 
587  if (interface_up == 0) {
588  /*
589  * Interface is not up
590  */
592  bind_address = &localhost;
593 
594  /*
595  * Add a timer to retry building interfaces and request memb_gather_enter
596  */
597  qb_loop_timer_add (instance->totemudpu_poll_handle,
598  QB_LOOP_MED,
599  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
600  (void *)instance,
601  timer_function_netif_check_timeout,
602  &instance->timer_netif_check_timeout);
603  } else {
604  /*
605  * Interface is up
606  */
608  bind_address = &instance->totem_interface->bindnet;
609  }
610  /*
611  * Create and bind the multicast and unicast sockets
612  */
613  totemudpu_build_sockets (instance,
614  bind_address,
615  &instance->totem_interface->boundto);
616 
617  qb_loop_poll_add (instance->totemudpu_poll_handle,
618  QB_LOOP_MED,
619  instance->token_socket,
620  POLLIN, instance, net_deliver_fn);
621 
622  totemip_copy (&instance->my_id, &instance->totem_interface->boundto);
623 
624  /*
625  * This reports changes in the interface to the user and totemsrp
626  */
627  if (instance->netif_bind_state == BIND_STATE_REGULAR) {
628  if (instance->netif_state_report & NETIF_STATE_REPORT_UP) {
630  "The network interface [%s] is now up.",
631  totemip_print (&instance->totem_interface->boundto));
633  instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
634  }
635  /*
636  * Add a timer to check for interface going down in single membership
637  */
638  if (instance->my_memb_entries == 1) {
639  qb_loop_timer_add (instance->totemudpu_poll_handle,
640  QB_LOOP_MED,
641  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
642  (void *)instance,
643  timer_function_netif_check_timeout,
644  &instance->timer_netif_check_timeout);
645  }
646 
647  } else {
650  "The network interface is down.");
651  instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
652  }
654 
655  }
656 }
657 
658 /* Set the socket priority to INTERACTIVE to ensure
659  that our messages don't get queued behind anything else */
660 static void totemudpu_traffic_control_set(struct totemudpu_instance *instance, int sock)
661 {
662 #ifdef SO_PRIORITY
663  int prio = 6; /* TC_PRIO_INTERACTIVE */
664 
665  if (setsockopt(sock, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(int))) {
666  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
667  "Could not set traffic priority");
668  }
669 #endif
670 }
671 
672 static int totemudpu_build_sockets_ip (
673  struct totemudpu_instance *instance,
674  struct totem_ip_address *bindnet_address,
675  struct totem_ip_address *bound_to,
676  int interface_num)
677 {
678  struct sockaddr_storage sockaddr;
679  int addrlen;
680  int res;
681  unsigned int recvbuf_size;
682  unsigned int optlen = sizeof (recvbuf_size);
683 
684  /*
685  * Setup unicast socket
686  */
687  instance->token_socket = socket (bindnet_address->family, SOCK_DGRAM, 0);
688  if (instance->token_socket == -1) {
689  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
690  "socket() failed");
691  return (-1);
692  }
693 
694  totemip_nosigpipe (instance->token_socket);
695  res = fcntl (instance->token_socket, F_SETFL, O_NONBLOCK);
696  if (res == -1) {
697  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
698  "Could not set non-blocking operation on token socket");
699  return (-1);
700  }
701 
702  /*
703  * Bind to unicast socket used for token send/receives
704  * This has the side effect of binding to the correct interface
705  */
706  totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &sockaddr, &addrlen);
707  res = bind (instance->token_socket, (struct sockaddr *)&sockaddr, addrlen);
708  if (res == -1) {
709  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
710  "bind token socket failed");
711  return (-1);
712  }
713 
714  /*
715  * the token_socket can receive many messages. Allow a large number
716  * of receive messages on this socket
717  */
718  recvbuf_size = MCAST_SOCKET_BUFFER_SIZE;
719  res = setsockopt (instance->token_socket, SOL_SOCKET, SO_RCVBUF,
720  &recvbuf_size, optlen);
721  if (res == -1) {
722  LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
723  "Could not set recvbuf size");
724  }
725 
726  return 0;
727 }
728 
729 static int totemudpu_build_sockets (
730  struct totemudpu_instance *instance,
731  struct totem_ip_address *bindnet_address,
732  struct totem_ip_address *bound_to)
733 {
734  int interface_num;
735  int interface_up;
736  int res;
737 
738  /*
739  * Determine the ip address bound to and the interface name
740  */
741  res = netif_determine (instance,
742  bindnet_address,
743  bound_to,
744  &interface_up,
745  &interface_num);
746 
747  if (res == -1) {
748  return (-1);
749  }
750 
751  totemip_copy(&instance->my_id, bound_to);
752 
753  res = totemudpu_build_sockets_ip (instance,
754  bindnet_address, bound_to, interface_num);
755 
756  /* We only send out of the token socket */
757  totemudpu_traffic_control_set(instance, instance->token_socket);
758 
759  /*
760  * Rebind all members to new ips
761  */
763 
764  return res;
765 }
766 
767 /*
768  * Totem Network interface - also does encryption/decryption
769  * depends on poll abstraction, POSIX, IPV4
770  */
771 
772 /*
773  * Create an instance
774  */
776  qb_loop_t *poll_handle,
777  void **udpu_context,
778  struct totem_config *totem_config,
779  totemsrp_stats_t *stats,
780  int interface_no,
781  void *context,
782 
783  void (*deliver_fn) (
784  void *context,
785  const void *msg,
786  unsigned int msg_len),
787 
788  void (*iface_change_fn) (
789  void *context,
790  const struct totem_ip_address *iface_address),
791 
792  void (*target_set_completed) (
793  void *context))
794 {
795  struct totemudpu_instance *instance;
796 
797  instance = malloc (sizeof (struct totemudpu_instance));
798  if (instance == NULL) {
799  return (-1);
800  }
801 
802  totemudpu_instance_initialize (instance);
803 
804  instance->totem_config = totem_config;
805  instance->stats = stats;
806 
807  /*
808  * Configure logging
809  */
810  instance->totemudpu_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
817 
818  /*
819  * Initialize random number generator for later use to generate salt
820  */
821  instance->crypto_inst = crypto_init (totem_config->private_key,
822  totem_config->private_key_len,
823  totem_config->crypto_cipher_type,
824  totem_config->crypto_hash_type,
825  instance->totemudpu_log_printf,
827  instance->totemudpu_log_level_notice,
828  instance->totemudpu_log_level_error,
829  instance->totemudpu_subsys_id);
830  if (instance->crypto_inst == NULL) {
831  free(instance);
832  return (-1);
833  }
834  /*
835  * Initialize local variables for totemudpu
836  */
837  instance->totem_interface = &totem_config->interfaces[interface_no];
838  memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
839 
840  instance->totemudpu_poll_handle = poll_handle;
841 
842  instance->totem_interface->bindnet.nodeid = instance->totem_config->node_id;
843 
844  instance->context = context;
845  instance->totemudpu_deliver_fn = deliver_fn;
846 
847  instance->totemudpu_iface_change_fn = iface_change_fn;
848 
849  instance->totemudpu_target_set_completed = target_set_completed;
850 
851  totemip_localhost (AF_INET, &localhost);
852  localhost.nodeid = instance->totem_config->node_id;
853 
854  /*
855  * RRP layer isn't ready to receive message because it hasn't
856  * initialized yet. Add short timer to check the interfaces.
857  */
858  qb_loop_timer_add (instance->totemudpu_poll_handle,
859  QB_LOOP_MED,
860  100*QB_TIME_NS_IN_MSEC,
861  (void *)instance,
862  timer_function_netif_check_timeout,
863  &instance->timer_netif_check_timeout);
864 
865  totemudpu_start_merge_detect_timeout(instance);
866 
867  *udpu_context = instance;
868  return (0);
869 }
870 
872 {
873  return malloc (FRAME_SIZE_MAX);
874 }
875 
876 void totemudpu_buffer_release (void *ptr)
877 {
878  return free (ptr);
879 }
880 
882  void *udpu_context,
883  int processor_count)
884 {
885  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
886  int res = 0;
887 
888  instance->my_memb_entries = processor_count;
889  qb_loop_timer_del (instance->totemudpu_poll_handle,
890  instance->timer_netif_check_timeout);
891  if (processor_count == 1) {
892  qb_loop_timer_add (instance->totemudpu_poll_handle,
893  QB_LOOP_MED,
894  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
895  (void *)instance,
896  timer_function_netif_check_timeout,
897  &instance->timer_netif_check_timeout);
898  }
899 
900  return (res);
901 }
902 
903 int totemudpu_recv_flush (void *udpu_context)
904 {
905  int res = 0;
906 
907  return (res);
908 }
909 
910 int totemudpu_send_flush (void *udpu_context)
911 {
912  int res = 0;
913 
914  return (res);
915 }
916 
918  void *udpu_context,
919  const void *msg,
920  unsigned int msg_len)
921 {
922  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
923  int res = 0;
924 
925  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
926 
927  return (res);
928 }
930  void *udpu_context,
931  const void *msg,
932  unsigned int msg_len)
933 {
934  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
935  int res = 0;
936 
937  mcast_sendmsg (instance, msg, msg_len, 0);
938 
939  return (res);
940 }
941 
943  void *udpu_context,
944  const void *msg,
945  unsigned int msg_len)
946 {
947  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
948  int res = 0;
949 
950  mcast_sendmsg (instance, msg, msg_len, 1);
951 
952  return (res);
953 }
954 
955 extern int totemudpu_iface_check (void *udpu_context)
956 {
957  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
958  int res = 0;
959 
960  timer_function_netif_check_timeout (instance);
961 
962  return (res);
963 }
964 
965 extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *totem_config)
966 {
967 
968  assert(totem_config->interface_count > 0);
969 
970  totem_config->net_mtu -= crypto_sec_header_size(totem_config->crypto_cipher_type,
971  totem_config->crypto_hash_type) +
973 }
974 
975 const char *totemudpu_iface_print (void *udpu_context) {
976  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
977  const char *ret_char;
978 
979  ret_char = totemip_print (&instance->my_id);
980 
981  return (ret_char);
982 }
983 
985  void *udpu_context,
986  struct totem_ip_address *addr)
987 {
988  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
989  int res = 0;
990 
991  memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
992 
993  return (res);
994 }
995 
997  void *udpu_context,
998  const struct totem_ip_address *token_target)
999 {
1000  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1001  int res = 0;
1002 
1003  memcpy (&instance->token_target, token_target,
1004  sizeof (struct totem_ip_address));
1005 
1006  instance->totemudpu_target_set_completed (instance->context);
1007 
1008  return (res);
1009 }
1010 
1012  void *udpu_context)
1013 {
1014  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1015  unsigned int res;
1016  struct sockaddr_storage system_from;
1017  struct msghdr msg_recv;
1018  struct pollfd ufd;
1019  int nfds;
1020  int msg_processed = 0;
1021 
1022  /*
1023  * Receive datagram
1024  */
1025  msg_recv.msg_name = &system_from;
1026  msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
1027  msg_recv.msg_iov = &instance->totemudpu_iov_recv;
1028  msg_recv.msg_iovlen = 1;
1029 #ifdef HAVE_MSGHDR_CONTROL
1030  msg_recv.msg_control = 0;
1031 #endif
1032 #ifdef HAVE_MSGHDR_CONTROLLEN
1033  msg_recv.msg_controllen = 0;
1034 #endif
1035 #ifdef HAVE_MSGHDR_FLAGS
1036  msg_recv.msg_flags = 0;
1037 #endif
1038 #ifdef HAVE_MSGHDR_ACCRIGHTS
1039  msg_recv.msg_accrights = NULL;
1040 #endif
1041 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1042  msg_recv.msg_accrightslen = 0;
1043 #endif
1044 
1045  do {
1046  ufd.fd = instance->token_socket;
1047  ufd.events = POLLIN;
1048  nfds = poll (&ufd, 1, 0);
1049  if (nfds == 1 && ufd.revents & POLLIN) {
1050  res = recvmsg (instance->token_socket, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
1051  if (res != -1) {
1052  msg_processed = 1;
1053  } else {
1054  msg_processed = -1;
1055  }
1056  }
1057  } while (nfds == 1);
1058 
1059  return (msg_processed);
1060 }
1061 
1062 static int totemudpu_create_sending_socket(
1063  void *udpu_context,
1064  const struct totem_ip_address *member)
1065 {
1066  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1067  int fd;
1068  int res;
1069  unsigned int sendbuf_size;
1070  unsigned int optlen = sizeof (sendbuf_size);
1071  struct sockaddr_storage sockaddr;
1072  int addrlen;
1073 
1074  fd = socket (member->family, SOCK_DGRAM, 0);
1075  if (fd == -1) {
1076  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1077  "Could not create socket for new member");
1078  return (-1);
1079  }
1080  totemip_nosigpipe (fd);
1081  res = fcntl (fd, F_SETFL, O_NONBLOCK);
1082  if (res == -1) {
1083  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1084  "Could not set non-blocking operation on token socket");
1085  goto error_close_fd;
1086  }
1087 
1088  /*
1089  * These sockets are used to send multicast messages, so their buffers
1090  * should be large
1091  */
1092  sendbuf_size = MCAST_SOCKET_BUFFER_SIZE;
1093  res = setsockopt (fd, SOL_SOCKET, SO_SNDBUF,
1094  &sendbuf_size, optlen);
1095  if (res == -1) {
1096  LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
1097  "Could not set sendbuf size");
1098  /*
1099  * Fail in setting sendbuf size is not fatal -> don't exit
1100  */
1101  }
1102 
1103  /*
1104  * Bind to sending interface
1105  */
1106  totemip_totemip_to_sockaddr_convert(&instance->my_id, 0, &sockaddr, &addrlen);
1107  res = bind (fd, (struct sockaddr *)&sockaddr, addrlen);
1108  if (res == -1) {
1109  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1110  "bind token socket failed");
1111  goto error_close_fd;
1112  }
1113 
1114  return (fd);
1115 
1116 error_close_fd:
1117  close(fd);
1118  return (-1);
1119 }
1120 
1122  void *udpu_context,
1123  const struct totem_ip_address *member)
1124 {
1125  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1126 
1127  struct totemudpu_member *new_member;
1128 
1129  new_member = malloc (sizeof (struct totemudpu_member));
1130  if (new_member == NULL) {
1131  return (-1);
1132  }
1133 
1134  memset(new_member, 0, sizeof(*new_member));
1135 
1136  log_printf (LOGSYS_LEVEL_NOTICE, "adding new UDPU member {%s}",
1137  totemip_print(member));
1138  list_init (&new_member->list);
1139  list_add_tail (&new_member->list, &instance->member_list);
1140  memcpy (&new_member->member, member, sizeof (struct totem_ip_address));
1141  new_member->fd = totemudpu_create_sending_socket(udpu_context, member);
1142  new_member->active = 0;
1143 
1144  return (0);
1145 }
1146 
1148  void *udpu_context,
1149  const struct totem_ip_address *token_target)
1150 {
1151  int found = 0;
1152  struct list_head *list;
1153  struct totemudpu_member *member;
1154 
1155  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1156 
1157  /*
1158  * Find the member to remove and close its socket
1159  */
1160  for (list = instance->member_list.next;
1161  list != &instance->member_list;
1162  list = list->next) {
1163 
1164  member = list_entry (list,
1165  struct totemudpu_member,
1166  list);
1167 
1168  if (totemip_compare (token_target, &member->member)==0) {
1170  "removing UDPU member {%s}",
1171  totemip_print(&member->member));
1172 
1173  if (member->fd > 0) {
1175  "Closing socket to: {%s}",
1176  totemip_print(&member->member));
1177  qb_loop_poll_del (instance->totemudpu_poll_handle,
1178  member->fd);
1179  close (member->fd);
1180  }
1181  found = 1;
1182  break;
1183  }
1184  }
1185 
1186  /*
1187  * Delete the member from the list
1188  */
1189  if (found) {
1190  list_del (list);
1191  }
1192 
1193  instance = NULL;
1194  return (0);
1195 }
1196 
1198  void *udpu_context)
1199 {
1200  struct list_head *list;
1201  struct totemudpu_member *member;
1202 
1203  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1204 
1205  for (list = instance->member_list.next;
1206  list != &instance->member_list;
1207  list = list->next) {
1208 
1209  member = list_entry (list,
1210  struct totemudpu_member,
1211  list);
1212 
1213  if (member->fd > 0) {
1214  close (member->fd);
1215  }
1216 
1217  member->fd = totemudpu_create_sending_socket(udpu_context, &member->member);
1218  }
1219 
1220  return (0);
1221 }
1222 
1224  void *udpu_context,
1225  const struct totem_ip_address *member_ip,
1226  int active)
1227 {
1228  struct list_head *list;
1229  struct totemudpu_member *member;
1230  int addr_found = 0;
1231 
1232  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1233 
1234  /*
1235  * Find the member to set active flag
1236  */
1237  for (list = instance->member_list.next; list != &instance->member_list; list = list->next) {
1238  member = list_entry (list, struct totemudpu_member, list);
1239 
1240  if (totemip_compare (member_ip, &member->member) == 0) {
1242  "Marking UDPU member %s %s",
1243  totemip_print(&member->member),
1244  (active ? "active" : "inactive"));
1245 
1246  member->active = active;
1247  addr_found = 1;
1248 
1249  break;
1250  }
1251  }
1252 
1253  if (!addr_found) {
1255  "Can't find UDPU member %s (should be marked as %s)",
1256  totemip_print(member_ip),
1257  (active ? "active" : "inactive"));
1258  }
1259 
1260  return (0);
1261 }
1262 
1263 static void timer_function_merge_detect_timeout (
1264  void *data)
1265 {
1266  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
1267 
1268  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1269  instance->send_merge_detect_message = 1;
1270  }
1271 
1273 
1274  totemudpu_start_merge_detect_timeout(instance);
1275 }
1276 
1277 static void totemudpu_start_merge_detect_timeout(
1278  void *udpu_context)
1279 {
1280  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1281 
1282  qb_loop_timer_add(instance->totemudpu_poll_handle,
1283  QB_LOOP_MED,
1284  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1285  (void *)instance,
1286  timer_function_merge_detect_timeout,
1287  &instance->timer_merge_detect_timeout);
1288 
1289 }
1290 
1291 static void totemudpu_stop_merge_detect_timeout(
1292  void *udpu_context)
1293 {
1294  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1295 
1296  qb_loop_timer_del(instance->totemudpu_poll_handle,
1297  instance->timer_merge_detect_timeout);
1298 }
unsigned int clear_node_high_bit
Definition: totem.h:117
unsigned short family
Definition: coroapi.h:97
#define NETIF_STATE_REPORT_UP
Definition: totemudpu.c:84
struct totem_config * totem_config
Definition: totemudpu.c:174
struct totem_ip_address member
Definition: totemudpu.c:93
int totemip_localhost(int family, struct totem_ip_address *localhost)
Definition: totemip.c:182
unsigned int my_memb_entries
Definition: totemudpu.c:172
void(* totemudpu_iface_change_fn)(void *context, const struct totem_ip_address *iface_address)
Definition: totemudpu.c:116
struct totem_interface * interfaces
Definition: totem.h:114
unsigned int interface_count
Definition: totem.h:115
struct list_head * next
Definition: list.h:47
size_t crypto_sec_header_size(const char *crypto_cipher_type, const char *crypto_hash_type)
Definition: totemcrypto.c:667
void(*) void udpu_context)
Definition: totemudpu.c:144
int totemudpu_token_target_set(void *udpu_context, const struct totem_ip_address *token_target)
Definition: totemudpu.c:996
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:214
struct totem_ip_address my_id
Definition: totemudpu.c:166
size_t crypto_get_current_sec_header_size(const struct crypto_instance *instance)
Definition: totemcrypto.c:702
struct totemudpu_instance * instance
Definition: totemudpu.c:192
#define NETIF_STATE_REPORT_DOWN
Definition: totemudpu.c:85
int totemip_compare(const void *a, const void *b)
Definition: totemip.c:130
#define log_printf(level, format, args...)
Definition: totemudpu.c:233
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN]
Definition: totem.h:122
int totemudpu_processor_count_set(void *udpu_context, int processor_count)
Definition: totemudpu.c:881
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:67
int totemudpu_log_level_security
Definition: totemudpu.c:125
struct crypto_instance * crypto_init(const unsigned char *private_key, unsigned int private_key_len, const char *crypto_cipher_type, const char *crypto_hash_type, void(*log_printf_func)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf, 6, 7))), int log_level_security, int log_level_notice, int log_level_error, int log_subsys_id)
Definition: totemcrypto.c:803
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:95
unsigned int downcheck_timeout
Definition: totem.h:145
unsigned int private_key_len
Definition: totem.h:124
char iov_buffer[FRAME_SIZE_MAX]
Definition: totemudpu.c:148
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemudpu.c:182
Definition: list.h:46
int send_merge_detect_message
Definition: totemudpu.c:184
#define totemip_nosigpipe(s)
Definition: totemip.h:56
int totemudpu_log_level_warning
Definition: totemudpu.c:129
int totemudpu_log_level_debug
Definition: totemudpu.c:133
const char * totemudpu_iface_print(void *udpu_context)
Definition: totemudpu.c:975
struct iovec totemudpu_iov_recv
Definition: totemudpu.c:150
unsigned int node_id
Definition: totem.h:116
#define BIND_STATE_REGULAR
Definition: totemudpu.c:88
int totemip_iface_check(struct totem_ip_address *bindnet, struct totem_ip_address *boundto, int *interface_up, int *interface_num, int mask_high_bit)
Definition: totemip.c:405
int crypto_encrypt_and_sign(struct crypto_instance *instance, const unsigned char *buf_in, const size_t buf_in_len, unsigned char *buf_out, size_t *buf_out_len)
Definition: totemcrypto.c:723
void totemudpu_buffer_release(void *ptr)
Definition: totemudpu.c:876
void * totemudpu_buffer_alloc(void)
Definition: totemudpu.c:871
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemudpu.c:186
qb_loop_t * totemudpu_poll_handle
Definition: totemudpu.c:101
int totemudpu_mcast_noflush_send(void *udpu_context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:942
void(* totemudpu_deliver_fn)(void *context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:111
unsigned int nodeid
Definition: coroapi.h:96
char * crypto_hash_type
Definition: totem.h:183
totemsrp_stats_t * stats
Definition: totemudpu.c:176
Linked list API.
struct totem_ip_address token_target
Definition: totemudpu.c:178
int totemudpu_crypto_set(void *udpu_context, const char *cipher_type, const char *hash_type)
Definition: totemudpu.c:250
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
struct totem_interface * totem_interface
Definition: totemudpu.c:103
int totemudpu_token_send(void *udpu_context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:917
struct totem_ip_address boundto
Definition: totem.h:66
typedef __attribute__
size_t totemip_udpip_header_size(int family)
Definition: totemip.c:496
struct list_head member_list
Definition: totemudpu.c:152
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:75
uint16_t ip_port
Definition: totem.h:68
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemudpu.c:170
void(* totemudpu_target_set_completed)(void *context)
Definition: totemudpu.c:120
#define BIND_STATE_LOOPBACK
Definition: totemudpu.c:89
unsigned int net_mtu
Definition: totem.h:165
#define MCAST_SOCKET_BUFFER_SIZE
Definition: totemudpu.c:83
int crypto_authenticate_and_decrypt(struct crypto_instance *instance, unsigned char *buf, int *buf_len)
Definition: totemcrypto.c:745
int totemudpu_member_remove(void *udpu_context, const struct totem_ip_address *token_target)
Definition: totemudpu.c:1147
int totemudpu_member_set_active(void *udpu_context, const struct totem_ip_address *member_ip, int active)
Definition: totemudpu.c:1223
#define FRAME_SIZE_MAX
Definition: totem.h:50
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:69
#define list_entry(ptr, type, member)
Definition: list.h:84
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:222
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:163
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
int totemudpu_recv_mcast_empty(void *udpu_context)
Definition: totemudpu.c:1011
int totemudpu_initialize(qb_loop_t *poll_handle, void **udpu_context, struct totem_config *totem_config, totemsrp_stats_t *stats, int interface_no, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address), void(*target_set_completed)(void *context))
Create an instance.
Definition: totemudpu.c:775
struct srp_addr system_from
Definition: totemsrp.c:61
char * crypto_cipher_type
Definition: totem.h:181
int totemudpu_log_level_error
Definition: totemudpu.c:127
unsigned int merge_timeout
Definition: totem.h:143
void totemudpu_net_mtu_adjust(void *udpu_context, struct totem_config *totem_config)
Definition: totemudpu.c:965
struct totem_ip_address bindnet
Definition: totem.h:65
#define MSG_NOSIGNAL
Definition: totemudpu.c:80
int totemudpu_mcast_flush_send(void *udpu_context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:929
int totemudpu_send_flush(void *udpu_context)
Definition: totemudpu.c:910
int totemudpu_finalize(void *udpu_context)
Definition: totemudpu.c:433
struct crypto_instance * crypto_inst
Definition: totemudpu.c:99
int totemudpu_recv_flush(void *udpu_context)
Definition: totemudpu.c:903
int totemudpu_member_add(void *udpu_context, const struct totem_ip_address *member)
Definition: totemudpu.c:1121
struct list_head list
Definition: totemudpu.c:92
#define LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemudpu.c:240
int totemudpu_iface_get(void *udpu_context, struct totem_ip_address *addr)
Definition: totemudpu.c:984
int totemudpu_member_list_rebind_ip(void *udpu_context)
Definition: totemudpu.c:1197
void(* totemudpu_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemudpu.c:137
int totemudpu_iface_check(void *udpu_context)
Definition: totemudpu.c:955
int totemudpu_log_level_notice
Definition: totemudpu.c:131