Explicitly storing and checking loop information in sockets

lib/socket.h

@@ -40,6 +40,7 @@ struct ssh_sock {
 typedef struct birdsock {
   resource r;
   pool *pool;				/* Pool where incoming connections should be allocated (for SK_xxx_PASSIVE) */
+  struct birdloop *loop;		/* The birdloop where this socket belongs to */
   int type;				/* Socket type */
   int subtype;				/* Socket subtype */
   void *data;				/* User data */

proto/bfd/bfd.c

@@ -1025,6 +1025,7 @@ bfd_notify_init(struct bfd_proto *p)
   sk->fd = pfds[1];
   sk->data = p;
   sk->flags = SKF_THREAD;
+  sk->loop = p->p.loop;
   if (sk_open(sk) < 0)
     die("bfd: sk_open failed");
   p->notify_ws = sk;

proto/bfd/packets.c

@@ -425,6 +425,7 @@ bfd_open_rx_sk(struct bfd_proto *p, int multihop, int af)
   sk->tos = IP_PREC_INTERNET_CONTROL;
   sk->priority = sk_priority_control;
   sk->flags = SKF_THREAD | SKF_LADDR_RX | (!multihop ? SKF_TTL_RX : 0);
+  sk->loop = p->p.loop;
 
   if (sk_open(sk) < 0)
     goto err;
@@ -457,6 +458,7 @@ bfd_open_tx_sk(struct bfd_proto *p, ip_addr local, struct iface *ifa)
   sk->priority = sk_priority_control;
   sk->ttl = ifa ? 255 : -1;
   sk->flags = SKF_THREAD | SKF_BIND | SKF_HIGH_PORT;
+  sk->loop = p->p.loop;
 
   if (sk_open(sk) < 0)
     goto err;

proto/rpki/transport.c

@@ -86,6 +86,7 @@ rpki_tr_open(struct rpki_tr_sock *tr)
   sk->tbsize = RPKI_TX_BUFFER_SIZE;
   sk->tos = IP_PREC_INTERNET_CONTROL;
   sk->flags |= SKF_THREAD;
+  sk->loop = cache->p->p.loop;
 
   if (ipa_zero(sk->daddr) && sk->host)
   {

sysdep/unix/io-loop.c

@@ -170,9 +170,12 @@ sockets_init(struct birdloop *loop)
 static void
 sockets_add(struct birdloop *loop, sock *s)
 {
+  ASSERT_DIE(!enlisted(&s->n));
+
   add_tail(&loop->sock_list, &s->n);
   loop->sock_num++;
 
+  s->loop = loop;
   s->index = -1;
   loop->poll_changed = 1;
 
@@ -189,6 +192,11 @@ sk_start(sock *s)
 static void
 sockets_remove(struct birdloop *loop, sock *s)
 {
+  ASSERT_DIE(s->loop == loop);
+
+  if (!enlisted(&s->n))
+    return;
+
   rem_node(&s->n);
   loop->sock_num--;
 
@@ -197,11 +205,10 @@ sockets_remove(struct birdloop *loop, sock *s)
     loop->poll_sk.data[s->index] = NULL;
     s->index = -1;
     loop->poll_changed = 1;
-    loop->close_scheduled = 1;
     birdloop_ping(loop);
   }
-  else
-    close(s->fd);
+
+  s->loop = NULL;
 }
 
 void
@@ -263,21 +270,6 @@ sockets_prepare(struct birdloop *loop)
   loop->poll_changed = 0;
 }
 
-static void
-sockets_close_fds(struct birdloop *loop)
-{
-  struct pollfd *pfd = loop->poll_fd.data;
-  sock **psk = loop->poll_sk.data;
-  int poll_num = loop->poll_fd.used - 1;
-
-  int i;
-  for (i = 0; i < poll_num; i++)
-    if (psk[i] == NULL)
-      close(pfd[i].fd);
-
-  loop->close_scheduled = 0;
-}
-
 int sk_read(sock *s, int revents);
 int sk_write(sock *s);
 
@@ -297,13 +289,16 @@ sockets_fire(struct birdloop *loop)
   int i;
   for (i = 0; i < poll_num; pfd++, psk++, i++)
   {
-    int e = 1;
+    if (!*psk)
+      continue;
 
     if (! pfd->revents)
       continue;
 
     if (pfd->revents & POLLNVAL)
-      die("poll: invalid fd %d", pfd->fd);
+      bug("poll: invalid fd %d", pfd->fd);
+
+    int e = 1;
 
     if (pfd->revents & POLLIN)
       while (e && *psk && (*psk)->rx_hook)
@@ -546,9 +541,6 @@ birdloop_main(void *arg)
 
     birdloop_enter(loop);
 
-    if (loop->close_scheduled)
-      sockets_close_fds(loop);
-
     if (loop->stopped)
       break;
 

sysdep/unix/io-loop.h

@@ -36,7 +36,6 @@ struct birdloop
   BUFFER(sock *) poll_sk;
   BUFFER(struct pollfd) poll_fd;
   u8 poll_changed;
-  u8 close_scheduled;
 
   _Atomic u32 ping_sent;
   int wakeup_fds[2];

sysdep/unix/io.c

@@ -794,6 +794,7 @@ sk_free(resource *r)
 {
   sock *s = (sock *) r;
 
+  ASSERT_DIE(!s->loop || birdloop_inside(s->loop));
   sk_free_bufs(s);
 
 #ifdef HAVE_LIBSSH
@@ -804,13 +805,20 @@ sk_free(resource *r)
   if ((s->fd < 0) || (s->flags & SKF_THREAD))
     return;
 
-  if (s == current_sock)
-    current_sock = sk_next(s);
-  if (s == stored_sock)
-    stored_sock = sk_next(s);
+  if (!s->loop)
+    ;
+  else if (s->flags & SKF_THREAD)
+    sk_stop(s);
+  else
+  {
+    if (s == current_sock)
+      current_sock = sk_next(s);
+    if (s == stored_sock)
+      stored_sock = sk_next(s);
 
-  if (enlisted(&s->n))
-    rem_node(&s->n);
+    if (enlisted(&s->n))
+      rem_node(&s->n);
+  }
 
   if (s->type != SK_SSH && s->type != SK_SSH_ACTIVE)
     close(s->fd);
@@ -1106,7 +1114,11 @@ sk_passive_connected(sock *s, int type)
   if (s->flags & SKF_PASSIVE_THREAD)
     t->flags |= SKF_THREAD;
   else
+  {
+    ASSERT_DIE(s->loop == &main_birdloop);
+    t->loop = &main_birdloop;
     sk_insert(t);
+  }
 
   sk_alloc_bufs(t);
   s->rx_hook(t, 0);
@@ -1328,6 +1340,17 @@ sk_open(sock *s)
   ip_addr bind_addr = IPA_NONE;
   sockaddr sa;
 
+  if (s->flags & SKF_THREAD)
+  {
+    ASSERT_DIE(s->loop && (s->loop != &main_birdloop));
+    ASSERT_DIE(birdloop_inside(s->loop));
+  }
+  else
+  {
+    ASSERT_DIE(!s->loop);
+    s->loop = &main_birdloop;
+  }
+
   if (s->type <= SK_IP)
   {
     /*
@@ -1507,6 +1530,7 @@ sk_open_unix(sock *s, char *name)
     return -1;
 
   s->fd = fd;
+  s->loop = &main_birdloop;
   sk_insert(s);
   return 0;
 }