hash.h: new macro HASH_WALK_RESIZABLE: works as HASH_WALK_DELSAFE, but if resize needed, it calls it in HASH_WALK_RESIZABLE_END (that is, at the end of itself)

lib/birdlib.h

@@ -50,7 +50,7 @@ extern const enum build_target {
   BT_TEST,
 } build_target;
 
-jmp_buf *get_test_bug_jump(char *msg);
+jmp_buf *get_test_bug_jump(const char *msg);
 
 static inline int uint_cmp(uint i1, uint i2)
 { return (int)(i1 > i2) - (int)(i1 < i2); }

lib/hash.h

@@ -10,7 +10,16 @@
 #ifndef _BIRD_HASH_H_
 #define _BIRD_HASH_H_
 
-#define HASH(type)		struct { type **data; uint count, order; char* is_in_walk; }
+enum hash_walk_state {
+  NO_WALK,
+  WALK,
+  WALK_DELSAFE,
+  WALK_RESIZABLE,
+  NEED_RESIZE
+};
+
+
+#define HASH(type)		struct { type **data; uint count, order; char* is_in_walk; int* deep_of_walks;}
 #define HASH_TYPE(v)		typeof(** (v).data)
 #define HASH_SIZE(v)		(1U << (v).order)
 
@@ -24,7 +33,9 @@
     (v).order = (init_order);						\
     (v).data = mb_allocz(pool, HASH_SIZE(v) * sizeof(* (v).data));	\
     (v).is_in_walk = mb_allocz(pool, sizeof(char));			\
-    *(v).is_in_walk = 0;						\
+    *(v).is_in_walk = NO_WALK;						\
+    (v).deep_of_walks = mb_allocz(pool, sizeof(char));			\
+    *(v).deep_of_walks = 0;						\
   })
 
 #define HASH_FREE(v)							\
@@ -44,8 +55,6 @@
 
 #define HASH_INSERT(v,id,node)						\
   ({									\
-    if (*(v).is_in_walk)						\
-      bug("HASH_INSERT: Attempt to insert in HASH_WALK");		\
     u32 _h = HASH_FN(v, id, id##_KEY((node)));				\
     HASH_TYPE(v) **_nn = (v).data + _h;					\
     id##_NEXT(node) = *_nn;						\
@@ -55,14 +64,16 @@
 
 #define HASH_DO_REMOVE(v,id,_nn)					\
   ({									\
-    if (*(v).is_in_walk)						\
-      bug("HASH_DELETE: Attempt to remove in HASH_WALK");		\
     *_nn = id##_NEXT((*_nn));						\
     (v).count--;							\
   })
 
 #define HASH_DELETE(v,id,key...)					\
   ({									\
+    if (*(v).is_in_walk == WALK)					\
+      bug("HASH_DELETE: Attempt to delete in HASH_WALK");		\
+    if (*(v).deep_of_walks > 1)						\
+      bug("HASH_DELETE: Attempt to delete inside multiple hash walks");	\
     u32 _h = HASH_FN(v, id, key);					\
     HASH_TYPE(v) *_n, **_nn = (v).data + _h;				\
 									\
@@ -76,6 +87,10 @@
 
 #define HASH_REMOVE(v,id,node)						\
   ({									\
+    if (*(v).is_in_walk == WALK)					\
+      bug("HASH_REMOVE: Attempt to remove in HASH_WALK");		\
+    if (*(v).deep_of_walks > 1)						\
+      bug("HASH_REMOVE: Attempt to remove inside multiple hash walks");	\
     u32 _h = HASH_FN(v, id, id##_KEY((node)));				\
     HASH_TYPE(v) *_n, **_nn = (v).data + _h;				\
 									\
@@ -124,46 +139,61 @@
 
 #define HASH_MAY_STEP_UP_(v,pool,rehash_fn,args)			\
   ({                                                                    \
-    if (((v).count > (HASH_SIZE(v) REHASH_HI_MARK(args))) &&		\
+    if (((v).count > (HASH_SIZE(v) REHASH_HI_MARK(args))) &&	\
-	((v).order < (REHASH_HI_BOUND(args))))				\
+	((v).order <= (REHASH_HI_BOUND(args) - REHASH_HI_STEP(args)))) \
       rehash_fn(&(v), pool, REHASH_HI_STEP(args));			\
   })
 
 #define HASH_MAY_STEP_DOWN_(v,pool,rehash_fn,args)			\
   ({                                                                    \
-    if (((v).count < (HASH_SIZE(v) REHASH_LO_MARK(args))) &&		\
+    if (((v).count < (HASH_SIZE(v) REHASH_LO_MARK(args))) &&	\
-	((v).order > (REHASH_LO_BOUND(args))))				\
+	((v).order >= (REHASH_LO_BOUND(args) + REHASH_LO_STEP(args)))) \
       rehash_fn(&(v), pool, -(REHASH_LO_STEP(args)));			\
   })
 
 #define HASH_MAY_RESIZE_DOWN_(v,pool,rehash_fn,args)			\
-  ({                                                                    \
+  ({  									\
-    uint _o = (v).order;							\
+    uint _o = (v).order;						\
     while (((v).count < ((1U << _o) REHASH_LO_MARK(args))) &&		\
 	   (_o > (REHASH_LO_BOUND(args))))				\
       _o -= (REHASH_LO_STEP(args));					\
-    if (_o < (v).order)							\
+    if (_o < (v).order)						\
       rehash_fn(&(v), pool, _o - (v).order);				\
   })
 
 
 #define HASH_INSERT2(v,id,pool,node)					\
   ({									\
+    if (*(v).is_in_walk == WALK || *(v).is_in_walk == WALK_DELSAFE)	\
+      bug("HASH_INSERT2: called in hash walk or hash delsafe walk");	\
     HASH_INSERT(v, id, node);						\
-    HASH_MAY_STEP_UP(v, id, pool);					\
+    if (*(v).is_in_walk == NO_WALK)					\
+      HASH_MAY_STEP_UP(v, id, pool);					\
+    else if (*(v).is_in_walk == WALK_RESIZABLE)				\
+      *(v).is_in_walk = NEED_RESIZE;					\
   })
 
 #define HASH_DELETE2(v,id,pool,key...)					\
   ({									\
+    if (*(v).is_in_walk == WALK || *(v).is_in_walk == WALK_DELSAFE)	\
+      bug("HASH_DELETE2 called in hash walk or hash delsafe walk");	\
     HASH_TYPE(v) *_n = HASH_DELETE(v, id, key);				\
-    if (_n) HASH_MAY_STEP_DOWN(v, id, pool);				\
+    if (*(v).is_in_walk == WALK_RESIZABLE)				\
+      *(v).is_in_walk = NEED_RESIZE;					\
+    else if (*(v).is_in_walk == NO_WALK)				\
+      if (_n) HASH_MAY_STEP_DOWN(v, id, pool);				\
     _n;									\
   })
 
 #define HASH_REMOVE2(v,id,pool,node)					\
   ({									\
+    if (*(v).is_in_walk == WALK || *(v).is_in_walk == WALK_DELSAFE)	\
+      bug("HASH_REMOVE2 called in hash walk or hash delsafe walk");	\
     HASH_TYPE(v) *_n = HASH_REMOVE(v, id, node);			\
-    if (_n) HASH_MAY_STEP_DOWN(v, id, pool);				\
+    if (*(v).is_in_walk == WALK_RESIZABLE)				\
+      *(v).is_in_walk = NEED_RESIZE;					\
+    else if (*(v).is_in_walk == NO_WALK)				\
+      if (_n) HASH_MAY_STEP_DOWN(v, id, pool);				\
     _n;									\
   })
 
@@ -171,26 +201,81 @@
 #define HASH_WALK(v,next,n)						\
   do {									\
     HASH_TYPE(v) *n;							\
-    *(v).is_in_walk = 1;						\
+    if (*(v).is_in_walk != WALK && *(v).is_in_walk != NO_WALK)		\
+      bug("HASH_WALK can not be called from other walks");		\
+    *(v).is_in_walk = WALK;						\
+    *(v).deep_of_walks += 1;						\
     uint _i;								\
     uint _s = HASH_SIZE(v);						\
     for (_i = 0; _i < _s; _i++)						\
       for (n = (v).data[_i]; n; n = n->next)
 
-#define HASH_WALK_END(v) *(v).is_in_walk = 0; } while (0)
+#define HASH_WALK_END(v)						\
+  if (*(v).is_in_walk != WALK)						\
+    bug("HASH_WALK_END called when HASH_WALK is not opened");		\
+  *(v).deep_of_walks -= 1;						\
+  if (*(v).deep_of_walks == 0)						\
+    *(v).is_in_walk = NO_WALK;						\
+  } while (0)								\
 
 
 #define HASH_WALK_DELSAFE(v,next,n)					\
   do {									\
     HASH_TYPE(v) *n, *_next;						\
+    if (*(v).is_in_walk != NO_WALK && *(v).is_in_walk != WALK_DELSAFE)	\
+      bug("HASH_WALK_DELSAFE can not be called from other walks");	\
+    *(v).is_in_walk = WALK_DELSAFE;					\
+    *(v).deep_of_walks += 1;						\
     uint _i;								\
     uint _s = HASH_SIZE(v);						\
     for (_i = 0; _i < _s; _i++)						\
       for (n = (v).data[_i]; n && (_next = n->next, 1); n = _next)
 
-#define HASH_WALK_DELSAFE_END } while (0)
+#define HASH_WALK_DELSAFE_END(v) 					\
+  if (*(v).is_in_walk != WALK_DELSAFE)					\
+    bug("HASH_WALK_DELSAFE_END called when HASH_WALK_DELSAFE is not opened"); \
+  *(v).deep_of_walks -= 1;						\
+  if (*(v).deep_of_walks == 0)						\
+    *(v).is_in_walk = NO_WALK;						\
+  } while (0)
 
 
+#define HASH_WALK_RESIZABLE(v,next,n)					\
+  do {									\
+    HASH_TYPE(v) *n, *_next;						\
+    if (*(v).is_in_walk == NO_WALK)					\
+      *(v).is_in_walk = WALK_RESIZABLE;					\
+    else if (*(v).is_in_walk != WALK_RESIZABLE && *(v).is_in_walk != NEED_RESIZE) \
+      bug("HASH_WALK_RESIZABLE can not be called from other walks");	\
+    *(v).deep_of_walks += 1;						\
+    uint _i;								\
+    uint _s = HASH_SIZE(v);						\
+    for (_i = 0; _i < _s; _i++)						\
+      for (n = (v).data[_i]; n && (_next = n->next, 1); n = _next)
+
+#define HASH_WALK_RESIZABLE_END(v, id, pool) 				\
+  if (*(v).is_in_walk != WALK_RESIZABLE && *(v).is_in_walk != NEED_RESIZE) \
+    bug("HASH_WALK_RESIZABLE_END called when HASH_WALK_RESIZABLE is not opened"); \
+  *(v).deep_of_walks -= 1;						\
+  if (*(v).deep_of_walks == 0)						\
+  {									\
+    if (*(v).is_in_walk == NEED_RESIZE)					\
+    {									\
+      *(v).is_in_walk = NO_WALK;					\
+      uint order;							\
+      do {								\
+        order = (v).order;						\
+        HASH_MAY_STEP_DOWN(v, id, pool);				\
+      } while (order!=(v).order);					\
+      do {								\
+        order = (v).order;						\
+        HASH_MAY_STEP_UP(v, id, pool);					\
+      } while (order!=(v).order);					\
+    }									\
+    *(v).is_in_walk = NO_WALK;						\
+  }									\
+  } while (0)
+
 #define HASH_WALK_FILTER(v,next,n,nn)					\
   do {									\
     HASH_TYPE(v) *n, **nn;						\

lib/hash_test.c

@@ -22,7 +22,7 @@ struct test_node {
 #define TEST_EQ(n1,n2)		n1 == n2
 #define TEST_FN(n)		(n) ^ u32_hash((n))
 #define TEST_ORDER		13
-#define TEST_PARAMS		/TEST_ORDER, *2, 2, 2, TEST_ORDER, 20
+#define TEST_PARAMS		/TEST_ORDER, *2, 2, 2, 8, 20
 #define TEST_REHASH		test_rehash
 
 HASH_DEFINE_REHASH_FN(TEST, struct test_node);
@@ -203,7 +203,7 @@ t_walk_delsafe_delete(void)
   {
     HASH_DELETE(hash, TEST, n->key);
   }
-  HASH_WALK_DELSAFE_END;
+  HASH_WALK_DELSAFE_END(hash);
 
   validate_empty_hash();
 
@@ -220,7 +220,7 @@ t_walk_delsafe_remove(void)
   {
     HASH_REMOVE(hash, TEST, n);
   }
-  HASH_WALK_DELSAFE_END;
+  HASH_WALK_DELSAFE_END(hash);
 
   validate_empty_hash();
 
@@ -228,16 +228,35 @@ t_walk_delsafe_remove(void)
 }
 
 static int
-t_walk_delsafe_delete2(void)
+t_walk_resizable_delete2(void)
 {
   init_hash();
   fill_hash();
 
-  HASH_WALK_DELSAFE(hash, next, n)
+  HASH_WALK_RESIZABLE(hash, next, n)
   {
     HASH_DELETE2(hash, TEST, my_pool, n->key);
   }
-  HASH_WALK_DELSAFE_END;
+  HASH_WALK_RESIZABLE_END(hash, TEST, my_pool);
+
+  validate_empty_hash();
+  return 1;
+}
+
+static int
+t_walk_resizable_remove2(void)
+{
+  init_hash();
+  fill_hash();
+  bt_assert(hash.order == 13);
+
+  HASH_WALK_RESIZABLE(hash, next, n)
+  {
+    HASH_REMOVE2(hash, TEST, my_pool, n);
+  }
+  HASH_WALK_RESIZABLE_END(hash, TEST, my_pool);
+
+  bt_assert(hash.order == 9);
 
   validate_empty_hash();
 
@@ -245,22 +264,28 @@ t_walk_delsafe_delete2(void)
 }
 
 static int
-t_walk_delsafe_remove2(void)
+t_walk_multilevel(void)
 {
   init_hash();
   fill_hash();
 
+  int check = 0;
+
   HASH_WALK_DELSAFE(hash, next, n)
   {
-    HASH_REMOVE2(hash, TEST, my_pool, n);
+    HASH_WALK_DELSAFE(hash, next, n)
+    {
+      check++;
+    }
+    HASH_WALK_DELSAFE_END(hash);
   }
-  HASH_WALK_DELSAFE_END;
+  HASH_WALK_DELSAFE_END(hash);
-
-  validate_empty_hash();
 
+  bt_assert(check == MAX_NUM * MAX_NUM);
   return 1;
 }
 
+
 static int
 t_walk_filter(void)
 {
@@ -288,7 +313,7 @@ t_walk_filter(void)
 void
 do_walk_delete_error(void)
 {
-    init_hash();
+  init_hash();
   fill_hash();
 
   HASH_WALK(hash, next, n)
@@ -298,12 +323,201 @@ do_walk_delete_error(void)
   HASH_WALK_END(hash);
 }
 
-static int
+void
-t_walk_check_bug(void)
+do_walk_remove_error(void)
 {
-  return bt_assert_bug(do_walk_delete_error, "HASH_DELETE: Attempt to remove in HASH_WALK");
+  init_hash();
+  fill_hash();
+
+  HASH_WALK(hash, next, n)
+  {
+    HASH_REMOVE(hash, TEST, n);
+  }
+  HASH_WALK_END(hash);
 }
 
+void
+do_bad_end_error(void)
+{
+init_hash();
+  fill_hash();
+
+  int i = 0;
+  HASH_WALK(hash, next, n)
+  {
+    i++;
+  }
+  HASH_WALK_DELSAFE_END(hash);
+}
+
+void
+delete_from_multiple_walks_bug(void)
+{
+  init_hash();
+  fill_hash();
+
+  HASH_WALK_DELSAFE(hash, next, n)
+  {
+    HASH_WALK_DELSAFE(hash, next, n)
+    {
+      HASH_DELETE(hash, TEST, n->key);
+    }
+    HASH_WALK_DELSAFE_END(hash);
+  }
+  HASH_WALK_DELSAFE_END(hash);
+}
+
+void
+remove_from_multiple_walks_bug(void)
+{
+  init_hash();
+  fill_hash();
+
+  HASH_WALK_DELSAFE(hash, next, n)
+  {
+    HASH_WALK_DELSAFE(hash, next, n)
+    {
+      HASH_REMOVE(hash, TEST, n);
+    }
+    HASH_WALK_DELSAFE_END(hash);
+  }
+  HASH_WALK_DELSAFE_END(hash);
+}
+
+void
+delsafe_insert2_bug(void)
+{
+  init_hash();
+  fill_hash();
+
+  HASH_WALK_DELSAFE(hash, next, n)
+  {
+    struct test_node *node; // The test should crash soon enough not to recognise uninitialized pointer
+    HASH_INSERT2(hash, TEST, my_pool, node);
+  }
+  HASH_WALK_DELSAFE_END(hash);
+}
+
+void
+walk_delete2_bug(void)
+{
+  init_hash();
+  fill_hash();
+
+  HASH_WALK_DELSAFE(hash, next, n)
+  {
+    HASH_DELETE2(hash, TEST, my_pool, n->key);
+  }
+  HASH_WALK_DELSAFE_END(hash);
+}
+
+void
+delsafe_different_walks_bug(void)
+{
+  init_hash();
+  fill_hash();
+
+  int i = 0;
+  HASH_WALK(hash, next, n)
+  {
+    HASH_WALK_DELSAFE(hash, next, n)
+    {
+      i++;
+    }
+    HASH_WALK_DELSAFE_END(hash);
+  }
+  HASH_WALK_END(hash);
+}
+
+void
+walk_different_walks_bug(void)
+{
+  init_hash();
+  fill_hash();
+
+  int i = 0;
+  HASH_WALK_RESIZABLE(hash, next, n)
+  {
+    HASH_WALK(hash, next, n)
+    {
+      i++;
+    }
+    HASH_WALK_END(hash);
+  }
+  HASH_WALK_RESIZABLE_END(hash, TEST, my_pool);
+}
+
+void
+resizable_different_walks_bug(void)
+{
+  init_hash();
+  fill_hash();
+
+  int i = 0;
+  HASH_WALK(hash, next, n)
+  {
+    HASH_WALK_RESIZABLE(hash, next, n)
+    {
+      i++;
+    }
+    HASH_WALK_RESIZABLE_END(hash, TEST, my_pool);
+  }
+  HASH_WALK_END(hash);
+}
+
+static int
+t_walk_check_delete_bug(void)
+{
+  return bt_assert_bug(do_walk_delete_error, "HASH_DELETE: Attempt to delete in HASH_WALK");
+}
+
+static int
+t_walk_check_remove_bug(void)
+{
+  return bt_assert_bug(do_walk_remove_error, "HASH_REMOVE: Attempt to remove in HASH_WALK");
+}
+
+static int
+t_walk_check_end_bug(void)
+{
+  return bt_assert_bug(do_bad_end_error, "HASH_WALK_DELSAFE_END called when HASH_WALK_DELSAFE is not opened");
+}
+
+static int
+t_delete_from_multiple_walks_bug(void)
+{
+  return bt_assert_bug(delete_from_multiple_walks_bug, "HASH_DELETE: Attempt to delete inside multiple hash walks");
+}
+
+static int
+t_remove_from_multiple_walks_bug(void)
+{
+  return bt_assert_bug(remove_from_multiple_walks_bug, "HASH_REMOVE: Attempt to remove inside multiple hash walks");
+}
+
+static int
+t_delete2_bug(void)
+{
+  return bt_assert_bug(walk_delete2_bug, "HASH_DELETE2 called in hash walk or hash delsafe walk");
+}
+
+static int
+t_insert2_bug(void)
+{
+  return bt_assert_bug(delsafe_insert2_bug, "HASH_INSERT2: called in hash walk or hash delsafe walk");
+}
+
+static int
+t_mixing_walks_bug(void)
+{
+  int ret = 1;
+  ret = ret && bt_assert_bug(walk_different_walks_bug, "HASH_WALK can not be called from other walks");
+  ret = ret && bt_assert_bug(resizable_different_walks_bug, "HASH_WALK_RESIZABLE can not be called from other walks");
+  ret = ret && bt_assert_bug(delsafe_different_walks_bug, "HASH_WALK_DELSAFE can not be called from other walks");
+  return ret;
+}
+
+
 
 int
 main(int argc, char *argv[])
@@ -315,11 +529,19 @@ main(int argc, char *argv[])
   bt_test_suite(t_insert2_find, 	"HASH_INSERT2 and HASH_FIND. HASH_INSERT2 is HASH_INSERT and a smart auto-resize function");
   bt_test_suite(t_walk, 		"HASH_WALK");
   bt_test_suite(t_walk_delsafe_delete, 	"HASH_WALK_DELSAFE and HASH_DELETE");
-  bt_test_suite(t_walk_delsafe_delete2,	"HASH_WALK_DELSAFE and HASH_DELETE2. HASH_DELETE2 is HASH_DELETE and smart auto-resize function");
+  bt_test_suite(t_walk_resizable_delete2,	"HASH_WALK_DELSAFE and HASH_DELETE2. HASH_DELETE2 is HASH_DELETE and smart auto-resize function");
   bt_test_suite(t_walk_delsafe_remove, 	"HASH_WALK_DELSAFE and HASH_REMOVE");
-  bt_test_suite(t_walk_delsafe_remove2,	"HASH_WALK_DELSAFE and HASH_REMOVE2. HASH_REMOVE2 is HASH_REMOVE and smart auto-resize function");
+  bt_test_suite(t_walk_resizable_remove2,	"HASH_WALK_RESIZABLE and HASH_REMOVE2. HASH_REMOVE2 is HASH_REMOVE and smart auto-resize function");
   bt_test_suite(t_walk_filter,		"HASH_WALK_FILTER");
-  bt_test_suite(t_walk_check_bug,	"HASH_DO_REMOVE returns error, because called from HASH_WALK");
+  bt_test_suite(t_walk_check_remove_bug, "HASH_DO_REMOVE returns error, because called from HASH_WALK");
+  bt_test_suite(t_walk_check_delete_bug, "HASH_DO_DELETE returns error, because called from HASH_WALK");
+  bt_test_suite(t_walk_check_end_bug,	"HASH_WALK_DELSAFE_END called when HASH_WALK_DELSAFE is not opened");
+  bt_test_suite(t_delete_from_multiple_walks_bug, "HASH_DELETE called inside multiple hash walks");
+  bt_test_suite(t_remove_from_multiple_walks_bug, "HASH_REMOVE called inside multiple hash walks");
+  bt_test_suite(t_delete2_bug,		"HASH_DELETE2 called inside hash walk");
+  bt_test_suite(t_insert2_bug,		"HASH_INSERT2 called inside delsafe hash walk");
+  bt_test_suite(t_mixing_walks_bug,	"Mixing multiple types of walks");
+  bt_test_suite(t_walk_multilevel,	"HASH_WALK walk inside walk");
 
   return bt_exit_value();
 }

sysdep/unix/main.c

@@ -66,6 +66,17 @@ async_dump(void)
   debug("\n");
 }
 
+#ifndef BACKTRACE_MAX_LINES
+// not in test
+// To be compilatible with some systems, we need to define get_test_bug_jump here
+jmp_buf *
+get_test_bug_jump(const char *msg UNUSED)
+{
+  return NULL;
+}
+#endif
+
+
 /*
  *	Dropping privileges
  */

test/birdtest.c

@@ -445,7 +445,7 @@ bt_assert_bug(void (*functionPtr)(void), char *expected_message)
 }
 
 jmp_buf *
-get_test_bug_jump(char *msg)
+get_test_bug_jump(const char *msg)
 {
   if (!bug_expected || strcmp(msg, expected_bug_message) != 0)
     abort();