Trie Index: Growing works

lib/idm.c

@@ -40,7 +40,7 @@ idm_alloc(struct idm *m)
       goto found;
 
   /* If we are at least 7/8 full, expand (if we are allowed to) */
-  if ((m->used * 32 < m->max) && (m->used > m->size * 28))
+  if ((m->used < m->max) && (m->used > m->size * 28))
   {
     m->size *= 2;
     m->data = mb_realloc(m->data, m->size * sizeof(u32));

lib/tindex.c

@@ -17,6 +17,9 @@
 #define TI_MIN_UNIT_SIZE	4
 #define TI_MIN_ADDRESS_SIZE	6
 
+#define TDB	32
+#define uTDB	u32
+
 union tindex_data {
   u32 data4[0];
   u16 data6[0];
@@ -29,6 +32,7 @@ struct tindex {
   u64 *exists;
   pool *p;
   struct idm idm;
+  u16 depth;
   u8 unit_size;
   u8 address_size;
 };
@@ -43,33 +47,34 @@ tindex_new(pool *p)
   ti->index_data = mb_allocz(p, ti->unit_size * (1 << ti->address_size));
   ti->exists = mb_allocz(p, (1 << (ti->address_size - 3)));
   idm_init(&(ti->idm), p, (1 << (ti->address_size - 5)), (1 << ti->address_size));
-  u32 rootnode = idm_alloc(&(ti->idm));
+  u64 rootnode = idm_alloc(&(ti->idm));
   ASSERT(rootnode == 1);
   return ti;
 }
 
 static inline u64
-tindex_data(const struct tindex *ti, u64 asize, u64 usize, u64 dsize, u64 dshift, u64 idx, uint *len)
+tindex_data(const union tindex_data *id, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 idx, uint *len)
 {
+  ASSERT(dsize <= TDB);
   u64 data;
   switch (usize) {
     case 4:
-      data = ti->index_data->data4[idx] >> dshift;
+      data = id->data4[idx] >> dshift;
       break;
     case 6:
       data =
-	((u64)(ti->index_data->data6[idx * 3] >> asize) << (dshift * 2)) |
+	((u64)(id->data6[idx * 3] >> asize) << (dshift * 2)) |
-	((u64)(ti->index_data->data6[idx * 3 + 1] >> asize) << (dshift)) |
+	((u64)(id->data6[idx * 3 + 1] >> asize) << (dshift)) |
-	(u64)(ti->index_data->data6[idx * 3 + 2] >> asize);
+	(u64)(id->data6[idx * 3 + 2] >> asize);
       break;
     case 8:
-      data = ti->index_data->data8[idx] >> dshift;
+      data = id->data8[idx] >> dshift;
       break;
     case 12:
       data =
-	((u64)(ti->index_data->data12[idx * 3] >> asize) << (dshift * 2)) |
+	((u64)(id->data12[idx * 3] >> asize) << (dshift * 2)) |
-	((u64)(ti->index_data->data12[idx * 3 + 1] >> asize) << (dshift)) |
+	((u64)(id->data12[idx * 3 + 1] >> asize) << (dshift)) |
-	(u64)(ti->index_data->data12[idx * 3 + 2] >> asize);
+	(u64)(id->data12[idx * 3 + 2] >> asize);
       break;
     default:
       bug("This shall never happen");
@@ -86,128 +91,151 @@ tindex_data(const struct tindex *ti, u64 asize, u64 usize, u64 dsize, u64 dshift
 }
 
 static inline u64
-tindex_left(const struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask)
+tindex_left(const union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask)
 {
   switch (usize) {
-    case 4: return (ti->index_data->data4[idx] >> (asize * 2)) & addrmask;
+    case 4: return (id->data4[idx] >> (asize * 2)) & addrmask;
-    case 6: return ti->index_data->data6[idx * 3] & addrmask;
+    case 6: return id->data6[idx * 3] & addrmask;
-    case 8: return (ti->index_data->data8[idx] >> (asize * 2)) & addrmask;
+    case 8: return (id->data8[idx] >> (asize * 2)) & addrmask;
-    case 12: return ti->index_data->data12[idx * 3] & addrmask;
+    case 12: return id->data12[idx * 3] & addrmask;
     default: bug("This shall never happen");
   }
 }
 
 static inline u64
-tindex_right(const struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask)
+tindex_right(const union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask)
 {
   switch (usize) {
-    case 4: return (ti->index_data->data4[idx] >> (asize)) & addrmask;
+    case 4: return (id->data4[idx] >> (asize)) & addrmask;
-    case 6: return ti->index_data->data6[idx * 3 + 1] & addrmask;
+    case 6: return id->data6[idx * 3 + 1] & addrmask;
-    case 8: return (ti->index_data->data8[idx] >> (asize)) & addrmask;
+    case 8: return (id->data8[idx] >> (asize)) & addrmask;
-    case 12: return ti->index_data->data12[idx * 3 + 1] & addrmask;
+    case 12: return id->data12[idx * 3 + 1] & addrmask;
     default: bug("This shall never happen");
   }
 }
 
 static inline u64
-tindex_up(const struct tindex *ti, u64 idx, u64 usize, u64 addrmask)
+tindex_up(const union tindex_data *id, u64 idx, u64 usize, u64 addrmask)
 {
   switch (usize) {
-    case 4: return ti->index_data->data4[idx] & addrmask;
+    case 4: return id->data4[idx] & addrmask;
-    case 6: return ti->index_data->data6[idx * 3 + 2] & addrmask;
+    case 6: return id->data6[idx * 3 + 2] & addrmask;
-    case 8: return ti->index_data->data8[idx] & addrmask;
+    case 8: return id->data8[idx] & addrmask;
-    case 12: return ti->index_data->data12[idx * 3 + 2] & addrmask;
+    case 12: return id->data12[idx * 3 + 2] & addrmask;
     default: bug("This shall never happen");
   }
 }
 
 static inline void
-tindex_put(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 data, uint dlen, u64 left, u64 right, u64 up)
+tindex_put(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 data, uint dlen, u64 left, u64 right, u64 up)
 {
   const u64 dsmask = (1LL << dshift) - 1;
   data = u64_var_encode(data, dsize - dlen);
 
   switch (usize) {
     case 4:
-      ti->index_data->data4[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up;
+      id->data4[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up;
       return;
     case 6:
-      ti->index_data->data6[idx * 3    ] = left  | ((data >> (2 * dshift)) << asize);
+      id->data6[idx * 3    ] = left  | ((data >> (2 * dshift)) << asize);
-      ti->index_data->data6[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize);
+      id->data6[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize);
-      ti->index_data->data6[idx * 3 + 2] = up    | ((data & dsmask) << asize);
+      id->data6[idx * 3 + 2] = up    | ((data & dsmask) << asize);
       return;
     case 8:
-      ti->index_data->data8[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up;
+      id->data8[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up;
       return;
     case 12:
-      ti->index_data->data12[idx * 3    ] = left  | ((data >> (2 * dshift)) << asize);
+      id->data12[idx * 3    ] = left  | ((data >> (2 * dshift)) << asize);
-      ti->index_data->data12[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize);
+      id->data12[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize);
-      ti->index_data->data12[idx * 3 + 2] = up    | ((data & dsmask) << asize);
+      id->data12[idx * 3 + 2] = up    | ((data & dsmask) << asize);
       return;
     default: bug("This shall never happen");
   }
 }
 
 static inline void
-tindex_left_clear(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask)
+tindex_left_clear(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask)
 {
   switch (usize) {
-    case 4: ti->index_data->data4[idx] &= ~(addrmask << (asize * 2)); break;
+    case 4: id->data4[idx] &= ~(addrmask << (asize * 2)); break;
-    case 6: ti->index_data->data6[idx * 3] &= ~addrmask; break;
+    case 6: id->data6[idx * 3] &= ~addrmask; break;
-    case 8: ti->index_data->data8[idx] &= ~(addrmask << (asize * 2)); break;
+    case 8: id->data8[idx] &= ~(addrmask << (asize * 2)); break;
-    case 12: ti->index_data->data6[idx * 3] &= ~addrmask; break;
+    case 12: id->data12[idx * 3] &= ~addrmask; break;
   }
 }
 
 static inline void
-tindex_right_clear(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask)
+tindex_right_clear(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask)
 {
   switch (usize) {
-    case 4: ti->index_data->data4[idx] &= ~(addrmask << asize); break;
+    case 4: id->data4[idx] &= ~(addrmask << asize); break;
-    case 6: ti->index_data->data6[idx * 3 + 1] &= ~addrmask; break;
+    case 6: id->data6[idx * 3 + 1] &= ~addrmask; break;
-    case 8: ti->index_data->data8[idx] &= ~(addrmask << asize); break;
+    case 8: id->data8[idx] &= ~(addrmask << asize); break;
-    case 12: ti->index_data->data6[idx * 3 + 1] &= ~addrmask; break;
+    case 12: id->data12[idx * 3 + 1] &= ~addrmask; break;
   }
 }
 
 static inline void
-tindex_left_set(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 nidx)
+tindex_up_clear(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask)
+{
+  switch (usize) {
+    case 4: id->data4[idx] &= ~addrmask; break;
+    case 6: id->data6[idx * 3 + 2] &= ~addrmask; break;
+    case 8: id->data8[idx] &= ~addrmask; break;
+    case 12: id->data12[idx * 3 + 2] &= ~addrmask; break;
+  }
+}
+
+static inline void
+tindex_left_set(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 nidx)
 {
   /* The left child must have been zero before */
   switch (usize) {
-    case 4: ti->index_data->data4[idx] |= nidx << (asize * 2); break;
+    case 4: id->data4[idx] |= nidx << (asize * 2); break;
-    case 6: ti->index_data->data6[idx * 3] |= nidx; break;
+    case 6: id->data6[idx * 3] |= nidx; break;
-    case 8: ti->index_data->data8[idx] |= nidx << (asize * 2); break;
+    case 8: id->data8[idx] |= nidx << (asize * 2); break;
-    case 12: ti->index_data->data6[idx * 3] |= nidx; break;
+    case 12: id->data12[idx * 3] |= nidx; break;
   }
 }
 
 static inline void
-tindex_right_set(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 nidx)
+tindex_right_set(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 nidx)
 {
   /* The right child must have been zero before */
   switch (usize) {
-    case 4: ti->index_data->data4[idx] |= nidx << asize; break;
+    case 4: id->data4[idx] |= nidx << asize; break;
-    case 6: ti->index_data->data6[idx * 3 + 1] |= nidx; break;
+    case 6: id->data6[idx * 3 + 1] |= nidx; break;
-    case 8: ti->index_data->data8[idx] |= nidx << asize; break;
+    case 8: id->data8[idx] |= nidx << asize; break;
-    case 12: ti->index_data->data6[idx * 3 + 1] |= nidx; break;
+    case 12: id->data12[idx * 3 + 1] |= nidx; break;
   }
 }
 
 static inline void
-tindex_child_update(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask, u64 oidx, u64 nidx)
+tindex_up_set(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 nidx)
 {
-  if (oidx == tindex_left(ti, idx, usize, asize, addrmask)) {
+  /* The parent must have been zero before */
-    tindex_left_clear(ti, idx, usize, asize, addrmask);
+  switch (usize) {
-    tindex_left_set(ti, idx, usize, asize, nidx);
+    case 4: id->data4[idx] |= nidx; break;
-  } else {
+    case 6: id->data6[idx * 3 + 2] |= nidx; break;
-    ASSERT(oidx == tindex_right(ti, idx, usize, asize, addrmask));
+    case 8: id->data8[idx] |= nidx; break;
-    tindex_right_clear(ti, idx, usize, asize, addrmask);
+    case 12: id->data12[idx * 3 + 2] |= nidx; break;
-    tindex_right_set(ti, idx, usize, asize, nidx);
   }
 }
 
-static inline uint tindex_input_bits(const u64 *bits_in, const uint blen, uint *bpos, const uint dlen, u64 *bits) {
+static inline void
+tindex_child_update(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask, u64 oidx, u64 nidx)
+{
+  if (oidx == tindex_left(id, idx, usize, asize, addrmask)) {
+    tindex_left_clear(id, idx, usize, asize, addrmask);
+    tindex_left_set(id, idx, usize, asize, nidx);
+  } else {
+    ASSERT(oidx == tindex_right(id, idx, usize, asize, addrmask));
+    tindex_right_clear(id, idx, usize, asize, addrmask);
+    tindex_right_set(id, idx, usize, asize, nidx);
+  }
+}
+
+static inline uint tindex_input_bits(const uTDB *bits_in, const uint blen, uint *bpos, const uint dlen, u64 *bits) {
   uint bmax = blen - *bpos;	/* How much remains in the input */
   uint ilen = MIN(bmax, dlen);	/* How much we really take */
 
@@ -216,23 +244,23 @@ static inline uint tindex_input_bits(const u64 *bits_in, const uint blen, uint *
     return 0;
   }
 
-  ASSERT(ilen <= 64);		/* The limit of output bit count is 64 */
+  ASSERT(ilen <= TDB);		/* The limit of output bit count is TDB */
   uint bend = *bpos + ilen - 1;	/* The last bit, inclusive (!) */
 
   /* Crop the bits at the end */
-  *bits = (bits_in[bend / 64] >> (63 - (bend % 64)));
+  *bits = (bits_in[bend / TDB] >> (TDB - 1 - (bend % TDB)));
 
   /* Prepend bits from the previous item if the range goes over */
-  if (bend / 64 > *bpos / 64)
+  if (bend / TDB > *bpos / TDB)
-    *bits |= bits_in[*bpos / 64] << (1 + bend % 64);
+    *bits |= bits_in[*bpos / TDB] << (1 + bend % TDB);
   else
-    ASSERT(bend / 64 == *bpos / 64);
+    ASSERT(bend / TDB == *bpos / TDB);
 
   /* Advance the bit pointer */
   *bpos += ilen;
 
   /* Return the wanted bits */
-  *bits &= ((1 << ilen) - 1);
+  *bits &= ((1ULL << ilen) - 1);
   return ilen;
 }
 
@@ -264,6 +292,8 @@ _tindex_dump(const struct tindex *ti, u64 idx, uint depth, uint bit)
   const uint usize = ti->unit_size;
   const uint dsize = usize * 8 - asize * 3;
 
+  union tindex_data *idata = ti->index_data;
+
   const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3);
   const u64 addrmask = (1ULL << ti->address_size) - 1;
 
@@ -277,18 +307,18 @@ _tindex_dump(const struct tindex *ti, u64 idx, uint depth, uint bit)
   }
 
   uint dlen;
-  u64 data = tindex_data(ti, asize, usize, dsize, dshift, idx, &dlen);
+  u64 data = tindex_data(idata, usize, asize, dsize, dshift, idx, &dlen);
   if (depth && bit)
     data |= 1ULL << dlen;
   if (depth)
     dlen++;
 
   debug("%s0x%x/%u (%lu %c)\n", INDENT, data, dlen, idx, tindex_exists(ti, idx) ? '*' : ' ');
-  u64 left = tindex_left(ti, idx, usize, asize, addrmask);
+  u64 left = tindex_left(idata, idx, usize, asize, addrmask);
   if (left)
     _tindex_dump(ti, left, depth+1, 0);
 
-  u64 right = tindex_right(ti, idx, usize, asize, addrmask);
+  u64 right = tindex_right(idata, idx, usize, asize, addrmask);
   if (right)
     _tindex_dump(ti, right, depth+1, 1);
 }
@@ -296,11 +326,14 @@ _tindex_dump(const struct tindex *ti, u64 idx, uint depth, uint bit)
 void
 tindex_dump(const struct tindex *ti)
 {
+  debug("Trie index; usize = %u, asize = %u, dsize = %u, depth = %u\n",
+      ti->unit_size, ti->address_size, ti->unit_size * 8 - ti->address_size * 3,
+      ti->depth);
   _tindex_dump(ti, 1, 0, 0);
 }
 
-u64
+void
-tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int create)
+tindex_do_grow(struct tindex *ti, const uint nasize, const uint nusize)
 {
   const uint asize = ti->address_size;
   const uint usize = ti->unit_size;
@@ -309,6 +342,193 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr
   const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3);
   const u64 addrmask = (1ULL << ti->address_size) - 1;
 
+  ti->unit_size = nusize;
+  ti->address_size = nasize;
+
+  union tindex_data *odata = ti->index_data;
+  ti->index_data = mb_allocz(ti->p, nusize * (1 << nasize));
+
+  u64 *oexists = ti->exists;
+  ti->exists = mb_allocz(ti->p, (1 << (nasize - 3)));
+  memcpy(ti->exists, oexists, 1 << (asize - 3));
+  mb_free(oexists);
+
+  ti->idm.max = 1 << nasize;
+
+  uint bpos = 0;
+  uTDB *bits = alloca(((ti->depth / TDB) + 1)*sizeof(uTDB));
+  memset(bits, 0, ((ti->depth / TDB) + 1)*sizeof(uTDB));
+
+  void migrate(u64 idx) {
+    uint dlen;
+    u64 data = tindex_data(odata, usize, asize, dsize, dshift, idx, &dlen);
+    u64 mask = (1 << dlen) - 1;
+    uint sbpos = bpos;
+    if (dlen) {
+      uint bend = bpos + dlen - 1;
+
+      if (bend / TDB > bpos / TDB) {
+	bits[bpos / TDB] &= ~(mask >> (1 + bend % TDB));
+	bits[bpos / TDB] |= data >> (1 + bend % TDB);
+      }
+
+      bits[bend / TDB] &= ~(mask << (TDB - 1 - (bend % TDB)));
+      bits[bend / TDB] |= data << (TDB - 1 - (bend % TDB));
+
+      bpos = bend + 1;
+    }
+
+    /* Migration of non-root nodes */
+    if (idx > 1)
+      if (tindex_exists(ti, idx))
+	tindex_find(ti, bits, bpos, idx);
+      else
+	idm_free(&(ti->idm), idx);
+
+    u64 left = tindex_left(odata, idx, usize, asize, addrmask);
+    if (left) {
+      bits[bpos / TDB] &= ~(1ULL << (TDB - 1 - (bpos % TDB)));
+      bpos++;
+      migrate(left);
+      bpos--;
+    }
+
+    u64 right = tindex_right(odata, idx, usize, asize, addrmask);
+    if (right) {
+      bits[bpos / TDB] |= 1ULL << (TDB - 1 - (bpos % TDB));
+      bpos++;
+      migrate(right);
+      bpos--;
+    }
+
+    bpos = sbpos;
+  }
+
+  migrate(1);
+  mb_free(odata);
+}
+
+void
+tindex_grow(struct tindex *ti)
+{
+  /* We want bigger index space so we have to change parameters
+   * of the tindex and completely rebuild it. Then we'll free the
+   * old index_data.
+   *
+   * Assigned indices are kept, internal nodes may be rearranged
+   * and renumbered.
+   */
+
+  const uint asize = ti->address_size;
+  const uint usize = ti->unit_size;
+  const uint dsize = usize * 8 - asize * 3;
+  const union tindex_data *idata = ti->index_data;
+
+  const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3);
+  const u64 addrmask = (1ULL << ti->address_size) - 1;
+
+  if (dsize > 3) {
+    /* First we'll try to estimate whether it is feasible to shorten
+     * the data part while getting more space for the indices */
+
+    u64 needsplit = 0;
+    u64 total = 0;
+
+    void lencnt(u64 idx) {
+      uint dlen;
+      tindex_data(idata, usize, asize, dsize, dshift, idx, &dlen);
+      ASSERT(dlen < dsize);
+      if (dlen >= dsize - 3)
+	needsplit++;
+      total++;
+
+      u64 left = tindex_left(idata, idx, usize, asize, addrmask);
+      if (left)
+	lencnt(left);
+
+      u64 right = tindex_right(idata, idx, usize, asize, addrmask);
+      if (right)
+	lencnt(right);
+    }
+
+    lencnt(1);
+
+    /* After shortening the data part, needsplit/total nodes will duplicate (or triplicate!).
+     * If the overall index usage goes up by at most 20% by doing this change,
+     * we consider it feasible. By math:
+     *
+     * ((float)(needsplit / total)) * ((int)(dsize / (dsize - 3)) + 1) < 0.2
+     * needsplit * ((dsize / (dsize - 3)) + 1) < 0.2 * total
+     * 5 * needsplit * ((dsize / (dsize - 3)) + 1) < total
+     */
+
+    if (5 * needsplit * ((dsize / (dsize - 3)) + 1) < total)
+      return tindex_do_grow(ti, asize + 1, usize);
+  }
+
+  switch (usize) {
+#define UP_ASIZE(usize) (1+MAX((((usize-4)*8)/3),asize))
+    case 4: return tindex_do_grow(ti, UP_ASIZE(6), 6);
+    case 6: return tindex_do_grow(ti, UP_ASIZE(8), 8);
+    case 8: return tindex_do_grow(ti, UP_ASIZE(12), 12);
+    case 12: bug("Not implemented yet.");
+    default: bug("This shall not happen.");
+  }
+}
+
+static inline void
+tindex_renumber(union tindex_data *idata, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 addrmask, u64 oidx, u64 nidx)
+{
+  u64 up = tindex_up(idata, oidx, usize, addrmask);
+  u64 left = tindex_left(idata, oidx, usize, asize, addrmask);
+  u64 right = tindex_right(idata, oidx, usize, asize, addrmask);
+
+  if (up)
+    tindex_child_update(idata, up, usize, asize, addrmask, oidx, nidx);
+
+  if (left) {
+    tindex_up_clear(idata, left, usize, asize, addrmask);
+    tindex_up_set(idata, left, usize, asize, nidx);
+  }
+
+  if (right) {
+    tindex_up_clear(idata, right, usize, asize, addrmask);
+    tindex_up_set(idata, right, usize, asize, nidx);
+  }
+
+  switch (usize) {
+    case 4: idata->data4[nidx] = idata->data4[oidx];
+	    break;
+    case 6: memcpy(&(idata->data6[nidx * 3]), &(idata->data6[oidx * 3]), 3*sizeof(idata->data6[0]));
+	    break;
+    case 8: idata->data8[nidx] = idata->data8[oidx];
+	    break;
+    case 12: memcpy(&(idata->data12[nidx * 3]), &(idata->data12[oidx * 3]), 3*sizeof(idata->data12[0]));
+	    break;
+    default: bug("This shall never happen");
+  }
+}
+
+#define TINDEX_ALLOC_IDX ({ u64 out = idm_alloc(&(ti->idm)); if (!out) goto noidx; out; })
+
+u64
+tindex_find(struct tindex *ti, const uTDB *bits_in, const uint blen, const u64 create)
+{
+  if (blen > ti->depth)
+    if (create)
+      ti->depth = blen;
+    else
+      return 0;
+
+  const uint asize = ti->address_size;
+  const uint usize = ti->unit_size;
+  const uint dsize = usize * 8 - asize * 3;
+
+  union tindex_data *idata = ti->index_data;
+
+  const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3);
+  const u64 addrmask = (1ULL << ti->address_size) - 1;
+
   /* Validate unit size */
   switch (usize) {
     case 4:
@@ -326,7 +546,7 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr
   while (1) {
     /* Get data from trie */
     uint dlen;
-    u64 data = tindex_data(ti, asize, usize, dsize, dshift, idx, &dlen);
+    u64 data = tindex_data(idata, usize, asize, dsize, dshift, idx, &dlen);
 
     /* Get data from input */
     u64 bits;
@@ -346,13 +566,20 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr
 
       /* No more bits, we're done */
       if (!ilen) {
-	/* Existence bits fiddling */
+	if (create == TINDEX_CREATE) {
-	if (create)
+	  /* Creating at any index -> do it */
 	  tindex_exists_set(ti, idx);
-	else if (!tindex_exists(ti, idx))
+	  return idx;
+	} else if (create) {
+	  /* Migration from old version -> renumber */
+	  tindex_renumber(idata, usize, asize, dsize, dshift, addrmask, idx, create);
+	  idm_free(&(ti->idm), idx);
+	  return create;
+	} else if (tindex_exists(ti, idx))
+	  /* Shan't create but it already exists */
+	  return idx;
+	else
 	  return 0;
-
-	return idx;
       }
 
       /* Just one bit, to be sure */
@@ -360,7 +587,7 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr
       ASSERT(ilen == 1);
 
       /* Go left or right? */
-      u64 nidx = bits ? tindex_right(ti, idx, usize, asize, addrmask) : tindex_left(ti, idx, usize, asize, addrmask);
+      u64 nidx = bits ? tindex_right(idata, idx, usize, asize, addrmask) : tindex_left(idata, idx, usize, asize, addrmask);
 
       /* There is a path, we'll follow it. */
       if (nidx) {
@@ -374,13 +601,13 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr
 	return 0;
 
       /* So there will be a new node on path. */
-      nidx = idm_alloc(&(ti->idm));
+      nidx = TINDEX_ALLOC_IDX;
 
       /* Left or right? */
       if (bits)
-	tindex_right_set(ti, idx, usize, asize, nidx);
+	tindex_right_set(idata, idx, usize, asize, nidx);
       else
-	tindex_left_set(ti, idx, usize, asize, nidx);
+	tindex_left_set(idata, idx, usize, asize, nidx);
 
       /* Go there. */
       uidx = idx;
@@ -416,31 +643,41 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr
     data &= (1 << dlen) - 1;
 
     /* Allocate the splitting index */
-    u64 midx = idm_alloc(&(ti->idm));
+    u64 midx = TINDEX_ALLOC_IDX;
 
     /* Allocate the new node if it shall exist */
-    u64 nidx = split ? idm_alloc(&(ti->idm)) : 0;
+    u64 nidx = split ? TINDEX_ALLOC_IDX : 0;
 
     /* Relink idx -> midx in the parent node */
     if (uidx)
-      tindex_child_update(ti, uidx, usize, asize, addrmask, idx, midx);
+      tindex_child_update(idata, uidx, usize, asize, addrmask, idx, midx);
 
     /* Setup the splitting index (midx) */
-    tindex_put(ti, midx, usize, asize, dsize, dshift, common, comlen, dataright ? nidx : idx, dataright ? idx : nidx, uidx);
+    tindex_put(idata, midx, usize, asize, dsize, dshift, common, comlen, dataright ? nidx : idx, dataright ? idx : nidx, uidx);
 
     /* Update the existing index (idx) */
-    tindex_put(ti, idx, usize, asize, dsize, dshift, data, dlen, tindex_left(ti, idx, usize, asize, addrmask), tindex_right(ti, idx, usize, asize, addrmask), midx);
+    tindex_put(idata, idx, usize, asize, dsize, dshift, data, dlen, tindex_left(idata, idx, usize, asize, addrmask), tindex_right(idata, idx, usize, asize, addrmask), midx);
 
-    /* Go down to the child */
+    if (split) {
-    uidx = idx;
+      /* The new parent is the splitting node */
-    idx = nidx;
+      uidx = midx;
 
-    /* Grow there a branch if it has to be grown, otherwise return */
+      /* The current node is the newly allocated */
-    if (split)
+      idx = nidx;
+
+      /* Grow there a branch */
       break;
-    else {
+
+    } else if (create == TINDEX_CREATE) {
+      /* This internal node exists */
       tindex_exists_set(ti, midx);
       return midx;
+
+    } else {
+      /* This internal node must be renumbered to the right one */
+      tindex_renumber(idata, usize, asize, dsize, dshift, addrmask, midx, create);
+      idm_free(&(ti->idm), midx);
+      return create;
     }
   }
 
@@ -455,22 +692,43 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr
 
     /* End of input data */
     if ((ilen < dsize - 1) || !tindex_input_bits(bits_in, blen, &bpos, 1, &dataright)) {
-      tindex_put(ti, idx, usize, asize, dsize, dshift, data, ilen, 0, 0, uidx);
+      tindex_put(idata, idx, usize, asize, dsize, dshift, data, ilen, 0, 0, uidx);
-      tindex_exists_set(ti, idx);
+      if (create == TINDEX_CREATE) {
-      return idx;
+	tindex_exists_set(ti, idx);
+	return idx;
+      } else {
+	tindex_renumber(idata, usize, asize, dsize, dshift, addrmask, idx, create);
+	return create;
+      }
     }
 
     /* Just one bit. */
     ASSERT(dataright < 2);
 
     /* Create a new node */
-    uint nidx = idm_alloc(&(ti->idm));
+    uint nidx = TINDEX_ALLOC_IDX;
 
     /* Link it into the trie */
-    tindex_put(ti, idx, usize, asize, dsize, dshift, data, ilen, dataright ? 0 : nidx, dataright ? nidx : 0, uidx);
+    tindex_put(idata, idx, usize, asize, dsize, dshift, data, ilen, dataright ? 0 : nidx, dataright ? nidx : 0, uidx);
 
     /* And continue there */
     uidx = idx;
     idx = nidx;
   }
+
+  /* This statement should be unreachable */
+  ASSERT(0);
+
+  /* No index available for alloc */
+noidx:
+  /* This may happen only directly while adding.
+   * It should never hapṕen when growing.
+   * */
+  ASSERT(create == TINDEX_CREATE);
+
+  /* Grow the tindex */
+  tindex_grow(ti);
+
+  /* And retry */
+  return tindex_find(ti, bits_in, blen, create);
 }

lib/tindex.h

@@ -17,15 +17,30 @@
  */
 struct tindex* tindex_new(pool *p);
 
+#define TINDEX_CREATE	(~(0ULL))
+#define TINDEX_FIND	0
+
 /**
- * Find an index by the auxiliary funcction @tib.
+ * Find an index
- * @t: the index to look into
+ * @ti: the tindex to look into
- * @tib: the auxiliary function; see before
+ * @bits_in: data
- * @create: 0 to find only existing records, 1 to create new
+ * @blen: number of bits to extract from bits_in.
- * Return value: 0 for not found (create == 0) or retry (create == 1); nonzero = the index
+ *	  If @blen is not multiple of 64, the LSB's of the last u64 are ignored.
+ * @create: TINDEX_FIND to find existing, TINDEX_CREATE to create new records,
+ *	    every other value is for internal use
+ *
+ * Return value: 0 for not found; nonzero = the index
  */
 
-u64 tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int create);
+u64 tindex_find(struct tindex *ti, const u32 *bits_in, const uint blen, const u64 create);
+
+/**
+ * Delete an index.
+ * @ti: the tindex to use
+ * @idx: the index to delete
+ **/
+
+u64 tindex_delete(struct tindex *ti, const u64 idx);
 
 /**
  * Dump the index. Useful for debugging.

lib/tindex_test.c

@@ -18,11 +18,12 @@ struct test_trie {
 };
 
 static inline void test_trie_add(struct test_trie *tt, u64 data) {
-  u64 idx = tindex_find(tt->ti, &data, 64, 1);
+  u32 dtb[2] = { data >> 32, data };
+  u64 idx = tindex_find(tt->ti, dtb, 64, TINDEX_CREATE);
   bt_assert(idx > 0);
 
   u64 nlen = tt->len;
-  while (idx > nlen)
+  while (idx >= nlen)
     nlen *= 2;
 
   if (nlen > tt->len) {
@@ -34,15 +35,29 @@ static inline void test_trie_add(struct test_trie *tt, u64 data) {
   tt->data[idx]++;
 }
 
-static inline u64 test_trie_get(struct test_trie *tt, u64 data) {
+static inline void test_trie_get(struct test_trie *tt, u64 data, u64 cnt) {
-  u64 idx = tindex_find(tt->ti, &data, 64, 0);
+  u64 out = 0;
-  if (!idx) return 0;
+  u32 dtb[2] = { data >> 32, data };
-  return tt->data[idx];
+  u64 idx = tindex_find(tt->ti, dtb, 64, TINDEX_FIND);
+  if (idx) out = tt->data[idx];
+  bt_assert_msg(out == cnt, "Index %lu shall be in trie %lu times, is %lu times.", data, cnt, out);
 }
 
+/*
+static inline void test_trie_remove(struct test_trie *tt, u64 data) {
+  u64 idx = tindex_find(tt->ti, &data, 64, TINDEX_FIND);
+  ASSERT(idx);
+  ASSERT(tt->data[idx]);
+  if (!--tt->data[idx])
+    tindex_delete(tt->ti, idx);
+}
+*/
+
 static int
 t_simple(void)
 {
+  const u64 mul = 0xf906f046b1fd4863ULL;
+  const u64 add = 0xb3a35ec46d09489bULL;
   pool *p = rp_new(&root_pool, "tindex test");
   struct test_trie tt = {
     .ti = tindex_new(p),
@@ -51,14 +66,23 @@ t_simple(void)
   };
  
   bt_assert(tt.ti);
-  for (u64 i = 0; i < 20; i++) {
+  for (u64 i = 0; i < (1<<16); i++) {
     bt_debug("Trie add: %lu\n", i);
     test_trie_add(&tt, i);
-    tindex_dump(tt.ti);
+    test_trie_add(&tt, i * mul + add);
   }
 
+  for (u64 i = 0; i < (1<<16); i++) {
+    test_trie_get(&tt, i, 1);
+    test_trie_get(&tt, i * mul + add, 1);
+  }
+
+  /*
   for (u64 i = 0; i < 20; i++)
-    bt_assert_msg(test_trie_get(&tt, i) == 1, "Index %lu shall be in trie", i);
+    test_trie_remove(&tt, i);
+    */
+
+  tindex_dump(tt.ti);
 
   return 1;
 }